See
<https://builds.apache.org/job/HBase-1.3/jdk=latest1.7,label=Hadoop/440/changes>
Changes:
[busbey] HBASE-14838 Add clarification docs to SimpleRegionNormalizer.
------------------------------------------
[...truncated 46338 lines...]
at java.lang.Thread.run(Thread.java:745)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@533377b" #37 daemon prio=5
os_prio=0 tid=0x00007fb3a526f000 nid=0x2c3d waiting on condition
[0x00007fb30da0f000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:180)
at java.lang.Thread.run(Thread.java:745)
"IPC Server idle connection scanner for port 32879" #35 daemon prio=5 os_prio=0
tid=0x00007fb3a524d000 nid=0x2c21 in Object.wait() [0x00007fb30db10000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007168332d0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Socket Reader #1 for port 32879" #34 prio=5 os_prio=0 tid=0x00007fb3a5218800
nid=0x2c18 runnable [0x00007fb30dc11000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000716846788> (a sun.nio.ch.Util$2)
- locked <0x0000000716846778> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000716828e68> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0x0000000716835768> (a
org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"Timer-0" #30 daemon prio=5 os_prio=0 tid=0x00007fb3a511d000 nid=0x2b7b in
Object.wait() [0x00007fb30df12000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x0000000716836e70> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"851912430@qtp-462039519-1" #29 daemon prio=5 os_prio=0 tid=0x00007fb3a5105000
nid=0x2b53 in Object.wait() [0x00007fb30e013000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
- locked <0x0000000715c34dd8> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)
"240630125@qtp-462039519-0 - Acceptor0
HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54052" #28 daemon
prio=5 os_prio=0 tid=0x00007fb3a5104000 nid=0x2b52 runnable [0x00007fb30f087000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x0000000715c358c8> (a sun.nio.ch.Util$2)
- locked <0x0000000715c358b8> (a java.util.Collections$UnmodifiableSet)
- locked <0x0000000715c354e8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
"Timer for 'NameNode' metrics system" #26 daemon prio=5 os_prio=0
tid=0x00007fb3a509e000 nid=0x2b1e in Object.wait() [0x00007fb30e514000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x00000007110512f0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Service Thread" #17 daemon prio=9 os_prio=0 tid=0x00007fb3a42ce800 nid=0x2876
runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread11" #16 daemon prio=9 os_prio=0 tid=0x00007fb3a42c9800
nid=0x286d waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread10" #15 daemon prio=9 os_prio=0 tid=0x00007fb3a42c7800
nid=0x286b waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread9" #14 daemon prio=9 os_prio=0 tid=0x00007fb3a42c6000
nid=0x286a waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread8" #13 daemon prio=9 os_prio=0 tid=0x00007fb3a42c3800
nid=0x2869 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread7" #12 daemon prio=9 os_prio=0 tid=0x00007fb3a42c1800
nid=0x2868 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread6" #11 daemon prio=9 os_prio=0 tid=0x00007fb3a42bf800
nid=0x2867 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread5" #10 daemon prio=9 os_prio=0 tid=0x00007fb3a42bd000
nid=0x2866 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread4" #9 daemon prio=9 os_prio=0 tid=0x00007fb3a42b3000
nid=0x2865 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread3" #8 daemon prio=9 os_prio=0 tid=0x00007fb3a42b1000
nid=0x2864 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #7 daemon prio=9 os_prio=0 tid=0x00007fb3a42ac800
nid=0x2863 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #6 daemon prio=9 os_prio=0 tid=0x00007fb3a42aa800
nid=0x2862 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 tid=0x00007fb3a42a7800
nid=0x2861 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" #4 daemon prio=9 os_prio=0 tid=0x00007fb3a42a5800
nid=0x2860 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fb3a4279000 nid=0x2852 in
Object.wait() [0x00007fb35556b000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x000000071107cd00> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fb3a4277000
nid=0x2851 in Object.wait() [0x00007fb35566c000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
- locked <0x00000007110856d8> (a java.lang.ref.Reference$Lock)
"main" #1 prio=5 os_prio=0 tid=0x00007fb3a400a800 nid=0x2832 in Object.wait()
[0x00007fb3aa35a000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1253)
- locked <0x0000000717a7de30> (a
org.apache.hadoop.hbase.util.JVMClusterUtil$RegionServerThread)
at
org.apache.hadoop.hbase.util.Threads.threadDumpingIsAlive(Threads.java:131)
at
org.apache.hadoop.hbase.LocalHBaseCluster.join(LocalHBaseCluster.java:423)
at
org.apache.hadoop.hbase.MiniHBaseCluster.waitUntilShutDown(MiniHBaseCluster.java:758)
at
org.apache.hadoop.hbase.HBaseTestingUtility.shutdownMiniHBaseCluster(HBaseTestingUtility.java:1161)
at
org.apache.hadoop.hbase.HBaseTestingUtility.shutdownMiniCluster(HBaseTestingUtility.java:1120)
at
org.apache.hadoop.hbase.client.TestFromClientSide3.tearDownAfterClass(TestFromClientSide3.java:83)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:108)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:78)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54)
at
org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:144)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
"VM Thread" os_prio=0 tid=0x00007fb3a4271800 nid=0x284b runnable
"GC task thread#0 (ParallelGC)" os_prio=0 tid=0x00007fb3a401f800 nid=0x2835
runnable
"GC task thread#1 (ParallelGC)" os_prio=0 tid=0x00007fb3a4021800 nid=0x2836
runnable
"GC task thread#2 (ParallelGC)" os_prio=0 tid=0x00007fb3a4023000 nid=0x2837
runnable
"GC task thread#3 (ParallelGC)" os_prio=0 tid=0x00007fb3a4025000 nid=0x2838
runnable
"GC task thread#4 (ParallelGC)" os_prio=0 tid=0x00007fb3a4026800 nid=0x2839
runnable
"GC task thread#5 (ParallelGC)" os_prio=0 tid=0x00007fb3a4028800 nid=0x283a
runnable
"GC task thread#6 (ParallelGC)" os_prio=0 tid=0x00007fb3a402a000 nid=0x283b
runnable
"GC task thread#7 (ParallelGC)" os_prio=0 tid=0x00007fb3a402c000 nid=0x283c
runnable
"GC task thread#8 (ParallelGC)" os_prio=0 tid=0x00007fb3a402d800 nid=0x283d
runnable
"GC task thread#9 (ParallelGC)" os_prio=0 tid=0x00007fb3a402f800 nid=0x283e
runnable
"GC task thread#10 (ParallelGC)" os_prio=0 tid=0x00007fb3a4031000 nid=0x283f
runnable
"GC task thread#11 (ParallelGC)" os_prio=0 tid=0x00007fb3a4033000 nid=0x2840
runnable
"GC task thread#12 (ParallelGC)" os_prio=0 tid=0x00007fb3a4034800 nid=0x2841
runnable
"VM Periodic Task Thread" os_prio=0 tid=0x00007fb3a42d1800 nid=0x287f waiting
on condition
JNI global references: 281
Found one Java-level deadlock:
=============================
"asf905.gq1.ygridcore.net,58035,1450285004553_ChoreService_1":
waiting to lock monitor 0x00007fb37c006e68 (object 0x0000000716b5cc70, a
org.apache.hadoop.hbase.ChoreService),
which is held by "RS:1;asf905:58035"
"RS:1;asf905:58035":
waiting to lock monitor 0x00007fb38853e0a8 (object 0x0000000716b5d1c0, a
org.apache.hadoop.hbase.regionserver.HRegionServer$PeriodicMemstoreFlusher),
which is held by "asf905.gq1.ygridcore.net,58035,1450285004553_ChoreService_1"
Java stack information for the threads listed above:
===================================================
"asf905.gq1.ygridcore.net,58035,1450285004553_ChoreService_1":
at
org.apache.hadoop.hbase.ChoreService.cancelChore(ChoreService.java:189)
- waiting to lock <0x0000000716b5cc70> (a
org.apache.hadoop.hbase.ChoreService)
at
org.apache.hadoop.hbase.ScheduledChore.cancel(ScheduledChore.java:266)
- locked <0x0000000716b5d1c0> (a
org.apache.hadoop.hbase.regionserver.HRegionServer$PeriodicMemstoreFlusher)
at org.apache.hadoop.hbase.ScheduledChore.run(ScheduledChore.java:177)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at
org.apache.hadoop.hbase.JitterScheduledThreadPoolExecutorImpl$JitteredRunnableScheduledFuture.run(JitterScheduledThreadPoolExecutorImpl.java:97)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"RS:1;asf905:58035":
at
org.apache.hadoop.hbase.ScheduledChore.getName(ScheduledChore.java:272)
- waiting to lock <0x0000000716b5d1c0> (a
org.apache.hadoop.hbase.regionserver.HRegionServer$PeriodicMemstoreFlusher)
at
org.apache.hadoop.hbase.ScheduledChore.toString(ScheduledChore.java:344)
at java.lang.String.valueOf(String.java:2994)
at java.lang.StringBuilder.append(StringBuilder.java:131)
at java.util.AbstractCollection.toString(AbstractCollection.java:462)
at java.lang.String.valueOf(String.java:2994)
at java.lang.StringBuilder.append(StringBuilder.java:131)
at org.apache.hadoop.hbase.ChoreService.shutdown(ChoreService.java:323)
- locked <0x0000000716b5cc70> (a org.apache.hadoop.hbase.ChoreService)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.stopServiceThreads(HRegionServer.java:2127)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1084)
at
org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(MiniHBaseCluster.java:156)
at
org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$000(MiniHBaseCluster.java:108)
at
org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$1.run(MiniHBaseCluster.java:140)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:360)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1594)
at
org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:334)
at
org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.run(MiniHBaseCluster.java:138)
at java.lang.Thread.run(Thread.java:745)
Found 1 deadlock.
+ exit 1
POST BUILD TASK : FAILURE
END OF POST BUILD TASK : 0
Archiving artifacts
Recording test results
