See <https://builds.apache.org/job/HBase-TRUNK/5572/changes>
Changes:
[stack] HBASE-6290 Add a function a mark a server as dead and start the
recovery the process (Talat Uyarer)
[stack] HBASE-6290 Add a function a mark a server as dead and start the
recovery the process (Talat Uyarer)
[stack] HBASE-6290 Add a function a mark a server as dead and start the
recovery the process (Talat Uyarer) DOC ADDENDUM
------------------------------------------
[...truncated 7175 lines...]
- locked <0xd7fb7a40> (a java.util.Collections$UnmodifiableSet)
- locked <0xd7fb7638> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668)
"IPC Server Responder" daemon prio=10 tid=0x606b7800 nid=0x6345 runnable
[0x634fe000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0xd7fc16b0> (a sun.nio.ch.Util$2)
- locked <0xd7fc16a0> (a java.util.Collections$UnmodifiableSet)
- locked <0xd7fc14a0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835)
at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818)
"org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@1b82b7"
daemon prio=10 tid=0x606ae800 nid=0x6343 waiting on condition [0x632ad000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor.run(BlockManager.java:3336)
at java.lang.Thread.run(Thread.java:724)
"org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@22bd75"
daemon prio=10 tid=0x606ad000 nid=0x6342 waiting on condition [0x632fe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:307)
at java.lang.Thread.run(Thread.java:724)
"org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor@11ddb1d"
daemon prio=10 tid=0x606ac800 nid=0x6341 waiting on condition [0x6345c000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor.run(DecommissionManager.java:76)
at java.lang.Thread.run(Thread.java:724)
"org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@1d16b98"
daemon prio=10 tid=0x606ab000 nid=0x6340 waiting on condition [0x634ad000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:221)
at java.lang.Thread.run(Thread.java:724)
"org.apache.hadoop.util.JvmPauseMonitor$Monitor@11ce791" daemon prio=10
tid=0x606a4000 nid=0x633a waiting on condition [0x6385c000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:161)
at java.lang.Thread.run(Thread.java:724)
"IPC Server idle connection scanner for port 52827" daemon prio=10
tid=0x60671400 nid=0x6335 in Object.wait() [0x638ad000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0xd7fb6f90> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0xd7fb6f90> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Socket Reader #1 for port 52827" prio=10 tid=0x60673800 nid=0x6334 runnable
[0x638fe000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0xd7fb6860> (a sun.nio.ch.Util$2)
- locked <0xd7fb6850> (a java.util.Collections$UnmodifiableSet)
- locked <0xd7fb5f60> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
at
org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
- locked <0xd7fb7c20> (a org.apache.hadoop.ipc.Server$Listener$Reader)
at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)
"Timer-0" daemon prio=10 tid=0x60b8cc00 nid=0x6328 in Object.wait() [0x6407d000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x7cd805e8> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x7cd805e8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"14104738@qtp-14699647-1 - Acceptor0 SelectChannelConnector@localhost:58600"
daemon prio=10 tid=0x60b7e000 nid=0x6325 runnable [0x6447d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
- locked <0x7cd2ee00> (a sun.nio.ch.Util$2)
- locked <0x7cd2edf0> (a java.util.Collections$UnmodifiableSet)
- locked <0x7cd2e9f0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
"28088616@qtp-14699647-0" daemon prio=10 tid=0x60b7f400 nid=0x6324 in
Object.wait() [0x64660000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x7cd2ddc0> (a
org.mortbay.thread.QueuedThreadPool$PoolThread)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
- locked <0x7cd2ddc0> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
"Timer for 'NameNode' metrics system" daemon prio=10 tid=0x60b2dc00 nid=0x6323
in Object.wait() [0x6aa7d000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x7cd3df70> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:552)
- locked <0x7cd3df70> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:505)
"Service Thread" daemon prio=10 tid=0x64cf7000 nid=0x624f runnable [0x00000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0x64cf5400 nid=0x624e waiting on
condition [0x00000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0x64cf3400 nid=0x624d waiting on
condition [0x00000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x64cf1800 nid=0x624c runnable
[0x00000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x64cb7000 nid=0x624a in Object.wait()
[0x6be7d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x7cae9dc8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
- locked <0x7cae9dc8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)
"Reference Handler" daemon prio=10 tid=0x64cb5400 nid=0x6249 in Object.wait()
[0x6c07d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x7cae9de8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:503)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
- locked <0x7cae9de8> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0xf6706800 nid=0x6230 waiting on condition [0xf68ce000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0xd88b2530> (a
java.util.concurrent.FutureTask$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:248)
at java.util.concurrent.FutureTask.get(FutureTask.java:111)
at
org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.bulkLoadPhase(LoadIncrementalHFiles.java:382)
at
org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.doBulkLoad(LoadIncrementalHFiles.java:324)
at
org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.run(LoadIncrementalHFiles.java:914)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:253)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:227)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:214)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:204)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.testSimpleLoad(TestLoadIncrementalHFiles.java:100)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runners.Suite.runChild(Suite.java:127)
at org.junit.runners.Suite.runChild(Suite.java:26)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at org.junit.runner.JUnitCore.run(JUnitCore.java:138)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:113)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:85)
at
org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54)
at
org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:134)
at
org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
"VM Thread" prio=10 tid=0x64bf8800 nid=0x6248 runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0xf6710800 nid=0x6232 runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0xf6712000 nid=0x6233 runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0xf6713800 nid=0x6234 runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0xf6715000 nid=0x6235 runnable
"GC task thread#4 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x6236 runnable
"GC task thread#5 (ParallelGC)" prio=10 tid=0xf6717c00 nid=0x6237 runnable
"GC task thread#6 (ParallelGC)" prio=10 tid=0xf6719400 nid=0x6238 runnable
"GC task thread#7 (ParallelGC)" prio=10 tid=0xf671a800 nid=0x6239 runnable
"GC task thread#8 (ParallelGC)" prio=10 tid=0xf671c000 nid=0x623a runnable
"GC task thread#9 (ParallelGC)" prio=10 tid=0xf671d800 nid=0x623b runnable
"GC task thread#10 (ParallelGC)" prio=10 tid=0xf671ec00 nid=0x623c runnable
"GC task thread#11 (ParallelGC)" prio=10 tid=0xf6720400 nid=0x623d runnable
"GC task thread#12 (ParallelGC)" prio=10 tid=0xf6721c00 nid=0x623e runnable
"VM Periodic Task Thread" prio=10 tid=0x64208400 nid=0x6250 waiting on
condition
JNI global references: 313
+ echo '************ END zombies jstack extract'
************ END zombies jstack extract
+ JIRA_COMMENT='
{color:red}-1 core zombie tests{color}. There are 2 zombie test(s):
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFilesSplitRecovery.testSplitWhileBulkLoadPhase(TestLoadIncrementalHFilesSplitRecovery.java:339)
at
org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.testSimpleLoad(TestLoadIncrementalHFiles.java:100)'
+ BAD=1
+ jps
+ grep surefirebooter
+ cut -d ' ' -f 1
+ xargs kill -9
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Archiving artifacts
Sending artifact delta relative to HBase-TRUNK #5564
Archived 1802 artifacts
Archive block size is 32768
Received 73 blocks and 542807948 bytes
Compression is 0.4%
Took 2 min 56 sec
Recording test results
Updating HBASE-6290
