TestLoadIncrementalHFiles hung
On Fri, Sep 26, 2014 at 4:19 PM, Apache Jenkins Server < [email protected]> wrote: > See <https://builds.apache.org/job/HBase-TRUNK/5565/changes> > > Changes: > > [jeffreyz] HBASE-12052: BulkLoad Failed due to no write permission on > input files > > ------------------------------------------ > [...truncated 7227 lines...] > - locked <0xd4b39290> (a java.util.Collections$UnmodifiableSet) > - locked <0xd4b388b0> (a sun.nio.ch.EPollSelectorImpl) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) > at org.apache.hadoop.ipc.Server$Listener.run(Server.java:668) > > "IPC Server Responder" daemon prio=10 tid=0x60430400 nid=0x2618 runnable > [0x6365c000] > java.lang.Thread.State: RUNNABLE > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) > - locked <0xd4b43650> (a sun.nio.ch.Util$2) > - locked <0xd4b43640> (a java.util.Collections$UnmodifiableSet) > - locked <0xd4b43420> (a sun.nio.ch.EPollSelectorImpl) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) > at > org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:835) > at org.apache.hadoop.ipc.Server$Responder.run(Server.java:818) > > > "org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@13a9eed" > daemon prio=10 tid=0x6041d000 nid=0x25d5 waiting on condition [0x62bfe000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor.run(BlockManager.java:3336) > at java.lang.Thread.run(Thread.java:724) > > > "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@19819d6" > daemon prio=10 tid=0x6041b400 nid=0x25d3 waiting on condition [0x62d5c000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:307) > at java.lang.Thread.run(Thread.java:724) > > > "org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor@390365" > daemon prio=10 tid=0x6041ac00 nid=0x25d2 waiting on condition [0x62dad000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager$Monitor.run(DecommissionManager.java:76) > at java.lang.Thread.run(Thread.java:724) > > > "org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@161fefb" > daemon prio=10 tid=0x60419400 nid=0x25cd waiting on condition [0x62dfe000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:221) > at java.lang.Thread.run(Thread.java:724) > > "org.apache.hadoop.util.JvmPauseMonitor$Monitor@ea2be1" daemon prio=10 > tid=0x60413400 nid=0x25b2 waiting on condition [0x636ad000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:161) > at java.lang.Thread.run(Thread.java:724) > > "IPC Server idle connection scanner for port 46564" daemon prio=10 > tid=0x60bfcc00 nid=0x2598 in Object.wait() [0x636fe000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0xd4adc2e8> (a java.util.TaskQueue) > at java.util.TimerThread.mainLoop(Timer.java:552) > - locked <0xd4adc2e8> (a java.util.TaskQueue) > at java.util.TimerThread.run(Timer.java:505) > > "Socket Reader #1 for port 46564" prio=10 tid=0x60be2400 nid=0x2597 > runnable [0x6385c000] > java.lang.Thread.State: RUNNABLE > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) > - locked <0xd4adcdc8> (a sun.nio.ch.Util$2) > - locked <0xd4adcdb8> (a java.util.Collections$UnmodifiableSet) > - locked <0xd4adcb70> (a sun.nio.ch.EPollSelectorImpl) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) > at > org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616) > - locked <0xd4b39470> (a > org.apache.hadoop.ipc.Server$Listener$Reader) > at > org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595) > > "Timer-0" daemon prio=10 tid=0x60b1a800 nid=0x23f9 in Object.wait() > [0x638ad000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cd57a08> (a java.util.TaskQueue) > at java.util.TimerThread.mainLoop(Timer.java:552) > - locked <0x7cd57a08> (a java.util.TaskQueue) > at java.util.TimerThread.run(Timer.java:505) > > "7074139@qtp-30324039-1 - Acceptor0 SelectChannelConnector@localhost:57124" > daemon prio=10 tid=0x60cfec00 nid=0x23a6 runnable [0x638fe000] > java.lang.Thread.State: RUNNABLE > at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) > at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) > at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) > - locked <0x7cd96bc0> (a sun.nio.ch.Util$2) > - locked <0x7cd96bd0> (a java.util.Collections$UnmodifiableSet) > - locked <0x7cd96b80> (a sun.nio.ch.EPollSelectorImpl) > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) > at > org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) > at > org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) > at > org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) > at > org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) > at > org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) > > "14880852@qtp-30324039-0" daemon prio=10 tid=0x60cfe000 nid=0x23a5 in > Object.wait() [0x6437d000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cd99be8> (a > org.mortbay.thread.QueuedThreadPool$PoolThread) > at > org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) > - locked <0x7cd99be8> (a > org.mortbay.thread.QueuedThreadPool$PoolThread) > > "Timer for 'NameNode' metrics system" daemon prio=10 tid=0x60cad800 > nid=0x236b in Object.wait() [0x6477d000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cdadfe0> (a java.util.TaskQueue) > at java.util.TimerThread.mainLoop(Timer.java:552) > - locked <0x7cdadfe0> (a java.util.TaskQueue) > at java.util.TimerThread.run(Timer.java:505) > > "Service Thread" daemon prio=10 tid=0x64df7000 nid=0x22a8 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread1" daemon prio=10 tid=0x64df5400 nid=0x22a7 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread0" daemon prio=10 tid=0x64df3400 nid=0x22a6 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Signal Dispatcher" daemon prio=10 tid=0x64df1800 nid=0x22a5 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Finalizer" daemon prio=10 tid=0x64db7000 nid=0x22a4 in Object.wait() > [0x6bb7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cb15028> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) > - locked <0x7cb15028> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) > at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) > > "Reference Handler" daemon prio=10 tid=0x64db5400 nid=0x22a3 in > Object.wait() [0x6bd7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cb15048> (a java.lang.ref.Reference$Lock) > at java.lang.Object.wait(Object.java:503) > at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) > - locked <0x7cb15048> (a java.lang.ref.Reference$Lock) > > "main" prio=10 tid=0xf6706800 nid=0x2294 waiting on condition [0xf6903000] > java.lang.Thread.State: WAITING (parking) > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0xd4a73c00> (a > java.util.concurrent.FutureTask$Sync) > at > java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) > at > java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:248) > at java.util.concurrent.FutureTask.get(FutureTask.java:111) > at > org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.bulkLoadPhase(LoadIncrementalHFiles.java:382) > at > org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.doBulkLoad(LoadIncrementalHFiles.java:324) > at > org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.run(LoadIncrementalHFiles.java:914) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:251) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:225) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:212) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.runTest(TestLoadIncrementalHFiles.java:202) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.testSimpleLoad(TestLoadIncrementalHFiles.java:98) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63) > at > org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.run(ParentRunner.java:309) > at org.junit.runners.Suite.runChild(Suite.java:127) > at org.junit.runners.Suite.runChild(Suite.java:26) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63) > at > org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229) > at org.junit.runners.ParentRunner.run(ParentRunner.java:309) > at org.junit.runner.JUnitCore.run(JUnitCore.java:160) > at org.junit.runner.JUnitCore.run(JUnitCore.java:138) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:113) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:85) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54) > at > org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:134) > at > org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200) > at > org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153) > at > org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103) > > "VM Thread" prio=10 tid=0x64cf8800 nid=0x22a2 runnable > > "GC task thread#0 (ParallelGC)" prio=10 tid=0xf6710800 nid=0x2295 runnable > > "GC task thread#1 (ParallelGC)" prio=10 tid=0xf6712000 nid=0x2296 runnable > > "GC task thread#2 (ParallelGC)" prio=10 tid=0xf6713800 nid=0x2297 runnable > > "GC task thread#3 (ParallelGC)" prio=10 tid=0xf6715000 nid=0x2298 runnable > > "GC task thread#4 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x2299 runnable > > "GC task thread#5 (ParallelGC)" prio=10 tid=0xf6717c00 nid=0x229a runnable > > "GC task thread#6 (ParallelGC)" prio=10 tid=0xf6719400 nid=0x229b runnable > > "GC task thread#7 (ParallelGC)" prio=10 tid=0xf671a800 nid=0x229c runnable > > "GC task thread#8 (ParallelGC)" prio=10 tid=0xf671c000 nid=0x229d runnable > > "GC task thread#9 (ParallelGC)" prio=10 tid=0xf671d800 nid=0x229e runnable > > "GC task thread#10 (ParallelGC)" prio=10 tid=0xf671ec00 nid=0x229f runnable > > "GC task thread#11 (ParallelGC)" prio=10 tid=0xf6720400 nid=0x22a0 runnable > > "GC task thread#12 (ParallelGC)" prio=10 tid=0xf6721c00 nid=0x22a1 runnable > > "VM Periodic Task Thread" prio=10 tid=0x64df8c00 nid=0x22a9 waiting on > condition > > JNI global references: 319 > > + echo '************ END zombies jstack extract' > ************ END zombies jstack extract > + JIRA_COMMENT=' > > {color:red}-1 core zombie tests{color}. There are 2 zombie test(s): > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFilesSplitRecovery.testSplitWhileBulkLoadPhase(TestLoadIncrementalHFilesSplitRecovery.java:337) > at > org.apache.hadoop.hbase.mapreduce.TestLoadIncrementalHFiles.testSimpleLoad(TestLoadIncrementalHFiles.java:98)' > + BAD=1 > + jps > + grep surefirebooter > + cut -d ' ' -f 1 > + xargs kill -9 > POST BUILD TASK : SUCCESS > END OF POST BUILD TASK : 0 > Archiving artifacts > Sending artifact delta relative to HBase-TRUNK #5564 > Archived 1802 artifacts > Archive block size is 32768 > Received 73 blocks and 540331401 bytes > Compression is 0.4% > Took 3 min 20 sec > Recording test results > Updating HBASE-12052 >
