TestTableMapReduce hung. St.Ack
On Fri, Sep 12, 2014 at 11:19 AM, Apache Jenkins Server < [email protected]> wrote: > See <https://builds.apache.org/job/HBase-TRUNK/5497/changes> > > Changes: > > [stack] HBASE-11897 Add append and remove peer table-cfs cmds for > replication (Liu Shaoqui) > > ------------------------------------------ > [...truncated 8713 lines...] > at > org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) > > "21712743@qtp-22206221-0" daemon prio=10 tid=0x60cb3000 nid=0x947 in > Object.wait() [0x6476c000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > at > org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) > - locked <0x7fd6f048> (a > org.mortbay.thread.QueuedThreadPool$PoolThread) > > "Timer for 'NameNode' metrics system" daemon prio=10 tid=0x60c61800 > nid=0x934 in Object.wait() [0x6437d000] > java.lang.Thread.State: TIMED_WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > at java.util.TimerThread.mainLoop(Timer.java:552) > - locked <0x7fd64290> (a java.util.TaskQueue) > at java.util.TimerThread.run(Timer.java:505) > > "Service Thread" daemon prio=10 tid=0x64df7000 nid=0x770 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread1" daemon prio=10 tid=0x64df5400 nid=0x76a waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread0" daemon prio=10 tid=0x64df3400 nid=0x769 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Signal Dispatcher" daemon prio=10 tid=0x64df1800 nid=0x768 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Finalizer" daemon prio=10 tid=0x64db7000 nid=0x741 in Object.wait() > [0x6bb7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cb731c8> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) > - locked <0x7cb731c8> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) > at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) > > "Reference Handler" daemon prio=10 tid=0x64db5400 nid=0x73e in > Object.wait() [0x6bd7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0x7cb72ce8> (a java.lang.ref.Reference$Lock) > at java.lang.Object.wait(Object.java:503) > at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) > - locked <0x7cb72ce8> (a java.lang.ref.Reference$Lock) > > "main" prio=10 tid=0xf6706800 nid=0x717 waiting on condition [0xf68f3000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at > org.apache.hadoop.util.ThreadUtil.sleepAtLeastIgnoreInterrupts(ThreadUtil.java:43) > at > org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:154) > at com.sun.proxy.$Proxy89.getNewApplication(Unknown Source) > at > org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getNewApplication(YarnClientImpl.java:165) > at > org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createApplication(YarnClientImpl.java:173) > at > org.apache.hadoop.mapred.ResourceMgrDelegate.getNewJobID(ResourceMgrDelegate.java:179) > at > org.apache.hadoop.mapred.YARNRunner.getNewJobID(YARNRunner.java:230) > at > org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:357) > at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285) > at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) > at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282) > at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:562) > at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:557) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548) > at > org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:557) > at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:548) > at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:833) > at > org.apache.hadoop.hbase.mapred.TestTableMapReduce.runTestOnTable(TestTableMapReduce.java:89) > at > org.apache.hadoop.hbase.mapreduce.TestTableMapReduceBase.testMultiRegionTable(TestTableMapReduceBase.java:97) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63) > at > org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.run(ParentRunner.java:309) > at org.junit.runners.Suite.runChild(Suite.java:127) > at org.junit.runners.Suite.runChild(Suite.java:26) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63) > at > org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229) > at org.junit.runners.ParentRunner.run(ParentRunner.java:309) > at org.junit.runner.JUnitCore.run(JUnitCore.java:160) > at org.junit.runner.JUnitCore.run(JUnitCore.java:138) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:113) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:85) > at > org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:54) > at > org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:134) > at > org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200) > at > org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153) > at > org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103) > > "VM Thread" prio=10 tid=0x64cf8800 nid=0x734 runnable > > "GC task thread#0 (ParallelGC)" prio=10 tid=0xf6710800 nid=0x71a runnable > > "GC task thread#1 (ParallelGC)" prio=10 tid=0xf6712000 nid=0x71b runnable > > "GC task thread#2 (ParallelGC)" prio=10 tid=0xf6713800 nid=0x71e runnable > > "GC task thread#3 (ParallelGC)" prio=10 tid=0xf6715000 nid=0x721 runnable > > "GC task thread#4 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x722 runnable > > "GC task thread#5 (ParallelGC)" prio=10 tid=0xf6717c00 nid=0x723 runnable > > "GC task thread#6 (ParallelGC)" prio=10 tid=0xf6719400 nid=0x724 runnable > > "GC task thread#7 (ParallelGC)" prio=10 tid=0xf671a800 nid=0x725 runnable > > "GC task thread#8 (ParallelGC)" prio=10 tid=0xf671c000 nid=0x726 runnable > > "GC task thread#9 (ParallelGC)" prio=10 tid=0xf671d800 nid=0x727 runnable > > "GC task thread#10 (ParallelGC)" prio=10 tid=0xf671ec00 nid=0x728 runnable > > "GC task thread#11 (ParallelGC)" prio=10 tid=0xf6720400 nid=0x729 runnable > > "GC task thread#12 (ParallelGC)" prio=10 tid=0xf6721c00 nid=0x72a runnable > > "VM Periodic Task Thread" prio=10 tid=0x64df8c00 nid=0x773 waiting on > condition > > JNI global references: 595 > > 2014-09-12 18:12:56 > Full thread dump Java HotSpot(TM) Server VM (23.25-b01 mixed mode): > > "Attach Listener" daemon prio=10 tid=0xaa400800 nid=0x3bd9 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Service Thread" daemon prio=10 tid=0xaa8f7000 nid=0x3ba4 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread1" daemon prio=10 tid=0xaa8f5400 nid=0x3ba3 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "C2 CompilerThread0" daemon prio=10 tid=0xaa8f3400 nid=0x3ba2 waiting on > condition [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Signal Dispatcher" daemon prio=10 tid=0xaa8f1800 nid=0x3ba1 runnable > [0x00000000] > java.lang.Thread.State: RUNNABLE > > "Finalizer" daemon prio=10 tid=0xaa8b7000 nid=0x3b90 in Object.wait() > [0xaec7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0xdf3a4928> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) > - locked <0xdf3a4928> (a java.lang.ref.ReferenceQueue$Lock) > at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) > at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) > > "Reference Handler" daemon prio=10 tid=0xaa8b5400 nid=0x3b8f in > Object.wait() [0xaee7d000] > java.lang.Thread.State: WAITING (on object monitor) > at java.lang.Object.wait(Native Method) > - waiting on <0xdf3a49b0> (a java.lang.ref.Reference$Lock) > at java.lang.Object.wait(Object.java:503) > at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) > - locked <0xdf3a49b0> (a java.lang.ref.Reference$Lock) > > "main" prio=10 tid=0xf6706800 nid=0x3b7b runnable [0xf691e000] > java.lang.Thread.State: RUNNABLE > at java.io.PrintStream.write(PrintStream.java:480) > - locked <0xdf3c0810> (a java.io.PrintStream) > at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) > at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) > at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104) > - locked <0xdf3c0940> (a java.io.OutputStreamWriter) > at > java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185) > at java.io.PrintStream.write(PrintStream.java:527) > - locked <0xdf3c0810> (a java.io.PrintStream) > at java.io.PrintStream.print(PrintStream.java:669) > at > org.apache.maven.surefire.booter.ForkingRunListener.sendProps(ForkingRunListener.java:163) > at > org.apache.maven.surefire.booter.ForkingRunListener.<init>(ForkingRunListener.java:102) > at > org.apache.maven.surefire.booter.ForkingReporterFactory.createReporter(ForkingReporterFactory.java:53) > - locked <0xdf3a6bd0> (a > org.apache.maven.surefire.booter.ForkingReporterFactory) > at > org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:111) > at > org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200) > at > org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153) > at > org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103) > > "VM Thread" prio=10 tid=0xaa7f8800 nid=0x3b8d runnable > > "GC task thread#0 (ParallelGC)" prio=10 tid=0xf6710800 nid=0x3b7c runnable > > "GC task thread#1 (ParallelGC)" prio=10 tid=0xf6712000 nid=0x3b7d runnable > > "GC task thread#2 (ParallelGC)" prio=10 tid=0xf6713400 nid=0x3b7e runnable > > "GC task thread#3 (ParallelGC)" prio=10 tid=0xf6714c00 nid=0x3b7f runnable > > "GC task thread#4 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x3b80 runnable > > "GC task thread#5 (ParallelGC)" prio=10 tid=0xf6717800 nid=0x3b81 runnable > > "GC task thread#6 (ParallelGC)" prio=10 tid=0xf6719000 nid=0x3b82 runnable > > "GC task thread#7 (ParallelGC)" prio=10 tid=0xf671a800 nid=0x3b83 runnable > > "GC task thread#8 (ParallelGC)" prio=10 tid=0xf671bc00 nid=0x3b84 runnable > > "GC task thread#9 (ParallelGC)" prio=10 tid=0xf671d400 nid=0x3b85 runnable > > "GC task thread#10 (ParallelGC)" prio=10 tid=0xf671ec00 nid=0x3b86 runnable > > "GC task thread#11 (ParallelGC)" prio=10 tid=0xf6720400 nid=0x3b87 runnable > > "GC task thread#12 (ParallelGC)" prio=10 tid=0xf6721800 nid=0x3b88 runnable > > "VM Periodic Task Thread" prio=10 tid=0xaa8f9000 nid=0x3ba5 waiting on > condition > > JNI global references: 127 > > + echo '************ END zombies jstack extract' > ************ END zombies jstack extract > + JIRA_COMMENT=' > > {color:red}-1 core zombie tests{color}. There are 1 zombie test(s): > at > org.apache.hadoop.hbase.mapreduce.TestTableMapReduceBase.testMultiRegionTable(TestTableMapReduceBase.java:97)' > + BAD=1 > + grep surefirebooter > + jps > + cut -d ' ' -f 1 > + xargs kill -9 > POST BUILD TASK : SUCCESS > END OF POST BUILD TASK : 0 > Archiving artifacts > Sending artifact delta relative to HBase-TRUNK #5496 > Archived 1788 artifacts > Archive block size is 32768 > Received 75 blocks and 551641167 bytes > Compression is 0.4% > Took 3 min 26 sec > Recording test results > Updating HBASE-11897 >
