See <https://builds.apache.org/job/HBase-TRUNK/4555/changes>
Changes:
[ddas] HBASE-9642. AM ZK Workers stuck doing 100% CPU on HashMap.put
------------------------------------------
[...truncated 4485 lines...]
at com.hazelcast.impl.MulticastService.run(MulticastService.java:110)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.IO.thread-Acceptor" prio=10 tid=0xad083000 nid=0x7950
runnable [0xada7d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0xdea160b8> (a sun.nio.ch.Util$2)
- locked <0xdea160a8> (a java.util.Collections$UnmodifiableSet)
- locked <0xdea15ec0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
at com.hazelcast.nio.SocketAcceptor.run(SocketAcceptor.java:48)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.IO.thread-2" prio=10 tid=0xad4fa400 nid=0x794f runnable
[0xad77d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0xde7817b8> (a sun.nio.ch.Util$2)
- locked <0xde7817a8> (a java.util.Collections$UnmodifiableSet)
- locked <0xde7815c0> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at com.hazelcast.nio.InOutSelector.run(InOutSelector.java:121)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.IO.thread-1" prio=10 tid=0xad4d5800 nid=0x794e runnable
[0xae47d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0xde780f10> (a sun.nio.ch.Util$2)
- locked <0xde780f00> (a java.util.Collections$UnmodifiableSet)
- locked <0xde780d18> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at com.hazelcast.nio.InOutSelector.run(InOutSelector.java:121)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.IO.thread-0" prio=10 tid=0xad6efc00 nid=0x794d runnable
[0xae67d000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
- locked <0xde780668> (a sun.nio.ch.Util$2)
- locked <0xde780658> (a java.util.Collections$UnmodifiableSet)
- locked <0xde780470> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
at com.hazelcast.nio.InOutSelector.run(InOutSelector.java:121)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.ServiceThread" prio=10 tid=0xad6ea800 nid=0x794c waiting
on condition [0xae87d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:317)
at com.hazelcast.cluster.ClusterService.run(ClusterService.java:195)
at java.lang.Thread.run(Thread.java:662)
"hz._hzInstance_2_dev.MigrationThread" prio=10 tid=0xad405800 nid=0x794b
waiting on condition [0xacffe000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
com.hazelcast.impl.PartitionManager$MigrationService.run(PartitionManager.java:1058)
"hz._hzInstance_2_dev.scheduled.thread-3" prio=10 tid=0xad434c00 nid=0x794a
waiting on condition [0xacfad000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0xde73a230> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:957)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:917)
at java.lang.Thread.run(Thread.java:662)
at
com.hazelcast.impl.ExecutorThreadFactory$1.run(ExecutorThreadFactory.java:45)
"hz._hzInstance_2_dev.scheduled.thread-2" prio=10 tid=0xad078400 nid=0x7949
waiting on condition [0xad15c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0xde73a230> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:957)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:917)
at java.lang.Thread.run(Thread.java:662)
at
com.hazelcast.impl.ExecutorThreadFactory$1.run(ExecutorThreadFactory.java:45)
"hz._hzInstance_2_dev.scheduled.thread-1" prio=10 tid=0xad4f7c00 nid=0x7948
waiting on condition [0xacf5c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0xde73a230> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609)
at
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602)
at
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:957)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:917)
at java.lang.Thread.run(Thread.java:662)
at
com.hazelcast.impl.ExecutorThreadFactory$1.run(ExecutorThreadFactory.java:45)
"Low Memory Detector" daemon prio=10 tid=0xadb07000 nid=0x790b runnable
[0x00000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" daemon prio=10 tid=0xadb05400 nid=0x790a waiting on
condition [0x00000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" daemon prio=10 tid=0xadb03400 nid=0x7909 waiting on
condition [0x00000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0xadb02400 nid=0x7908 runnable
[0x00000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0xf67ef000 nid=0x7907 in Object.wait()
[0xaee7d000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0xdf1305a0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0xdf1305a0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:171)
"Reference Handler" daemon prio=10 tid=0xf67edc00 nid=0x7905 in Object.wait()
[0xae0e4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0xdf12c080> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0xdf12c080> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0xf6705800 nid=0x78fb waiting on condition [0xf68e1000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at
com.hazelcast.impl.MulticastJoiner.findMasterWithMulticast(MulticastJoiner.java:166)
at com.hazelcast.impl.MulticastJoiner.doJoin(MulticastJoiner.java:55)
at com.hazelcast.impl.AbstractJoiner.join(AbstractJoiner.java:56)
at com.hazelcast.impl.Node.join(Node.java:563)
at com.hazelcast.impl.Node.start(Node.java:438)
at com.hazelcast.impl.FactoryImpl.<init>(FactoryImpl.java:391)
at
com.hazelcast.impl.FactoryImpl.newHazelcastInstanceProxy(FactoryImpl.java:133)
at
com.hazelcast.impl.FactoryImpl.newHazelcastInstanceProxy(FactoryImpl.java:119)
at
com.hazelcast.impl.FactoryImpl.newHazelcastInstanceProxy(FactoryImpl.java:104)
at com.hazelcast.core.Hazelcast.newHazelcastInstance(Hazelcast.java:507)
at
org.apache.camel.component.hazelcast.HazelcastComponent.doStart(HazelcastComponent.java:118)
at org.apache.camel.support.ServiceSupport.start(ServiceSupport.java:61)
at
org.apache.camel.impl.DefaultCamelContext.startService(DefaultCamelContext.java:1819)
at
org.apache.camel.impl.DefaultCamelContext.getComponent(DefaultCamelContext.java:334)
- locked <0xde5c74d0> (a java.util.HashMap)
at
org.apache.camel.impl.DefaultCamelContext.getEndpoint(DefaultCamelContext.java:479)
at
org.apache.camel.util.CamelContextHelper.getMandatoryEndpoint(CamelContextHelper.java:62)
at
org.apache.camel.model.RouteDefinition.resolveEndpoint(RouteDefinition.java:191)
at
org.apache.camel.impl.DefaultRouteContext.resolveEndpoint(DefaultRouteContext.java:108)
at
org.apache.camel.impl.DefaultRouteContext.resolveEndpoint(DefaultRouteContext.java:114)
at
org.apache.camel.model.FromDefinition.resolveEndpoint(FromDefinition.java:72)
at
org.apache.camel.impl.DefaultRouteContext.getEndpoint(DefaultRouteContext.java:90)
at
org.apache.camel.model.RouteDefinition.addRoutes(RouteDefinition.java:861)
at
org.apache.camel.model.RouteDefinition.addRoutes(RouteDefinition.java:176)
at
org.apache.camel.impl.DefaultCamelContext.startRoute(DefaultCamelContext.java:751)
at
org.apache.camel.impl.DefaultCamelContext.startRouteDefinitions(DefaultCamelContext.java:1845)
at
org.apache.camel.impl.DefaultCamelContext.doStartCamel(DefaultCamelContext.java:1625)
at
org.apache.camel.impl.DefaultCamelContext.doStart(DefaultCamelContext.java:1500)
- locked <0xde5c5050> (a org.apache.camel.impl.DefaultCamelContext)
at org.apache.camel.support.ServiceSupport.start(ServiceSupport.java:61)
at
org.apache.camel.impl.DefaultCamelContext.start(DefaultCamelContext.java:1468)
at
org.apache.camel.test.junit4.CamelTestSupport.startCamelContext(CamelTestSupport.java:456)
at
org.apache.camel.test.junit4.CamelTestSupport.doSetUp(CamelTestSupport.java:302)
at
org.apache.camel.test.junit4.CamelTestSupport.setUp(CamelTestSupport.java:209)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray2(ReflectionUtils.java:208)
at
org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:158)
at
org.apache.maven.surefire.booter.ProviderFactory.invokeProvider(ProviderFactory.java:86)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:95)
"VM Thread" prio=10 tid=0xf67ea000 nid=0x7904 runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0xf670cc00 nid=0x78fc runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0xf670e000 nid=0x78fd runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0xf670f800 nid=0x78fe runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0xf6710c00 nid=0x78ff runnable
"GC task thread#4 (ParallelGC)" prio=10 tid=0xf6712400 nid=0x7900 runnable
"GC task thread#5 (ParallelGC)" prio=10 tid=0xf6713800 nid=0x7901 runnable
"GC task thread#6 (ParallelGC)" prio=10 tid=0xf6715000 nid=0x7902 runnable
"GC task thread#7 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x7903 runnable
"VM Periodic Task Thread" prio=10 tid=0xadb11000 nid=0x790c waiting on
condition
JNI global references: 941
+ echo '************ END zombies jstack extract'
************ END zombies jstack extract
+ JIRA_COMMENT='
{color:red}-1 core zombie tests{color}. There are 1 zombie test(s):
at
org.apache.camel.test.junit4.CamelTestSupport.startCamelContext(CamelTestSupport.java:456)
at
org.apache.camel.test.junit4.CamelTestSupport.doSetUp(CamelTestSupport.java:302)
at
org.apache.camel.test.junit4.CamelTestSupport.setUp(CamelTestSupport.java:209)'
+ BAD=1
+ jps
+ grep surefirebooter
+ cut -d ' ' -f 1
+ xargs kill -9
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Archiving artifacts
Recording test results
Updating HBASE-9642
