[
https://issues.apache.org/jira/browse/GEODE-6044?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16762168#comment-16762168
]
Mark Hanson commented on GEODE-6044:
------------------------------------
Recurred again:
Failure Link:
[https://concourse.apachegeode-ci.info/teams/main/pipelines/apache-develop-main/jobs/DistributedTestOpenJDK11/builds/374]
Log Archives:
=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Test Results URI =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
http://files.apachegeode-ci.info/builds/apache-develop-main/1.9.0-SNAPSHOT.0413/test-results/distributedTest/1549404448/
=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
Test report artifacts from this job are available at:
http://files.apachegeode-ci.info/builds/apache-develop-main/1.9.0-SNAPSHOT.0413/test-artifacts/1549404448/distributedtestfiles-OpenJDK11-1.9.0-SNAPSHOT.0413.tgz
Stack trace:
org.apache.geode.test.dunit.RMIException: While invoking
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest$$Lambda$390/0x0000000840596c40.run
in VM 2 running on Host f2be4f73cee1 with 8 VMs at
org.apache.geode.test.dunit.VM.executeMethodOnObject(VM.java:533) at
org.apache.geode.test.dunit.VM.invoke(VM.java:365) at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest.testCreateMaximumSenders(SerialGatewaySenderQueueDUnitTest.java:370)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566) at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at
org.junit.rules.RunRules.evaluate(RunRules.java:20) at
org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at
org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at
org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at
org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:110)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38)
at
org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:66)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566) at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) at
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:118)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566) at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:175)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:157)
at
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:404)
at
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:63)
at
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:46)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:55)
at java.lang.Thread.run(Thread.java:834) Caused by:
org.awaitility.core.ConditionTimeoutException: Condition with lambda expression
in org.apache.geode.internal.cache.wan.WANTestBase that uses long was not
fulfilled within 300 seconds. at
org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:145) at
org.awaitility.core.CallableCondition.await(CallableCondition.java:79) at
org.awaitility.core.CallableCondition.await(CallableCondition.java:27) at
org.awaitility.core.ConditionFactory.until(ConditionFactory.java:902) at
org.awaitility.core.ConditionFactory.until(ConditionFactory.java:860) at
org.apache.geode.internal.cache.wan.WANTestBase.verifyListenerEvents(WANTestBase.java:3542)
at
org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest.lambda$testCreateMaximumSenders$faf964a3$1(SerialGatewaySenderQueueDUnitTest.java:370)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566) at
org.apache.geode.test.dunit.internal.MethodInvoker.executeObject(MethodInvoker.java:123)
at
org.apache.geode.test.dunit.internal.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:69)
at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:566) at
sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:359) at
sun.rmi.transport.Transport$1.run(Transport.java:200) at
sun.rmi.transport.Transport$1.run(Transport.java:197) at
java.security.AccessController.doPrivileged(Native Method) at
sun.rmi.transport.Transport.serviceCall(Transport.java:196) at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:562) at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:796)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:677)
at java.security.AccessController.doPrivileged(Native Method) at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:676)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
... 1 more
> java.lang.OutOfMemoryError: Java heap space when running DUnit tests using
> java11
> ---------------------------------------------------------------------------------
>
> Key: GEODE-6044
> URL: https://issues.apache.org/jira/browse/GEODE-6044
> Project: Geode
> Issue Type: Sub-task
> Affects Versions: 1.8.0
> Reporter: Robert Houghton
> Assignee: Jinmei Liao
> Priority: Major
>
> The OOME errors seem to happen after executing these tests:
> org.apache.geode.internal.cache.wan.offheap.SerialGatewaySenderQueueOffHeapDUnitTest
> > testCreateMaximumSenders
> testCreateMaximumPlusOneSenders
> REPRODUCED:
> DistributedTestOpenJDK11 job 105 failed in the same way.
> http://files.apachegeode-ci.info/builds/apache-develop-main/1.9.0-build.140/test-results/distributedTest/1542056138/
>
> Log (abridged)
> {noformat}
> org.apache.geode.test.dunit.RMIException: While invoking
> org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest$$Lambda$377/0x0000000840571840.run
> in VM 2 running on Host 72ac34082783 with 8 VMs
> at org.apache.geode.test.dunit.VM.invoke(VM.java:433)
> at org.apache.geode.test.dunit.VM.invoke(VM.java:402)
> at org.apache.geode.test.dunit.VM.invoke(VM.java:361)
> at
> org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest.testCreateMaximumSenders(SerialGatewaySenderQueueDUnitTest.java:370)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:106)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38)
> at
> org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:66)
> at
> org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
> at
> org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
> at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:117)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:155)
> at
> org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:137)
> at
> org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:404)
> at
> org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:63)
> at
> org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:46)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at
> org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:55)
> at java.lang.Thread.run(Thread.java:834)
> Caused by: org.awaitility.core.ConditionTimeoutException: Condition with
> lambda expression in org.apache.geode.internal.cache.wan.WANTestBase that
> uses long was not fulfilled within 300 seconds.
> at org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:145)
> at
> org.awaitility.core.CallableCondition.await(CallableCondition.java:79)
> at
> org.awaitility.core.CallableCondition.await(CallableCondition.java:27)
> at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:890)
> at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:848)
> at
> org.apache.geode.internal.cache.wan.WANTestBase.verifyListenerEvents(WANTestBase.java:3540)
> at
> org.apache.geode.internal.cache.wan.serial.SerialGatewaySenderQueueDUnitTest.lambda$testCreateMaximumSenders$faf964a3$1(SerialGatewaySenderQueueDUnitTest.java:370)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:566)
> at hydra.MethExecutor.executeObject(MethExecutor.java:244)
> at
> org.apache.geode.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:70)
> at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:566)
> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:359)
> at sun.rmi.transport.Transport$1.run(Transport.java:200)
> at sun.rmi.transport.Transport$1.run(Transport.java:197)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
> at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:562)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:796)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:677)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:676)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> ... 1 more
> java.lang.AssertionError: Suspicious strings were written to the log during
> this run.
> Fix the strings or use IgnoredException.addIgnoredException to ignore.
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6124
> [fatal 2018/11/12 20:06:18.048 UTC <Geode Membership View Creator> tid=280]
> Possible loss of quorum due to the loss of 1 cache processes:
> [172.17.0.23(483)<v1>:41004]
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6126
> [fatal 2018/11/12 20:06:19.050 UTC <Geode Membership View Creator> tid=280]
> Membership service failure: Exiting due to possible network partition event
> due to loss of 1 cache processes: [172.17.0.23(483)<v1>:41004]
> org.apache.geode.ForcedDisconnectException: Exiting due to possible network
> partition event due to loss of 1 cache processes: [172.17.0.23(483)<v1>:41004]
> at
> org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.forceDisconnect(GMSMembershipManager.java:2503)
> at
> org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.forceDisconnect(GMSJoinLeave.java:1049)
> at
> org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.access$1500(GMSJoinLeave.java:90)
> at
> org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.prepareAndSendView(GMSJoinLeave.java:2506)
> at
> org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.createAndSendView(GMSJoinLeave.java:2483)
> at
> org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.run(GMSJoinLeave.java:2267)
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6749
> [fatal 2018/11/12 20:10:30.023 UTC <Asynchronous disk writer for region
> ln-59> tid=2640] Fatal error from asynchronous flusher thread
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6752
> [fatal 2018/11/12 20:10:30.023 UTC <Asynchronous disk writer for region
> ln-67> tid=2707] Fatal error from asynchronous flusher thread
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6755
> [fatal 2018/11/12 20:10:30.024 UTC <Asynchronous disk writer for region
> ln-90> tid=2898] Fatal error from asynchronous flusher thread
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6758
> [fatal 2018/11/12 20:10:30.024 UTC <Asynchronous disk writer for region
> ln-81> tid=2824] Fatal error from asynchronous flusher thread
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6761
> [error 2018/11/12 20:10:30.024 UTC <unicast receiver,72ac34082783-15304>
> tid=2012] JGRP000190: failed receiving packet
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6764
> [fatal 2018/11/12 20:10:30.024 UTC <Asynchronous disk writer for region
> ln-109> tid=3052] Fatal error from asynchronous flusher thread
> java.lang.OutOfMemoryError: Java heap space
> -----------------------------------------------------------------------
> Found suspect string in log4j at line 6843
> {noformat}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
