[
https://issues.apache.org/jira/browse/GEODE-911?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15195845#comment-15195845
]
Jason Huynh commented on GEODE-911:
-----------------------------------
I can get this to reproduce
{noformat}
com.gemstone.gemfire.test.dunit.RMIException: While invoking
com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$27/422330142.run
in VM 2 running on Host 192.168.1.84 with 8 VMs
at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:378)
at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:321)
at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:272)
at
com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.test_Bug44153_StopOneSender_StartAnotherSender_CheckQueueSize(SerialGatewaySenderOperationsDUnitTest.java:441)
at
com.gemstone.gemfire.internal.cache.wan.concurrent.ConcurrentSerialGatewaySenderOperationsOffHeapDUnitTest.testMany(ConcurrentSerialGatewaySenderOperationsOffHeapDUnitTest.java:37)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:252)
at junit.framework.TestSuite.run(TestSuite.java:247)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86)
at
org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
at
org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
at
org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
Caused by: java.lang.AssertionError: Event never occurred after 240000 ms:
Expected region entries: 110 but actual entries: 108 present region keyset [40,
95, 22, 67, 49, 12, 31, 20, 75, 27, 55, 96, 103, 89, 68, 62, 34, 19, 83, 13,
41, 48, 5, 28, 69, 33, 97, 18, 88, 61, 74, 42, 25, 47, 56, 84, 70, 7, 91, 46,
102, 81, 0, 98, 101, 73, 108, 53, 90, 99, 36, 26, 8, 45, 63, 109, 100, 17, 35,
72, 54, 82, 2, 9, 51, 86, 44, 37, 58, 64, 92, 16, 71, 10, 52, 78, 43, 30, 23,
15, 85, 38, 29, 80, 57, 107, 24, 106, 65, 79, 93, 77, 14, 21, 105, 60, 3, 94,
66, 39, 32, 11, 4, 50, 104, 59, 76, 87]
at org.junit.Assert.fail(Assert.java:88)
at com.gemstone.gemfire.test.dunit.Wait.waitForCriterion(Wait.java:119)
at
com.gemstone.gemfire.internal.cache.wan.WANTestBase.validateRegionSize(WANTestBase.java:3695)
at
com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.lambda$131(SerialGatewaySenderOperationsDUnitTest.java:441)
at
com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$6/684900051.run(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at hydra.MethExecutor.executeObject(MethExecutor.java:268)
at
com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:84)
at sun.reflect.GeneratedMethodAccessor135.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
at sun.rmi.transport.Transport$1.run(Transport.java:178)
at sun.rmi.transport.Transport$1.run(Transport.java:175)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:174)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:557)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:812)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:671)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
However in the logs I see:
[fatal 2016/03/15 11:17:07.962 PDT <AckReaderThread for : ln>
tid=AckReaderThread for : lnid] Uncaught exception in thread
Thread[AckReaderThread for : ln,5,Event Processor for GatewaySender_ln.2]
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method)
at java.lang.Thread.start(Thread.java:714)
at
com.gemstone.gemfire.SystemFailure.startWatchDog(SystemFailure.java:387)
at
com.gemstone.gemfire.SystemFailure.notifyWatchDog(SystemFailure.java:955)
at
com.gemstone.gemfire.SystemFailure.setFailure(SystemFailure.java:1014)
at
com.gemstone.gemfire.SystemFailure.initiateFailure(SystemFailure.java:988)
at
com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderEventProcessor.run(SerialGatewaySenderEventProcessor.java:236)
This could be due to the way I am running the tests (looping until failure) or
perhaps this is a true reproduction where the node runs out of memory while the
receiver waits for a lingering event.
> CI failure:
> ConcurrentSerialGatewaySenderOperationsOffHeapDUnitTest.test_Bug44153_StopOneSender_StartAnotherSender_CheckQueueSize
> ---------------------------------------------------------------------------------------------------------------------------------
>
> Key: GEODE-911
> URL: https://issues.apache.org/jira/browse/GEODE-911
> Project: Geode
> Issue Type: Bug
> Reporter: Sai Boorlagadda
> Assignee: Dan Smith
> Labels: CI
>
> {noformat}
> Error Message
> com.gemstone.gemfire.test.dunit.RMIException: While invoking
> com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$92/1788087112.run
> in VM 2 running on Host buildtest.gemstone.com with 8 VMs
> Stacktrace
> com.gemstone.gemfire.test.dunit.RMIException: While invoking
> com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest$$Lambda$92/1788087112.run
> in VM 2 running on Host buildtest.gemstone.com with 8 VMs
> at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:370)
> at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:313)
> at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:267)
> at
> com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.test_Bug44153_StopOneSender_StartAnotherSender_CheckQueueSize(SerialGatewaySenderOperationsDUnitTest.java:434)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at junit.framework.TestCase.runTest(TestCase.java:176)
> at junit.framework.TestCase.runBare(TestCase.java:141)
> at junit.framework.TestResult$1.protect(TestResult.java:122)
> at junit.framework.TestResult.runProtected(TestResult.java:142)
> at junit.framework.TestResult.run(TestResult.java:125)
> at junit.framework.TestCase.run(TestCase.java:129)
> at junit.framework.TestSuite.runTest(TestSuite.java:252)
> at junit.framework.TestSuite.run(TestSuite.java:247)
> at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:105)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:56)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64)
> at
> org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
> at
> org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
> at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360)
> at
> org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54)
> at
> org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: junit.framework.AssertionFailedError: Event never occurred after
> 240000 ms: Expected region entries: 100 but actual entries: 90 present region
> keyset [40, 95, 67, 49, 31, 96, 20, 75, 103, 89, 68, 55, 62, 34, 19, 83, 13,
> 41, 48, 69, 33, 28, 97, 18, 25, 88, 61, 74, 56, 84, 70, 91, 81, 98, 46, 102,
> 101, 73, 108, 53, 90, 99, 36, 26, 45, 35, 63, 72, 54, 109, 82, 100, 51, 86,
> 44, 58, 64, 92, 16, 71, 10, 78, 43, 30, 23, 15, 38, 85, 29, 80, 107, 24, 106,
> 65, 79, 93, 77, 14, 21, 105, 60, 94, 66, 11, 39, 50, 104, 59, 76, 87]
> at junit.framework.Assert.fail(Assert.java:57)
> at junit.framework.TestCase.fail(TestCase.java:227)
> at
> com.gemstone.gemfire.test.dunit.DistributedTestCase.waitForCriterion(DistributedTestCase.java:1135)
> at
> com.gemstone.gemfire.internal.cache.wan.WANTestBase.validateRegionSize(WANTestBase.java:3691)
> at
> com.gemstone.gemfire.internal.cache.wan.serial.SerialGatewaySenderOperationsDUnitTest.lambda$test_Bug44153_StopOneSender_StartAnotherSender_CheckQueueSize$81c80a4a$15(SerialGatewaySenderOperationsDUnitTest.java:434)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at hydra.MethExecutor.executeObject(MethExecutor.java:267)
> at
> com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:84)
> at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
> at sun.rmi.transport.Transport$1.run(Transport.java:200)
> at sun.rmi.transport.Transport$1.run(Transport.java:197)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
> at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
> at java.security.AccessController.doPrivileged(Native Method)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
> ... 3 more
> Standard Output
> Previously run tests: [WANManagementDUnitTest, TestRemoteClusterDUnitTest,
> ClusterConfigurationDUnitTest, UpdateVersionDUnitTest,
> ConcurrentWANPropogation_2_DUnitTest,
> ConcurrentParallelGatewaySenderOperation_1_DUnitTest,
> ConcurrentSerialGatewaySenderOperationsOffHeapDUnitTest]
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
