[
https://issues.apache.org/jira/browse/GEODE-2488?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Kirk Lund updated GEODE-2488:
-----------------------------
Description:
Note: this can occur outside of dunit tests as well. Just using the gfsh
netstat command on locator or server with too little heap space will hit this.
See the 1st comment about not streaming the netstat output -- the entire output
is held in memory in a string and two byte arrays before sending the result
back from NetstatFunction.
The JUnit controller JVM for NetstatDUnitTest fails with something like the
follow stack. The cause appears to be one of the DUnit VMs running out of
memory (see the OOME stack down further in this description.
{noformat}
org.junit.ComparisonFailure: [{"errorCode":505,"message":["Could not process
command due to GemFire error. Error occurred while executing netstat on
[server-1]"]}] expected:<[OK]> but was:<[ERROR]>
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
org.apache.geode.test.dunit.rules.GfshShellConnectionRule.executeAndVerifyCommand(GfshShellConnectionRule.java:163)
at
org.apache.geode.management.internal.cli.NetstatDUnitTest.testConnectToJmxManagerOne(NetstatDUnitTest.java:81)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.apache.geode.test.junit.rules.DescribedExternalResource$1.evaluate(DescribedExternalResource.java:37)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:114)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:57)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:66)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:109)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:377)
at
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54)
at
org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
The JVM that runs out of memory ends up logging multiple OutOfMemoryError
stacks:
{noformat}
[vm1] 25.584: [GC (System.gc()) [PSYoungGen: 7724K->591K(161792K)]
242411K->235286K(511488K), 0.0047912 secs] [Times: user=0.02 sys=0.00,
real=0.01 secs]
[vm1] 25.589: [Full GC (System.gc()) [PSYoungGen: 591K->0K(161792K)]
[ParOldGen: 234695K->9294K(349696K)] 235286K->9294K(511488K), [Metaspace:
34478K->34478K(1081344K)], 0.0891637 secs] [Times: user=0.72 sys=0.01,
real=0.09 secs]
[vm1] [error 2017/02/14 09:36:13.738 UTC <Cache Server Acceptor
0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Cache server: Unexpected
IOException from accept
[vm1] [fatal 2017/02/14 09:36:13.739 UTC <unused p2p reader> tid=0x4f] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.738 UTC <Function Execution Processor1>
tid=0x48] Uncaught exception in thread Thread[Function Execution
Processor1,10,DistributionManager Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.740 UTC <Cache Server Acceptor
0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Uncaught exception in thread
Thread[Cache Server Acceptor 0.0.0.0/0.0.0.0:36823 local port: 36823,5,Acceptor
0.0.0.0/0.0.0.0:36823]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.744 UTC <unused p2p reader> tid=0x47] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.748 UTC <unused p2p reader> tid=0x54] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:14.602 UTC <ClientHealthMonitor Thread> tid=0x45]
Uncaught exception in thread Thread[ClientHealthMonitor
Thread,5,ClientHealthMonitor Thread Group]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
{noformat}
was:
The JUnit controller JVM for NetstatDUnitTest fails with something like the
follow stack. The cause appears to be one of the DUnit VMs running out of
memory (see the OOME stack down further in this description.
{noformat}
org.junit.ComparisonFailure: [{"errorCode":505,"message":["Could not process
command due to GemFire error. Error occurred while executing netstat on
[server-1]"]}] expected:<[OK]> but was:<[ERROR]>
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
org.apache.geode.test.dunit.rules.GfshShellConnectionRule.executeAndVerifyCommand(GfshShellConnectionRule.java:163)
at
org.apache.geode.management.internal.cli.NetstatDUnitTest.testConnectToJmxManagerOne(NetstatDUnitTest.java:81)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.apache.geode.test.junit.rules.DescribedExternalResource$1.evaluate(DescribedExternalResource.java:37)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:114)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:57)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:66)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:109)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:377)
at
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54)
at
org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
The JVM that runs out of memory ends up logging multiple OutOfMemoryError
stacks:
{noformat}
[vm1] 25.584: [GC (System.gc()) [PSYoungGen: 7724K->591K(161792K)]
242411K->235286K(511488K), 0.0047912 secs] [Times: user=0.02 sys=0.00,
real=0.01 secs]
[vm1] 25.589: [Full GC (System.gc()) [PSYoungGen: 591K->0K(161792K)]
[ParOldGen: 234695K->9294K(349696K)] 235286K->9294K(511488K), [Metaspace:
34478K->34478K(1081344K)], 0.0891637 secs] [Times: user=0.72 sys=0.01,
real=0.09 secs]
[vm1] [error 2017/02/14 09:36:13.738 UTC <Cache Server Acceptor
0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Cache server: Unexpected
IOException from accept
[vm1] [fatal 2017/02/14 09:36:13.739 UTC <unused p2p reader> tid=0x4f] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.738 UTC <Function Execution Processor1>
tid=0x48] Uncaught exception in thread Thread[Function Execution
Processor1,10,DistributionManager Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.740 UTC <Cache Server Acceptor
0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Uncaught exception in thread
Thread[Cache Server Acceptor 0.0.0.0/0.0.0.0:36823 local port: 36823,5,Acceptor
0.0.0.0/0.0.0.0:36823]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.744 UTC <unused p2p reader> tid=0x47] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:13.748 UTC <unused p2p reader> tid=0x54] Uncaught
exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
[vm1] [fatal 2017/02/14 09:36:14.602 UTC <ClientHealthMonitor Thread> tid=0x45]
Uncaught exception in thread Thread[ClientHealthMonitor
Thread,5,ClientHealthMonitor Thread Group]
[vm1] java.lang.OutOfMemoryError: Java heap space
[vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
[vm1] at java.lang.String.<init>(String.java:207)
[vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
[vm1] at
org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
[vm1] at
org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
[vm1] at
org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
[vm1] at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[vm1] at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
[vm1] at
org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
[vm1] at java.lang.Thread.run(Thread.java:745)
{noformat}
> Netstat command fails with OutOfMemoryError
> -------------------------------------------
>
> Key: GEODE-2488
> URL: https://issues.apache.org/jira/browse/GEODE-2488
> Project: Geode
> Issue Type: Bug
> Components: gfsh, management
> Reporter: Kirk Lund
> Labels: MiscellaneousCommands, NetstatCommand, netstat
>
> Note: this can occur outside of dunit tests as well. Just using the gfsh
> netstat command on locator or server with too little heap space will hit
> this. See the 1st comment about not streaming the netstat output -- the
> entire output is held in memory in a string and two byte arrays before
> sending the result back from NetstatFunction.
> The JUnit controller JVM for NetstatDUnitTest fails with something like the
> follow stack. The cause appears to be one of the DUnit VMs running out of
> memory (see the OOME stack down further in this description.
> {noformat}
> org.junit.ComparisonFailure: [{"errorCode":505,"message":["Could not process
> command due to GemFire error. Error occurred while executing netstat on
> [server-1]"]}] expected:<[OK]> but was:<[ERROR]>
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at
> org.apache.geode.test.dunit.rules.GfshShellConnectionRule.executeAndVerifyCommand(GfshShellConnectionRule.java:163)
> at
> org.apache.geode.management.internal.cli.NetstatDUnitTest.testConnectToJmxManagerOne(NetstatDUnitTest.java:81)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.apache.geode.test.junit.rules.DescribedExternalResource$1.evaluate(DescribedExternalResource.java:37)
> at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:114)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:57)
> at
> org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:66)
> at
> org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
> at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
> at
> org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
> at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:109)
> at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:377)
> at
> org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54)
> at
> org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
> The JVM that runs out of memory ends up logging multiple OutOfMemoryError
> stacks:
> {noformat}
> [vm1] 25.584: [GC (System.gc()) [PSYoungGen: 7724K->591K(161792K)]
> 242411K->235286K(511488K), 0.0047912 secs] [Times: user=0.02 sys=0.00,
> real=0.01 secs]
> [vm1] 25.589: [Full GC (System.gc()) [PSYoungGen: 591K->0K(161792K)]
> [ParOldGen: 234695K->9294K(349696K)] 235286K->9294K(511488K), [Metaspace:
> 34478K->34478K(1081344K)], 0.0891637 secs] [Times: user=0.72 sys=0.01,
> real=0.09 secs]
> [vm1] [error 2017/02/14 09:36:13.738 UTC <Cache Server Acceptor
> 0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Cache server: Unexpected
> IOException from accept
> [vm1] [fatal 2017/02/14 09:36:13.739 UTC <unused p2p reader> tid=0x4f]
> Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> [vm1] [fatal 2017/02/14 09:36:13.738 UTC <Function Execution Processor1>
> tid=0x48] Uncaught exception in thread Thread[Function Execution
> Processor1,10,DistributionManager Threads]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> [vm1] [fatal 2017/02/14 09:36:13.740 UTC <Cache Server Acceptor
> 0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Uncaught exception in
> thread Thread[Cache Server Acceptor 0.0.0.0/0.0.0.0:36823 local port:
> 36823,5,Acceptor 0.0.0.0/0.0.0.0:36823]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> [vm1] [fatal 2017/02/14 09:36:13.744 UTC <unused p2p reader> tid=0x47]
> Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> [vm1] [fatal 2017/02/14 09:36:13.748 UTC <unused p2p reader> tid=0x54]
> Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> [vm1] [fatal 2017/02/14 09:36:14.602 UTC <ClientHealthMonitor Thread>
> tid=0x45] Uncaught exception in thread Thread[ClientHealthMonitor
> Thread,5,ClientHealthMonitor Thread Group]
> [vm1] java.lang.OutOfMemoryError: Java heap space
> [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664)
> [vm1] at java.lang.String.<init>(String.java:207)
> [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193)
> [vm1] at
> org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70)
> [vm1] at
> org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621)
> [vm1] at
> org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067)
> [vm1] at java.lang.Thread.run(Thread.java:745)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
