[
https://issues.apache.org/jira/browse/GEODE-6471?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Kirk Lund updated GEODE-6471:
-----------------------------
Description:
ReconnectWithUDPSecurityDUnitTest seems to fail inconsistently with the
following NullPointerException thrown from GMSEncrypt. The field that is null
is `GMSEncryptionCipherPool clusterEncryptor` which is set under
synchronization but read without synchronization.
I tried changing the field to volatile but I can still get the test to fail
intermittently with the same NPE.
Simply moving addIgnoredException statements around in the super class
ReconnectDUnitTest seems to be enough to occasionally trigger a cluster of
repeated failures with this NPE.
I can't tell if there's a GMS bug, a configuration bug, a test bug or something
rotten the construction/usage of GMSEncrypt. JGroupsMessenger has a reference
to GMSEncrypt and that reference doesn't appear to be thread-safe either.
{noformat}
[error 2019/02/28 13:53:20.935 PST <ReconnectThread> tid=70] Exception
deserializing message payload: [dst: 10.118.33.197<v2>:41003, src:
10.118.33.197<v0>:41001 (2 headers), size=122 bytes,
flags=OOB|DONT_BUNDLE|NO_FC|SKIP_BARRIER]
java.lang.Exception: Message id is -154
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1138)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readJGMessage(JGroupsMessenger.java:1042)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1296)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.access$000(JGroupsMessenger.java:1261)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.started(JGroupsMessenger.java:419)
at
org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:164)
at
org.apache.geode.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:106)
at
org.apache.geode.distributed.internal.membership.MemberFactory.newMembershipManager(MemberFactory.java:93)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:782)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:900)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:541)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:830)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:442)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:428)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:253)
at
org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:164)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.reconnect(InternalDistributedSystem.java:2636)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.tryReconnect(InternalDistributedSystem.java:2463)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1278)
at
org.apache.geode.distributed.internal.ClusterDistributionManager$DMListener.membershipFailure(ClusterDistributionManager.java:3424)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.uncleanShutdown(GMSMembershipManager.java:1554)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.lambda$forceDisconnect$3(GMSMembershipManager.java:2586)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NullPointerException
at
org.apache.geode.distributed.internal.membership.gms.messenger.GMSEncrypt.decryptData(GMSEncrypt.java:148)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1117)
... 22 more
at org.junit.Assert.fail(Assert.java:88)
at
org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:380)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.cleanupAllVms(JUnit4DistributedTestCase.java:553)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.doTearDownDistributedTestCase(JUnit4DistributedTestCase.java:500)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.tearDownDistributedTestCase(JUnit4DistributedTestCase.java:484)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at
org.apache.geode.test.dunit.rules.AbstractDistributedRule$1.evaluate(AbstractDistributedRule.java:59)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:449)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:393)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
{noformat}
was:
ReconnectWithUDPSecurityDUnitTest seems to fail inconsistently with the
following NullPointerException thrown from GMSEncrypt. The field that is null
is `GMSEncryptionCipherPool clusterEncryptor` which is set under
synchronization but read without synchronization.
I tried changing the field to volatile but I can still get the test to fail
intermittently with the same NPE.
Simply moving addIgnoredException statements around in the super class
ReconnectDUnitTest seems to be enough to occasionally trigger a cluster of
repeated failures with this NPE.
I can't tell if there's a GMS bug, a configuration bug, a test bug or something
rotten the construction/usage of GMSEncrypt. JGroupsMessenger has a reference
to GMSEncrypt and that reference doesn't appear to be thread-safe either.
[error 2019/02/28 13:53:20.935 PST <ReconnectThread> tid=70] Exception
deserializing message payload: [dst: 10.118.33.197<v2>:41003, src:
10.118.33.197<v0>:41001 (2 headers), size=122 bytes,
flags=OOB|DONT_BUNDLE|NO_FC|SKIP_BARRIER]
java.lang.Exception: Message id is -154
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1138)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readJGMessage(JGroupsMessenger.java:1042)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1296)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.access$000(JGroupsMessenger.java:1261)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.started(JGroupsMessenger.java:419)
at
org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:164)
at
org.apache.geode.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:106)
at
org.apache.geode.distributed.internal.membership.MemberFactory.newMembershipManager(MemberFactory.java:93)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:782)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:900)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:541)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:830)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:442)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:428)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:253)
at
org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:164)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.reconnect(InternalDistributedSystem.java:2636)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.tryReconnect(InternalDistributedSystem.java:2463)
at
org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1278)
at
org.apache.geode.distributed.internal.ClusterDistributionManager$DMListener.membershipFailure(ClusterDistributionManager.java:3424)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.uncleanShutdown(GMSMembershipManager.java:1554)
at
org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.lambda$forceDisconnect$3(GMSMembershipManager.java:2586)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NullPointerException
at
org.apache.geode.distributed.internal.membership.gms.messenger.GMSEncrypt.decryptData(GMSEncrypt.java:148)
at
org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1117)
... 22 more
at org.junit.Assert.fail(Assert.java:88)
at
org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:380)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.cleanupAllVms(JUnit4DistributedTestCase.java:553)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.doTearDownDistributedTestCase(JUnit4DistributedTestCase.java:500)
at
org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.tearDownDistributedTestCase(JUnit4DistributedTestCase.java:484)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at
org.apache.geode.test.dunit.rules.AbstractDistributedRule$1.evaluate(AbstractDistributedRule.java:59)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:449)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:393)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
> Reconnect with UDP security throws NullPointerException from GMSEncrypt
> -----------------------------------------------------------------------
>
> Key: GEODE-6471
> URL: https://issues.apache.org/jira/browse/GEODE-6471
> Project: Geode
> Issue Type: Bug
> Components: membership, security
> Reporter: Kirk Lund
> Priority: Major
>
> ReconnectWithUDPSecurityDUnitTest seems to fail inconsistently with the
> following NullPointerException thrown from GMSEncrypt. The field that is null
> is `GMSEncryptionCipherPool clusterEncryptor` which is set under
> synchronization but read without synchronization.
> I tried changing the field to volatile but I can still get the test to fail
> intermittently with the same NPE.
> Simply moving addIgnoredException statements around in the super class
> ReconnectDUnitTest seems to be enough to occasionally trigger a cluster of
> repeated failures with this NPE.
> I can't tell if there's a GMS bug, a configuration bug, a test bug or
> something rotten the construction/usage of GMSEncrypt. JGroupsMessenger has a
> reference to GMSEncrypt and that reference doesn't appear to be thread-safe
> either.
> {noformat}
> [error 2019/02/28 13:53:20.935 PST <ReconnectThread> tid=70] Exception
> deserializing message payload: [dst: 10.118.33.197<v2>:41003, src:
> 10.118.33.197<v0>:41001 (2 headers), size=122 bytes,
> flags=OOB|DONT_BUNDLE|NO_FC|SKIP_BARRIER]
> java.lang.Exception: Message id is -154
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1138)
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readJGMessage(JGroupsMessenger.java:1042)
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1296)
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.access$000(JGroupsMessenger.java:1261)
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.started(JGroupsMessenger.java:419)
> at
> org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:164)
> at
> org.apache.geode.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:106)
> at
> org.apache.geode.distributed.internal.membership.MemberFactory.newMembershipManager(MemberFactory.java:93)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:782)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:900)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:541)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:830)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:442)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:428)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:253)
> at
> org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:164)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.reconnect(InternalDistributedSystem.java:2636)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.tryReconnect(InternalDistributedSystem.java:2463)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.disconnect(InternalDistributedSystem.java:1278)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager$DMListener.membershipFailure(ClusterDistributionManager.java:3424)
> at
> org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.uncleanShutdown(GMSMembershipManager.java:1554)
> at
> org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.lambda$forceDisconnect$3(GMSMembershipManager.java:2586)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.NullPointerException
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.GMSEncrypt.decryptData(GMSEncrypt.java:148)
> at
> org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readEncryptedMessage(JGroupsMessenger.java:1117)
> ... 22 more
> at org.junit.Assert.fail(Assert.java:88)
> at
> org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:380)
> at
> org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.cleanupAllVms(JUnit4DistributedTestCase.java:553)
> at
> org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.doTearDownDistributedTestCase(JUnit4DistributedTestCase.java:500)
> at
> org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.tearDownDistributedTestCase(JUnit4DistributedTestCase.java:484)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
> at
> org.apache.geode.test.dunit.rules.AbstractDistributedRule$1.evaluate(AbstractDistributedRule.java:59)
> at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
> at org.junit.rules.RunRules.evaluate(RunRules.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
> at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:449)
> at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:393)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
> at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
> at
> com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
> at
> com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
> at
> com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
