Here's a log snippet with the failure -- looks like there's a UDP socket WARN stack logged by this test before the ERROR stack. Is suspect string search no longer including WARN log statements?
[vm2] [info 2017/10/17 21:03:46.296 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] No locator(s) found with cluster configuration service [vm2] [info 2017/10/17 21:03:46.347 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Requesting cluster configuration [vm2] [info 2017/10/17 21:03:46.430 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Initializing region _monitoringRegion_172.17.0.4<v13>32772 [vm2] [info 2017/10/17 21:03:46.437 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Initialization of region _monitoringRegion_172.17.0.4<v13>32772 completed [vm2] 15.848: [GC (Allocation Failure) [PSYoungGen: 131584K->9505K(153088K)] 138902K->16831K(502784K), 0.0131806 secs] [Times: user=0.06 sys=0.00, real=0.01 secs] [vm1] [warn 2017/10/17 21:03:46.794 UTC <Geode Heartbeat Sender> tid=0x155] Unable to send message to 172.17.0.4(155)<v13>:32772 [vm1] java.io.IOException: Operation not permitted (sendto failed) [vm1] at java.net.PlainDatagramSocketImpl.send(Native Method) [vm1] at java.net.DatagramSocket.send(DatagramSocket.java:693) [vm1] at org.jgroups.protocols.UDP._send(UDP.java:224) [vm1] at org.jgroups.protocols.UDP.sendUnicast(UDP.java:215) [vm1] at org.jgroups.protocols.TP.sendToSingleMember(TP.java:1906) [vm1] at org.jgroups.protocols.TP.doSend(TP.java:1883) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.Transport.doSend(Transport.java:86) [vm1] at org.jgroups.protocols.TP.send(TP.java:1869) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.Transport._send(Transport.java:52) [vm1] at org.jgroups.protocols.TP.down(TP.java:1474) [vm1] at org.jgroups.stack.Protocol.down(Protocol.java:439) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.StatRecorder.down(StatRecorder.java:89) [vm1] at org.jgroups.protocols.UNICAST3.down(UNICAST3.java:675) [vm1] at org.jgroups.protocols.FlowControl.down(FlowControl.java:347) [vm1] at org.jgroups.protocols.FRAG2.down(FRAG2.java:136) [vm1] at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:1039) [vm1] at org.jgroups.JChannel.down(JChannel.java:790) [vm1] at org.jgroups.JChannel.send(JChannel.java:426) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.send(JGroupsMessenger.java:774) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.sendUnreliably(JGroupsMessenger.java:629) [vm1] at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.sendHeartbeats(GMSHealthMonitor.java:811) [vm1] at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.sendPeriodicHeartbeats(GMSHealthMonitor.java:769) [vm1] at org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$3.run(GMSHealthMonitor.java:751) [vm1] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [vm1] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [vm1] at java.lang.Thread.run(Thread.java:748) [locator] [info 2017/10/17 21:03:46.798 UTC <unicast receiver,fea3cee77fa7-28483> tid=0x2a] received suspect message from fea3cee77fa7(150)<v12>:32771 for fea3cee77fa7(155)<v13>:32772: Unable to send messages to this member via JGroups [locator] [info 2017/10/17 21:03:46.800 UTC <Geode Failure Detection thread 2> tid=0x7f] Performing final check for suspect member fea3cee77fa7(155)<v13>:32772 reason=Unable to send messages to this member via JGroups [locator] [info 2017/10/17 21:03:46.801 UTC <unicast receiver,fea3cee77fa7-28483> tid=0x2a] No longer suspecting fea3cee77fa7(155)<v13>:32772 [locator] [info 2017/10/17 21:03:46.803 UTC <Geode Failure Detection thread 2> tid=0x7f] Final check passed for suspect member fea3cee77fa7(155)<v13>:32772 [vm1] [error 2017/10/17 21:03:46.803 UTC <unicast receiver,fea3cee77fa7-3885> tid=0x150] Exception deserializing message payload: [dst: 172.17.0.4<v12>:32771, src: 172.17.0.4<v0>:32769 (2 headers), size=108 bytes, flags=OOB|DONT_BUNDLE|NO_FC|SKIP_BARRIER] [vm1] org.apache.geode.internal.DSFIDNotFoundException: Unknown DataSerializableFixedID: -158 [vm1] at org.apache.geode.internal.DSFIDFactory.create(DSFIDFactory.java:1003) [vm1] at org.apache.geode.internal.InternalDataSerializer.basicReadObject(InternalDataSerializer.java:2693) [vm1] at org.apache.geode.DataSerializer.readObject(DataSerializer.java:2961) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.deserializeMessage(JGroupsMessenger.java:1121) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger.readJGMessage(JGroupsMessenger.java:1013) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1279) [vm1] at org.jgroups.JChannel.invokeCallback(JChannel.java:816) [vm1] at org.jgroups.JChannel.up(JChannel.java:741) [vm1] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) [vm1] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) [vm1] at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) [vm1] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1070) [vm1] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:785) [vm1] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.StatRecorder.up(StatRecorder.java:74) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.AddressManager.up(AddressManager.java:72) [vm1] at org.jgroups.protocols.TP.passMessageUp(TP.java:1601) [vm1] at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1817) [vm1] at org.jgroups.util.DirectExecutor.execute(DirectExecutor.java:10) [vm1] at org.jgroups.protocols.TP.handleSingleMessage(TP.java:1729) [vm1] at org.jgroups.protocols.TP.receive(TP.java:1654) [vm1] at org.apache.geode.distributed.internal.membership.gms.messenger.Transport.receive(Transport.java:160) [vm1] at org.jgroups.protocols.UDP$PacketReceiver.run(UDP.java:701) [vm1] at java.lang.Thread.run(Thread.java:748) [vm2] [info 2017/10/17 21:03:47.047 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Loading previously deployed jars [vm2] [info 2017/10/17 21:03:47.061 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Initializing region PdxTypes [vm2] [info 2017/10/17 21:03:47.085 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Region PdxTypes requesting initial image from 172.17.0.4(150)<v12>:32771 [vm2] [info 2017/10/17 21:03:47.092 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] PdxTypes is done getting image from 172.17.0.4(150)<v12>:32771. isDeltaGII is false [vm2] [info 2017/10/17 21:03:47.092 UTC <RMI TCP Connection(1)-172.17.0.4> tid=0x1b] Initialization of region PdxTypes completed On Wed, Oct 18, 2017 at 10:03 AM, Kirk Lund <kl...@apache.org> wrote: > My latest sighting of this was in > ResourceManagerDUnitTest.testRemoveDuringGetEntry > which just shows this for previously run tests: > > Previously run tests: [ResourceManagerDUnitTest] > > On Tue, Oct 17, 2017 at 7:30 PM, Bruce Schuchardt <bschucha...@pivotal.io> > wrote: > >> I think this is a secure UDP test leaving something behind that's >> infecting other tests. If we can identify the previously run tests that >> might help. >> >> >> >> On 10/17/17 4:23 PM, Kirk Lund wrote: >> >>> At first I was just seeing a couple WAN tests fail with this but just >>> now I >>> had GIIDeltaDUnitTest fail due to this suspect string as well. >>> >>> Anyone know what's causing this? >>> >>> org.apache.geode.internal.cache.GIIDeltaDUnitTest > testSavingRVVGC >>> FAILED >>> java.lang.AssertionError: Suspicious strings were written to the log >>> during this run. >>> Fix the strings or use IgnoredException.addIgnoredException to >>> ignore. >>> ----------------------------------------------------------- >>> ------------ >>> Found suspect string in log4j at line 916 >>> >>> [error 2017/10/17 19:37:34.907 UTC <unicast >>> receiver,4f2f4190aa4e-57131> tid=0x130] Exception deserializing message >>> payload: [dst: 172.17.0.4<v9>:32771, src: 172.17.0.4<v0>:32769 (2 >>> headers), >>> size=108 bytes, flags=OOB|DONT_BUNDLE|NO_FC|SKIP_BARRIER] >>> org.apache.geode.internal.DSFIDNotFoundException: Unknown >>> DataSerializableFixedID: -158 >>> at org.apache.geode.internal.DSFIDFactory.create(DSFIDFactory.j >>> ava:1003) >>> at >>> org.apache.geode.internal.InternalDataSerializer.basicReadOb >>> ject(InternalDataSerializer.java:2693) >>> at org.apache.geode.DataSerializer.readObject(DataSerializer.ja >>> va:2961) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.JGroupsMessenger.deserializeMessage(JGroupsMessenger.java:1121) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.JGroupsMessenger.readJGMessage(JGroupsMessenger.java:1013) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.JGroupsMessenger$JGroupsReceiver.receive(JGroupsMessenger.java:1279) >>> at org.jgroups.JChannel.invokeCallback(JChannel.java:816) >>> at org.jgroups.JChannel.up(JChannel.java:741) >>> at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) >>> at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) >>> at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) >>> at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java: >>> 1070) >>> at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.j >>> ava:785) >>> at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.StatRecorder.up(StatRecorder.java:74) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.AddressManager.up(AddressManager.java:72) >>> at org.jgroups.protocols.TP.passMessageUp(TP.java:1601) >>> at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1817) >>> at org.jgroups.util.DirectExecutor.execute(DirectExecutor.java:10) >>> at org.jgroups.protocols.TP.handleSingleMessage(TP.java:1729) >>> at org.jgroups.protocols.TP.receive(TP.java:1654) >>> at >>> org.apache.geode.distributed.internal.membership.gms.messeng >>> er.Transport.receive(Transport.java:160) >>> at org.jgroups.protocols.UDP$PacketReceiver.run(UDP.java:701) >>> at java.lang.Thread.run(Thread.java:748) >>> >>> Thanks, >>> Kirk >>> >>> >> >
