New unit tests & minor bugfixes 1) new unit tests 2) detection of IDs w/o UUIDs in them in UDP messaging 3) removal of JoinResponse messages - view messages are considered a join-response so we don't need to send both 4) detection of new members admitted by a different coordinator during becomeCoordinator processes (problem found by Jianxia) 5) adjusting the join-timeout if member-timeout is large
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/a5906e5b Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/a5906e5b Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/a5906e5b Branch: refs/heads/feature/GEODE-291 Commit: a5906e5be3165b6afaa8424631a7a7f504c73f27 Parents: 6e32ffe Author: Bruce Schuchardt <[email protected]> Authored: Thu Dec 10 09:14:51 2015 -0800 Committer: Bruce Schuchardt <[email protected]> Committed: Thu Dec 10 09:16:12 2015 -0800 ---------------------------------------------------------------------- .../internal/InternalDistributedSystem.java | 19 ++- .../distributed/internal/ReplyProcessor21.java | 2 +- .../internal/membership/NetView.java | 19 +++ .../internal/membership/gms/GMSMember.java | 8 ++ .../internal/membership/gms/ServiceConfig.java | 21 ++- .../membership/gms/membership/GMSJoinLeave.java | 126 ++++++++++++------ .../gms/mgr/GMSMembershipManager.java | 18 ++- .../internal/DistributionManagerDUnitTest.java | 43 +++++- .../membership/MembershipJUnitTest.java | 130 +++++++------------ .../internal/membership/NetViewJUnitTest.java | 66 +++++++++- .../gms/fd/GMSHealthMonitorJUnitTest.java | 54 ++++---- .../gms/membership/GMSJoinLeaveJUnitTest.java | 87 ++++++++++--- .../gms/mgr/GMSMembershipManagerJUnitTest.java | 74 +++++++++++ .../internal/DataSerializableJUnitTest.java | 35 ++--- 14 files changed, 511 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/InternalDistributedSystem.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/InternalDistributedSystem.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/InternalDistributedSystem.java index 261b8a9..8f604b4 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/InternalDistributedSystem.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/InternalDistributedSystem.java @@ -285,6 +285,19 @@ public final class InternalDistributedSystem } } } + + + /** + * creates a non-functional instance for testing + * @param nonDefault - non-default distributed system properties + */ + public static InternalDistributedSystem newInstanceForTesting(DM dm, Properties nonDefault) { + InternalDistributedSystem sys = new InternalDistributedSystem(nonDefault); + sys.config = new RuntimeDistributionConfigImpl(sys); + sys.dm = dm; + sys.isConnected = true; + return sys; + } /** * Returns a connection to the distributed system that is suitable @@ -529,10 +542,8 @@ public final class InternalDistributedSystem } } - if (this.isLoner) { - this.config = new RuntimeDistributionConfigImpl(this); - } else { - this.config = new RuntimeDistributionConfigImpl(this); + this.config = new RuntimeDistributionConfigImpl(this); + if (!this.isLoner) { this.attemptingToReconnect = (reconnectAttemptCounter > 0); } try { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/ReplyProcessor21.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/ReplyProcessor21.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/ReplyProcessor21.java index d12e78d..aa5f66c 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/ReplyProcessor21.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/ReplyProcessor21.java @@ -757,7 +757,7 @@ public class ReplyProcessor21 else { if (msecs > timeout) { if (!latch.await(timeout)) { - timeout(false, false); + timeout(isSevereAlertProcessingEnabled() && (severeAlertTimeout > 0), false); // after timeout alert, wait remaining time if (!latch.await(msecs-timeout)) { logger.info(LocalizedMessage.create(LocalizedStrings.ReplyProcessor21_WAIT_FOR_REPLIES_TIMING_OUT_AFTER_0_SEC, Long.valueOf(msecs / 1000))); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java index a90a45d..6a68619 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java @@ -230,6 +230,10 @@ public class NetView implements DataSerializableFixedID { this.crashedMembers.addAll(mbr); } + public void addShutdownMembers(Set<InternalDistributedMember> mbr) { + this.shutdownMembers.addAll(mbr); + } + public boolean remove(InternalDistributedMember mbr) { this.hashedMembers.remove(mbr); int idx = this.members.indexOf(mbr); @@ -518,6 +522,21 @@ public class NetView implements DataSerializableFixedID { sb.append("]"); return sb.toString(); } + + /** + * Returns the ID from this view that is equal to the argument. + * If no such ID exists the argument is returned. + */ + public synchronized InternalDistributedMember getCanonicalID(InternalDistributedMember id) { + if (hashedMembers.contains(id)) { + for (InternalDistributedMember m: this.members) { + if (id.equals(m)) { + return m; + } + } + } + return id; + } @Override public synchronized boolean equals(Object arg0) { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/GMSMember.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/GMSMember.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/GMSMember.java index 05b3aee..b1a4883 100755 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/GMSMember.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/GMSMember.java @@ -374,6 +374,14 @@ public class GMSMember implements NetMember, DataSerializableFixedID { public void setPort(int p) { this.udpPort = p; } + + /** + * checks to see if this address has UUID information needed + * to send messages via JGroups + */ + public boolean hasUUID() { + return !(this.uuidLSBs == 0 && this.uuidMSBs == 0); + } @Override public Version[] getSerializationVersions() { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/ServiceConfig.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/ServiceConfig.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/ServiceConfig.java index 1d67bbf..a412dfa 100755 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/ServiceConfig.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/ServiceConfig.java @@ -20,12 +20,17 @@ import java.net.InetAddress; import com.gemstone.gemfire.distributed.Locator; import com.gemstone.gemfire.distributed.internal.DistributionConfig; +import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave; import com.gemstone.gemfire.internal.SocketCreator; import com.gemstone.gemfire.internal.admin.remote.RemoteTransportConfig; public class ServiceConfig { + + /** stall time to wait for concurrent join/leave/remove requests to be received */ + public static final long MEMBER_REQUEST_COLLECTION_INTERVAL = Long.getLong("gemfire.member-request-collection-interval", 500); + /** various settings from Geode configuration */ - private int joinTimeout; + private long joinTimeout; private int[] membershipPortRange; private int udpRecvBufferSize; private int udpSendBufferSize; @@ -47,7 +52,7 @@ public class ServiceConfig { } - public int getJoinTimeout() { + public long getJoinTimeout() { return joinTimeout; } @@ -128,11 +133,19 @@ public class ServiceConfig { this.dconfig = theConfig; this.transport = transport; - int defaultJoinTimeout = 24000; + long defaultJoinTimeout = 24000; if (theConfig.getLocators().length() > 0 && !Locator.hasLocators()) { defaultJoinTimeout = 60000; } - joinTimeout = Integer.getInteger("p2p.joinTimeout", defaultJoinTimeout).intValue(); + + // we need to have enough time to figure out that the coordinator has crashed & + // find a new one + long minimumJoinTimeout = dconfig.getMemberTimeout() * 2 + MEMBER_REQUEST_COLLECTION_INTERVAL; + if (defaultJoinTimeout < minimumJoinTimeout) { + defaultJoinTimeout = minimumJoinTimeout; + }; + + joinTimeout = Long.getLong("p2p.joinTimeout", defaultJoinTimeout).longValue(); // if network partition detection is enabled, we must connect to the locators // more frequently in order to make sure we're not isolated from them http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java index 3e767ae..ccc9d8c 100755 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java @@ -25,6 +25,7 @@ import static com.gemstone.gemfire.internal.DataSerializableFixedID.LEAVE_REQUES import static com.gemstone.gemfire.internal.DataSerializableFixedID.NETWORK_PARTITION_MESSAGE; import static com.gemstone.gemfire.internal.DataSerializableFixedID.REMOVE_MEMBER_REQUEST; import static com.gemstone.gemfire.internal.DataSerializableFixedID.VIEW_ACK_MESSAGE; +import static com.gemstone.gemfire.distributed.internal.membership.gms.ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL; import java.io.IOException; import java.net.InetSocketAddress; @@ -96,9 +97,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { /** amount of time to sleep before trying to join after a failed attempt */ private static final int JOIN_RETRY_SLEEP = Integer.getInteger("gemfire.join-retry-sleep", 1000); - /** stall time to wait for concurrent join/leave/remove requests to be received */ - public static final long MEMBER_REQUEST_COLLECTION_INTERVAL = Long.getLong("gemfire.member-request-collection-interval", 500); - /** time to wait for a leave request to be transmitted by jgroups */ private static final long LEAVE_MESSAGE_SLEEP_TIME = Long.getLong("gemfire.leave-message-sleep-time", 1000); @@ -664,7 +662,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { } if (viewCreator == null || viewCreator.isShutdown()) { viewCreator = new ViewCreator("Geode Membership View Creator", Services.getThreadGroup()); - viewCreator.setInitialView(newView, leaving, removals); + viewCreator.setInitialView(newView, newView.getNewMembers(), leaving, removals); viewCreator.setDaemon(true); viewCreator.start(); startViewBroadcaster(); @@ -672,13 +670,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { } } - private void sendJoinResponses(List<InternalDistributedMember> newMbrs, NetView newView) { - for (InternalDistributedMember mbr : newMbrs) { - JoinResponseMessage response = new JoinResponseMessage(mbr, newView); - services.getMessenger().send(response); - } - } - private void sendRemoveMessages(List<InternalDistributedMember> removals, List<String> reasons, NetView newView) { Iterator<String> reason = reasons.iterator(); for (InternalDistributedMember mbr : removals) { @@ -747,12 +738,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { // only wait for responses during preparation if (preparing) { - // send join responses after other members at least have - // a prepared view announcing the new member - if (!(isNetworkPartition && quorumRequired)) { - sendJoinResponses(newMembers, view); - } - logger.debug("waiting for view responses"); Set<InternalDistributedMember> failedToRespond = rp.waitForResponses(); @@ -1330,22 +1315,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { stopCoordinatorServices(); if (view != null) { if (view.size() > 1) { - if (this.isCoordinator) { - logger.debug("JoinLeave stopping coordination services"); - NetView newView = new NetView(view, view.getViewId() + 1); - newView.remove(localAddress); - InstallViewMessage m = new InstallViewMessage(newView, services.getAuthenticator().getCredentials(this.localAddress)); - m.setRecipients(newView.getMembers()); - services.getMessenger().send(m); - waitForProcessing = true; - } else { - List<InternalDistributedMember> coords = view.getPreferredCoordinators(Collections.<InternalDistributedMember> emptySet(), localAddress, 5); + List<InternalDistributedMember> coords = view.getPreferredCoordinators(Collections.<InternalDistributedMember> emptySet(), localAddress, 5); - logger.debug("JoinLeave sending a leave request to {}", coords); - LeaveRequestMessage m = new LeaveRequestMessage(coords, this.localAddress, "this member is shutting down"); - services.getMessenger().send(m); - waitForProcessing = true; - } + logger.debug("JoinLeave sending a leave request to {}", coords); + LeaveRequestMessage m = new LeaveRequestMessage(coords, this.localAddress, "this member is shutting down"); + services.getMessenger().send(m); + waitForProcessing = true; } // view.size } // view != null } @@ -1690,8 +1665,21 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { boolean shutdown = false; volatile boolean waiting = false; + /** + * initial view to install. guarded by synch on ViewCreator + */ NetView initialView; + /** + * initial joining members. guarded by synch on ViewCreator + */ + List<InternalDistributedMember> initialJoins = Collections.<InternalDistributedMember>emptyList(); + /** + * initial leaving members guarded by synch on ViewCreator + */ Set<InternalDistributedMember> initialLeaving; + /** + * initial crashed members. guarded by synch on ViewCreator + */ Set<InternalDistributedMember> initialRemovals; ViewCreator(String name, ThreadGroup tg) { @@ -1723,22 +1711,72 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { * @param leaving - members leaving in this view * @param removals - members crashed in this view */ - void setInitialView(NetView newView, Set<InternalDistributedMember> leaving, Set<InternalDistributedMember> removals) { + synchronized void setInitialView(NetView newView, List<InternalDistributedMember> newMembers, + Set<InternalDistributedMember> leaving, Set<InternalDistributedMember> removals) { this.initialView = newView; + this.initialJoins = newMembers; this.initialLeaving = leaving; this.initialRemovals = removals; } private void sendInitialView() { - if (initialView != null) { - try { - prepareAndSendView(initialView, Collections.<InternalDistributedMember>emptyList(), initialLeaving, - initialRemovals); - } finally { - this.initialView = null; - this.initialLeaving = null; - this.initialRemovals = null; + if (initialView == null) { + return; + } + NetView v = preparedView; + if (v != null) { + processPreparedView(v); + } + try { + NetView iView; + List<InternalDistributedMember> iJoins; + Set<InternalDistributedMember> iLeaves; + Set<InternalDistributedMember> iRemoves; + synchronized(this) { + iView = initialView; + iJoins = initialJoins; + iLeaves = initialLeaving; + iRemoves = initialRemovals; + } + if (iView != null) { + prepareAndSendView(iView, iJoins, iLeaves, iRemoves); + } + } finally { + setInitialView(null, null, null, null); + } + } + + /** + * During initial view processing a prepared view was discovered. + * This method will extract its new members and create a new + * initial view containing them. + * + * @param v The prepared view + */ + private void processPreparedView(NetView v) { + assert initialView != null; + if (currentView == null || currentView.getViewId() < v.getViewId()) { + // we have a prepared view that is newer than the current view + // form a new View ID + int viewId = Math.max(initialView.getViewId(),v.getViewId()); + viewId += 1; + NetView newView = new NetView(initialView, viewId); + + // add the new members from the prepared view to the new view, + // preserving their failure-detection ports + List<InternalDistributedMember> newMembers; + if (currentView != null) { + newMembers = v.getNewMembers(currentView); + } else { + newMembers = v.getMembers(); } + for (InternalDistributedMember newMember: newMembers) { + newView.add(newMember); + newView.setFailureDetectionPort(newMember, v.getFailureDetectionPort(newMember)); + } + + // use the new view as the initial view + setInitialView(newView, newMembers, initialLeaving, initialRemovals); } } @@ -2007,6 +2045,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler { logger.info("adding these crashed members from a conflicting view to the crash-set for the next view: {}\nconflicting view: {}", unresponsive, conflictingView); failures.addAll(conflictingView.getCrashedMembers()); + // this member may have been kicked out of the conflicting view + if (failures.contains(localAddress)) { + forceDisconnect("I am no longer a member of the distributed system"); + shutdown = true; + return; + } List<InternalDistributedMember> newMembers = conflictingView.getNewMembers(); if (!newMembers.isEmpty()) { logger.info("adding these new members from a conflicting view to the new view: {}", newMembers); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManager.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManager.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManager.java index 4e108be..bbe7ab3 100755 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManager.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManager.java @@ -39,6 +39,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.logging.log4j.Logger; +import com.gemstone.gemfire.CancelCriterion; import com.gemstone.gemfire.CancelException; import com.gemstone.gemfire.ForcedDisconnectException; import com.gemstone.gemfire.GemFireConfigException; @@ -72,6 +73,7 @@ import com.gemstone.gemfire.distributed.internal.membership.MembershipManager; import com.gemstone.gemfire.distributed.internal.membership.MembershipTestHook; import com.gemstone.gemfire.distributed.internal.membership.NetView; import com.gemstone.gemfire.distributed.internal.membership.QuorumChecker; +import com.gemstone.gemfire.distributed.internal.membership.gms.GMSMember; import com.gemstone.gemfire.distributed.internal.membership.gms.GMSUtil; import com.gemstone.gemfire.distributed.internal.membership.gms.Services; import com.gemstone.gemfire.distributed.internal.membership.gms.SuspectMember; @@ -1429,7 +1431,6 @@ public class GMSMembershipManager implements MembershipManager, Manager } } - protected boolean isJoining() { return this.isJoining; } @@ -1971,6 +1972,7 @@ public class GMSMembershipManager implements MembershipManager, Manager boolean sendViaMessenger = isForceUDPCommunications(); // enable when bug #46438 is fixed: || msg.sendViaUDP(); if (useMcast || tcpDisabled || sendViaMessenger) { + checkAddressesForUUIDs(destinations); result = services.getMessenger().send(msg); } else { @@ -1991,6 +1993,20 @@ public class GMSMembershipManager implements MembershipManager, Manager forceUseUDPMessaging.set(null); } + void checkAddressesForUUIDs(InternalDistributedMember[] addresses) { + for (int i=0; i<addresses.length; i++) { + GMSMember id = (GMSMember)addresses[i].getNetMember(); + if (!id.hasUUID()) { + latestViewLock.readLock().lock(); + try { + addresses[i] = latestView.getCanonicalID(addresses[i]); + } finally { + latestViewLock.readLock().unlock(); + } + } + } + } + private boolean isForceUDPCommunications() { Boolean forced = forceUseUDPMessaging.get(); return forced == Boolean.TRUE; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/DistributionManagerDUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/DistributionManagerDUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/DistributionManagerDUnitTest.java index 51771cb..1f411bb 100644 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/DistributionManagerDUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/DistributionManagerDUnitTest.java @@ -19,9 +19,8 @@ package com.gemstone.gemfire.distributed.internal; import java.net.InetAddress; import java.util.Properties; -import junit.framework.Assert; - import org.apache.logging.log4j.Logger; +import org.junit.Assert; import com.gemstone.gemfire.LogWriter; import com.gemstone.gemfire.admin.AdminDistributedSystem; @@ -43,7 +42,9 @@ import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.distributed.DistributedSystem; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.distributed.internal.membership.MembershipManager; +import com.gemstone.gemfire.distributed.internal.membership.NetView; import com.gemstone.gemfire.distributed.internal.membership.gms.MembershipManagerHelper; +import com.gemstone.gemfire.distributed.internal.membership.gms.interfaces.Manager; import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager; import com.gemstone.gemfire.internal.AvailablePort; import com.gemstone.gemfire.internal.logging.LogService; @@ -538,4 +539,42 @@ public class DistributionManagerDUnitTest extends DistributedTestCase { props.setProperty(DistributionConfig.BIND_ADDRESS_NAME, InetAddress.getLocalHost().getCanonicalHostName()); getSystem().disconnect(); } + + /** + * install a new view and show that waitForViewInstallation works as expected + */ + public void testWaitForViewInstallation() { + getSystem(new Properties()); + + MembershipManager mgr = system.getDM().getMembershipManager(); + + final NetView v = mgr.getView(); + + final boolean[] passed = new boolean[1]; + Thread t = new Thread("wait for view installation") { + public void run() { + try { + ((DistributionManager)system.getDM()).waitForViewInstallation(v.getViewId()+1); + synchronized(passed) { + passed[0] = true; + } + } catch (InterruptedException e) { + // failed + } + } + }; + t.setDaemon(true); + t.start(); + + pause(2000); + + NetView newView = new NetView(v, v.getViewId()+1); + ((Manager)mgr).installView(newView); + + pause(2000); + + synchronized(passed) { + Assert.assertTrue(passed[0]); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/MembershipJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/MembershipJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/MembershipJUnitTest.java index bee2367..7a4971f 100755 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/MembershipJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/MembershipJUnitTest.java @@ -16,28 +16,17 @@ */ package com.gemstone.gemfire.distributed.internal.membership; -import static org.mockito.Mockito.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.File; import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; import java.util.Properties; -import java.util.Set; - -import junit.framework.TestCase; import org.apache.logging.log4j.Level; import org.junit.AfterClass; @@ -47,12 +36,10 @@ import org.junit.experimental.categories.Category; import com.gemstone.gemfire.GemFireConfigException; import com.gemstone.gemfire.distributed.Locator; -import com.gemstone.gemfire.distributed.internal.DM; import com.gemstone.gemfire.distributed.internal.DMStats; import com.gemstone.gemfire.distributed.internal.DistributionConfig; import com.gemstone.gemfire.distributed.internal.DistributionConfigImpl; import com.gemstone.gemfire.distributed.internal.DistributionManager; -import com.gemstone.gemfire.distributed.internal.DistributionMessage; import com.gemstone.gemfire.distributed.internal.InternalLocator; import com.gemstone.gemfire.distributed.internal.SerialAckedMessage; import com.gemstone.gemfire.distributed.internal.membership.gms.GMSUtil; @@ -90,70 +77,7 @@ public class MembershipJUnitTest { // LogService.setBaseLogLevel(baseLogLevel); } - /** - * Test that failed weight calculations are correctly performed. See bug #47342 - * @throws Exception - */ - public void testFailedWeight() throws Exception { - // in #47342 a new view was created that contained a member that was joining but - // was no longer reachable. The member was included in the failed-weight and not - // in the previous view-weight, causing a spurious network partition to be declared - InternalDistributedMember members[] = new InternalDistributedMember[] { - new InternalDistributedMember("localhost", 1), new InternalDistributedMember("localhost", 2), new InternalDistributedMember("localhost", 3), - new InternalDistributedMember("localhost", 4), new InternalDistributedMember("localhost", 5), new InternalDistributedMember("localhost", 6)}; - int i = 0; - // weight 3 - members[i].setVmKind(DistributionManager.LOCATOR_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(true); - // weight 3 - members[i].setVmKind(DistributionManager.LOCATOR_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(true); - // weight 15 (cache+leader) - members[i].setVmKind(DistributionManager.NORMAL_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(false); - // weight 0 - members[i].setVmKind(DistributionManager.ADMIN_ONLY_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(false); - // weight 0 - members[i].setVmKind(DistributionManager.ADMIN_ONLY_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(false); - // weight 10 - members[i].setVmKind(DistributionManager.NORMAL_DM_TYPE); - members[i++].getNetMember().setPreferredForCoordinator(false); - - List<InternalDistributedMember> vmbrs = new ArrayList<>(members.length); - for (i=0; i<members.length; i++) { - vmbrs.add(members[i]); - } - Set<InternalDistributedMember> empty = Collections.emptySet(); - NetView lastView = new NetView(members[0], 4, vmbrs, empty, empty); - InternalDistributedMember leader = members[2]; - assertTrue(!leader.getNetMember().preferredForCoordinator()); - - InternalDistributedMember joiningMember = new InternalDistributedMember("localhost", 7); - joiningMember.setVmKind(DistributionManager.NORMAL_DM_TYPE); - joiningMember.getNetMember().setPreferredForCoordinator(false); - - // have the joining member and another cache process (weight 10) in the failed members - // collection and check to make sure that the joining member is not included in failed - // weight calcs. - Set<InternalDistributedMember> failedMembers = new HashSet<>(3); - failedMembers.add(joiningMember); - failedMembers.add(members[members.length-1]); // cache - failedMembers.add(members[members.length-2]); // admin - List<InternalDistributedMember> newMbrs = new ArrayList<InternalDistributedMember>(lastView.getMembers()); - newMbrs.removeAll(failedMembers); - NetView newView = new NetView(members[0], 5, newMbrs, empty, failedMembers); - - int failedWeight = newView.getCrashedMemberWeight(lastView); -// System.out.println("last view = " + lastView); -// System.out.println("failed mbrs = " + failedMembers); -// System.out.println("failed weight = " + failedWeight); - assertEquals("failure weight calculation is incorrect", 10, failedWeight); - Set<InternalDistributedMember> actual = newView.getActualCrashedMembers(lastView); - assertTrue(!actual.contains(members[members.length-2])); - } - + // @Test // public void testRepeat() throws Exception { // for (int i=0; i<50; i++) { @@ -292,6 +216,48 @@ public class MembershipJUnitTest { } } } + + @Test + public void testJoinTimeoutSetting() throws Exception { + long timeout = 30000; + Properties nonDefault = new Properties(); + nonDefault.put(DistributionConfig.MEMBER_TIMEOUT_NAME, ""+timeout); + DistributionConfigImpl config = new DistributionConfigImpl(nonDefault); + RemoteTransportConfig transport = new RemoteTransportConfig(config, + DistributionManager.NORMAL_DM_TYPE); + ServiceConfig sc = new ServiceConfig(transport, config); + assertEquals(2 * timeout + ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL, sc.getJoinTimeout()); + + nonDefault.clear(); + config = new DistributionConfigImpl(nonDefault); + transport = new RemoteTransportConfig(config, + DistributionManager.NORMAL_DM_TYPE); + sc = new ServiceConfig(transport, config); + assertEquals(24000, sc.getJoinTimeout()); + + + nonDefault.clear(); + nonDefault.put(DistributionConfig.LOCATORS_NAME, SocketCreator.getLocalHost().getHostAddress()+"["+12345+"]"); + config = new DistributionConfigImpl(nonDefault); + transport = new RemoteTransportConfig(config, + DistributionManager.NORMAL_DM_TYPE); + sc = new ServiceConfig(transport, config); + assertEquals(60000, sc.getJoinTimeout()); + + + timeout = 2000; + System.setProperty("p2p.joinTimeout", ""+timeout); + try { + config = new DistributionConfigImpl(nonDefault); + transport = new RemoteTransportConfig(config, + DistributionManager.NORMAL_DM_TYPE); + sc = new ServiceConfig(transport, config); + assertEquals(timeout, sc.getJoinTimeout()); + } finally { + System.getProperties().remove("p2p.joinTimeout"); + } + + } @Test public void testMulticastDiscoveryNotAllowed() { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/NetViewJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/NetViewJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/NetViewJUnitTest.java index 603c7bf..9e39d0f 100755 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/NetViewJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/NetViewJUnitTest.java @@ -191,4 +191,68 @@ public class NetViewJUnitTest { assertEquals(100, view.getNewMembers(copy).size()); } -} + /** + * Test that failed weight calculations are correctly performed. See bug #47342 + * @throws Exception + */ + @Test + public void testFailedWeight() throws Exception { + // in #47342 a new view was created that contained a member that was joining but + // was no longer reachable. The member was included in the failed-weight and not + // in the previous view-weight, causing a spurious network partition to be declared + InternalDistributedMember members[] = new InternalDistributedMember[] { + new InternalDistributedMember("localhost", 1), new InternalDistributedMember("localhost", 2), new InternalDistributedMember("localhost", 3), + new InternalDistributedMember("localhost", 4), new InternalDistributedMember("localhost", 5), new InternalDistributedMember("localhost", 6)}; + int i = 0; + // weight 3 + members[i].setVmKind(DistributionManager.LOCATOR_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(true); + // weight 3 + members[i].setVmKind(DistributionManager.LOCATOR_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(true); + // weight 15 (cache+leader) + members[i].setVmKind(DistributionManager.NORMAL_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(false); + // weight 0 + members[i].setVmKind(DistributionManager.ADMIN_ONLY_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(false); + // weight 0 + members[i].setVmKind(DistributionManager.ADMIN_ONLY_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(false); + // weight 10 + members[i].setVmKind(DistributionManager.NORMAL_DM_TYPE); + members[i++].getNetMember().setPreferredForCoordinator(false); + + List<InternalDistributedMember> vmbrs = new ArrayList<>(members.length); + for (i=0; i<members.length; i++) { + vmbrs.add(members[i]); + } + Set<InternalDistributedMember> empty = Collections.emptySet(); + NetView lastView = new NetView(members[0], 4, vmbrs, empty, empty); + InternalDistributedMember leader = members[2]; + assertTrue(!leader.getNetMember().preferredForCoordinator()); + + InternalDistributedMember joiningMember = new InternalDistributedMember("localhost", 7); + joiningMember.setVmKind(DistributionManager.NORMAL_DM_TYPE); + joiningMember.getNetMember().setPreferredForCoordinator(false); + + // have the joining member and another cache process (weight 10) in the failed members + // collection and check to make sure that the joining member is not included in failed + // weight calcs. + Set<InternalDistributedMember> failedMembers = new HashSet<>(3); + failedMembers.add(joiningMember); + failedMembers.add(members[members.length-1]); // cache + failedMembers.add(members[members.length-2]); // admin + List<InternalDistributedMember> newMbrs = new ArrayList<InternalDistributedMember>(lastView.getMembers()); + newMbrs.removeAll(failedMembers); + NetView newView = new NetView(members[0], 5, newMbrs, empty, failedMembers); + + int failedWeight = newView.getCrashedMemberWeight(lastView); +// System.out.println("last view = " + lastView); +// System.out.println("failed mbrs = " + failedMembers); +// System.out.println("failed weight = " + failedWeight); + assertEquals("failure weight calculation is incorrect", 10, failedWeight); + Set<InternalDistributedMember> actual = newView.getActualCrashedMembers(lastView); + assertTrue(!actual.contains(members[members.length-2])); + } + } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java index 585ff17..c4ac3a6 100644 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java @@ -20,11 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -46,8 +42,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import com.gemstone.gemfire.distributed.internal.DistributionConfig; import com.gemstone.gemfire.distributed.internal.DistributionManager; @@ -292,7 +286,6 @@ public class GMSHealthMonitorJUnitTest { NetView v = new NetView(mockMembers.get(0), 2, mockMembers, new HashSet<InternalDistributedMember>(), new HashSet<InternalDistributedMember>()); - MethodExecuted messageSent = new MethodExecuted(); // 3rd is current member when(messenger.getMemberID()).thenReturn(mockMembers.get(3)); @@ -300,15 +293,13 @@ public class GMSHealthMonitorJUnitTest { gmsHealthMonitor.suspect(mockMembers.get(1), "Not responding"); - when(messenger.send(isA(SuspectMembersMessage.class))).thenAnswer(messageSent); - try { // suspect thread timeout is 200 ms Thread.sleep(100l); } catch (InterruptedException e) { } - assertTrue("SuspectMembersMessage shouldn't have sent", !messageSent.isMethodExecuted()); + verify(messenger, atLeastOnce()).send(isA(SuspectMembersMessage.class)); } /*** @@ -544,6 +535,33 @@ public class GMSHealthMonitorJUnitTest { int byteReply = dis.read(); Assert.assertEquals(expectedResult, byteReply); } + + @Test + public void testBeSickAndPlayDead() throws Exception { + NetView v = new NetView(mockMembers.get(0), 2, mockMembers, new HashSet<InternalDistributedMember>(), new HashSet<InternalDistributedMember>()); + gmsHealthMonitor.installView(v); + gmsHealthMonitor.beSick(); + + // a sick member will not respond to a heartbeat request + HeartbeatRequestMessage req = new HeartbeatRequestMessage(mockMembers.get(0), 10); + req.setSender(mockMembers.get(0)); + gmsHealthMonitor.processMessage(req); + verify(messenger, never()).send(isA(HeartbeatMessage.class)); + + // a sick member will not record a heartbeat from another member + HeartbeatMessage hb = new HeartbeatMessage(-1); + hb.setSender(mockMembers.get(0)); + gmsHealthMonitor.processMessage(hb); + assertTrue(gmsHealthMonitor.memberTimeStamps.get(hb.getSender()) == null); + + // a sick member will not take action on a Suspect message from another member + SuspectMembersMessage smm = mock(SuspectMembersMessage.class); + Error err = new AssertionError("expected suspect message to be ignored"); + when(smm.getMembers()).thenThrow(err); + when(smm.getSender()).thenThrow(err); + when(smm.getDSFID()).thenCallRealMethod(); + gmsHealthMonitor.processMessage(smm); + } private GMSMember createGMSMember(short version, int viewId, long msb, long lsb) { GMSMember gmsMember = new GMSMember(); @@ -560,18 +578,4 @@ public class GMSHealthMonitorJUnitTest { return baos.toByteArray(); } - - private class MethodExecuted implements Answer { - private boolean methodExecuted = false; - - public boolean isMethodExecuted() { - return methodExecuted; - } - - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - methodExecuted = true; - return null; - } - } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java index 9895f68..01c0695 100644 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java @@ -16,6 +16,7 @@ */ package com.gemstone.gemfire.distributed.internal.membership.gms.membership; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -43,12 +44,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.ArgumentCaptor; import org.mockito.internal.verification.Times; -import org.mockito.internal.verification.api.VerificationData; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.mockito.verification.Timeout; -import org.mockito.verification.VerificationMode; -import org.mockito.verification.VerificationWithTimeout; import com.gemstone.gemfire.distributed.internal.DistributionConfig; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; @@ -63,6 +61,7 @@ import com.gemstone.gemfire.distributed.internal.membership.gms.interfaces.Manag import com.gemstone.gemfire.distributed.internal.membership.gms.interfaces.Messenger; import com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorResponse; import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.SearchState; +import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.ViewCreator; import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.ViewReplyProcessor; import com.gemstone.gemfire.distributed.internal.membership.gms.messages.InstallViewMessage; import com.gemstone.gemfire.distributed.internal.membership.gms.messages.JoinRequestMessage; @@ -72,8 +71,6 @@ import com.gemstone.gemfire.distributed.internal.membership.gms.messages.Network import com.gemstone.gemfire.distributed.internal.membership.gms.messages.RemoveMemberMessage; import com.gemstone.gemfire.distributed.internal.membership.gms.messages.ViewAckMessage; import com.gemstone.gemfire.internal.Version; -import com.gemstone.gemfire.internal.admin.remote.AddStatListenerResponse; -import com.gemstone.gemfire.internal.admin.remote.StatListenerMessage; import com.gemstone.gemfire.security.AuthenticationFailedException; import com.gemstone.gemfire.test.junit.categories.UnitTest; @@ -297,7 +294,7 @@ public class GMSJoinLeaveJUnitTest { MethodExecuted removeMessageSent = new MethodExecuted(); when(messenger.send(any(RemoveMemberMessage.class))).thenAnswer(removeMessageSent); gmsJoinLeave.remove(mockMembers[0], "removing for test"); - Thread.sleep(GMSJoinLeave.MEMBER_REQUEST_COLLECTION_INTERVAL*2); + Thread.sleep(ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL*2); assertTrue(removeMessageSent.methodExecuted); } @@ -655,9 +652,6 @@ public class GMSJoinLeaveJUnitTest { public void testNetworkPartitionMessageReceived() throws Exception { initMocks(); gmsJoinLeave.becomeCoordinatorForTest(); - List<InternalDistributedMember> members = Arrays.asList(mockMembers); - Set<InternalDistributedMember> empty = Collections.<InternalDistributedMember>emptySet(); - NetView v = new NetView(mockMembers[0], 2, members, empty, empty); NetworkPartitionMessage message = new NetworkPartitionMessage(); gmsJoinLeave.processMessage(message); verify(manager).forceDisconnect(any(String.class)); @@ -805,7 +799,7 @@ public class GMSJoinLeaveJUnitTest { gmsJoinLeave.memberShutdown(mockMembers[2], "Shutdown"); //Install a view that still contains one of the left members (as if something like a new member, triggered a new view before coordinator leaves) - NetView netView = new NetView(mockMembers[0], 3/*new view id*/, createMemberList(mockMembers[0], gmsJoinLeaveMemberId, mockMembers[1], mockMembers[3]), new HashSet(), new HashSet()); + NetView netView = new NetView(mockMembers[0], 3/*new view id*/, createMemberList(mockMembers[0], gmsJoinLeaveMemberId, mockMembers[1], mockMembers[3]), new HashSet<InternalDistributedMember>(), new HashSet<InternalDistributedMember>()); InstallViewMessage installViewMessage = new InstallViewMessage(netView, credentials, false); gmsJoinLeave.processMessage(installViewMessage); @@ -866,11 +860,11 @@ public class GMSJoinLeaveJUnitTest { int viewRequests = gmsJoinLeave.getViewRequests().size(); assertTrue( "There should be 1 viewRequest but found " + viewRequests, viewRequests == 1); - Thread.sleep(2 * GMSJoinLeave.MEMBER_REQUEST_COLLECTION_INTERVAL); + Thread.sleep(2 * ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL); viewRequests = gmsJoinLeave.getViewRequests().size(); - assertTrue( "There should be 0 viewRequest but found " + viewRequests, viewRequests == 0); - }finally { + assertEquals( "Found view requests: " + gmsJoinLeave.getViewRequests(), 0, viewRequests); + } finally { System.getProperties().remove(GMSJoinLeave.BYPASS_DISCOVERY_PROPERTY); } } @@ -946,7 +940,7 @@ public class GMSJoinLeaveJUnitTest { InternalDistributedMember ids = new InternalDistributedMember("localhost", 97898); ids.getNetMember().setPreferredForCoordinator(true); gmsJoinLeave.processMessage(reqMsg); - ArgumentCaptor<JoinResponseMessage> ac = new ArgumentCaptor<>(); + ArgumentCaptor<JoinResponseMessage> ac = ArgumentCaptor.forClass(JoinResponseMessage.class); verify(messenger).send(ac.capture()); assertTrue("Should have asked for becoming a coordinator", ac.getValue().getBecomeCoordinator()); @@ -964,12 +958,75 @@ public class GMSJoinLeaveJUnitTest { msg.setSender(gmsJoinLeaveMemberId); gmsJoinLeave.processMessage(msg); } - Timeout to = new Timeout(2 * GMSJoinLeave.MEMBER_REQUEST_COLLECTION_INTERVAL, new Times(1)); + Timeout to = new Timeout(2 * ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL, new Times(1)); verify(messenger, to).send( isA(NetworkPartitionMessage.class)); }finally { System.getProperties().remove(GMSJoinLeave.BYPASS_DISCOVERY_PROPERTY); } } + + @Test + public void testViewIgnoredAfterShutdown() throws Exception { + try { + initMocks(true); + System.setProperty(GMSJoinLeave.BYPASS_DISCOVERY_PROPERTY, "true"); + gmsJoinLeave.join(); + installView(1, gmsJoinLeaveMemberId, createMemberList(mockMembers[0], mockMembers[1], mockMembers[2], gmsJoinLeaveMemberId, mockMembers[3])); + gmsJoinLeave.stop(); + for(int i = 1; i < 4; i++) { + RemoveMemberMessage msg = new RemoveMemberMessage(gmsJoinLeaveMemberId, mockMembers[i], "crashed"); + msg.setSender(gmsJoinLeaveMemberId); + gmsJoinLeave.processMessage(msg); + } + Timeout to = new Timeout(2 * ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL, never()); + verify(messenger, to).send( isA(NetworkPartitionMessage.class)); + + }finally { + System.getProperties().remove(GMSJoinLeave.BYPASS_DISCOVERY_PROPERTY); + } + } + + @Test + public void testPreparedViewFoundDuringBecomeCoordinator() throws Exception { + initMocks(false); + prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId)); + + // a new member is joining + NetView preparedView = new NetView(gmsJoinLeave.getView(), gmsJoinLeave.getView().getViewId()+5); + mockMembers[1].setVmViewId(preparedView.getViewId()); + preparedView.add(mockMembers[1]); + + InstallViewMessage msg = new InstallViewMessage(preparedView, null, true); + gmsJoinLeave.processMessage(msg); + + gmsJoinLeave.becomeCoordinatorForTest(); + + Thread.sleep(2000); + ViewCreator vc = gmsJoinLeave.getViewCreator(); + + ViewAckMessage vack = new ViewAckMessage(gmsJoinLeaveMemberId, gmsJoinLeave.getPreparedView().getViewId(), true); + vack.setSender(mockMembers[0]); + gmsJoinLeave.processMessage(vack); + vack = new ViewAckMessage(gmsJoinLeaveMemberId, gmsJoinLeave.getPreparedView().getViewId(), true); + vack.setSender(mockMembers[1]); + gmsJoinLeave.processMessage(vack); + vack = new ViewAckMessage(gmsJoinLeaveMemberId, gmsJoinLeave.getPreparedView().getViewId(), true); + vack.setSender(gmsJoinLeaveMemberId); + gmsJoinLeave.processMessage(vack); + + int tries = 0; + while (!vc.waiting) { + if (tries > 30) { + Assert.fail("view creator never finished"); + } + tries++; + Thread.sleep(1000); + } + NetView newView = gmsJoinLeave.getView(); + System.out.println("new view is " + newView); + assertTrue(newView.contains(mockMembers[1])); + assertTrue(newView.getViewId() > preparedView.getViewId()); + } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManagerJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManagerJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManagerJUnitTest.java index 2b59ca5..e133625 100755 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManagerJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/mgr/GMSMembershipManagerJUnitTest.java @@ -25,9 +25,11 @@ import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Properties; +import java.util.Random; import java.util.Set; import java.util.Timer; +import org.jgroups.util.UUID; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -35,16 +37,22 @@ import org.junit.experimental.categories.Category; import com.gemstone.gemfire.distributed.DistributedSystemDisconnectedException; import com.gemstone.gemfire.distributed.internal.AdminMessageType; +import com.gemstone.gemfire.distributed.internal.DM; +import com.gemstone.gemfire.distributed.internal.DMStats; import com.gemstone.gemfire.distributed.internal.DistributionConfig; import com.gemstone.gemfire.distributed.internal.DistributionConfigImpl; import com.gemstone.gemfire.distributed.internal.DistributionManager; import com.gemstone.gemfire.distributed.internal.DistributionMessage; import com.gemstone.gemfire.distributed.internal.HighPriorityAckedMessage; import com.gemstone.gemfire.distributed.internal.HighPriorityDistributionMessage; +import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; +import com.gemstone.gemfire.distributed.internal.MembershipListener; +import com.gemstone.gemfire.distributed.internal.ReplyProcessor21; import com.gemstone.gemfire.distributed.internal.direct.DirectChannel; import com.gemstone.gemfire.distributed.internal.membership.DistributedMembershipListener; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.distributed.internal.membership.NetView; +import com.gemstone.gemfire.distributed.internal.membership.gms.GMSMember; import com.gemstone.gemfire.distributed.internal.membership.gms.ServiceConfig; import com.gemstone.gemfire.distributed.internal.membership.gms.Services; import com.gemstone.gemfire.distributed.internal.membership.gms.Services.Stopper; @@ -72,6 +80,7 @@ public class GMSMembershipManagerJUnitTest { private Services services; private ServiceConfig mockConfig; private DistributionConfig distConfig; + private Properties distProperties; private Authenticator authenticator; private HealthMonitor healthMonitor; private InternalDistributedMember myMemberId; @@ -88,6 +97,8 @@ public class GMSMembershipManagerJUnitTest { @Before public void initMocks() throws Exception { Properties nonDefault = new Properties(); + nonDefault.put(DistributionConfig.ACK_WAIT_THRESHOLD_NAME, "1"); + nonDefault.put(DistributionConfig.ACK_SEVERE_ALERT_THRESHOLD_NAME, "10"); nonDefault.put(DistributionConfig.DISABLE_TCP_NAME, "true"); nonDefault.put(DistributionConfig.MCAST_PORT_NAME, "0"); nonDefault.put(DistributionConfig.MCAST_TTL_NAME, "0"); @@ -96,6 +107,7 @@ public class GMSMembershipManagerJUnitTest { nonDefault.put(DistributionConfig.MEMBER_TIMEOUT_NAME, "2000"); nonDefault.put(DistributionConfig.LOCATORS_NAME, "localhost[10344]"); distConfig = new DistributionConfigImpl(nonDefault); + distProperties = nonDefault; RemoteTransportConfig tconfig = new RemoteTransportConfig(distConfig, DistributionManager.NORMAL_DM_TYPE); @@ -129,9 +141,13 @@ public class GMSMembershipManagerJUnitTest { Timer t = new Timer(true); when(services.getTimer()).thenReturn(t); + Random r = new Random(); mockMembers = new InternalDistributedMember[5]; for (int i = 0; i < mockMembers.length; i++) { mockMembers[i] = new InternalDistributedMember("localhost", 8888 + i); + GMSMember m = (GMSMember)mockMembers[i].getNetMember(); + UUID uuid = new UUID(r.nextLong(), r.nextLong()); + m.setUUID(uuid); } members = new ArrayList<>(Arrays.asList(mockMembers)); @@ -366,5 +382,63 @@ public class GMSMembershipManagerJUnitTest { verify(dc).send(isA(GMSMembershipManager.class), isA(mockMembers.getClass()), isA(DistributionMessage.class), anyInt(), anyInt()); } + /** + * This test ensures that the membership manager can accept an ID that + * does not have a UUID and replace it with one that does have a UUID + * from the current membership view. + */ + @Test + public void testAddressesWithoutUUIDs() throws Exception { + manager.start(); + manager.started(); + manager.isJoining = true; + + List<InternalDistributedMember> viewmembers = Arrays.asList(new InternalDistributedMember[] {mockMembers[0], mockMembers[1], myMemberId}); + manager.installView(new NetView(myMemberId, 2, viewmembers, emptyMembersSet, emptyMembersSet)); + + InternalDistributedMember[] destinations = new InternalDistributedMember[viewmembers.size()]; + for (int i=0; i<destinations.length; i++) { + InternalDistributedMember id = viewmembers.get(i); + destinations[i] = new InternalDistributedMember(id.getHost(), id.getPort()); + } + manager.checkAddressesForUUIDs(destinations); + // each destination w/o a UUID should have been replaced with the corresponding + // ID from the membership view + for (int i=0; i<destinations.length; i++) { + assertTrue(viewmembers.get(i) == destinations[i]); + } + } + + @Test + public void testReplyProcessorInitiatesSuspicion() throws Exception { + DM dm = mock(DM.class); + DMStats stats = mock(DMStats.class); + + InternalDistributedSystem system = InternalDistributedSystem.newInstanceForTesting(dm, distProperties); + + when(dm.getStats()).thenReturn(stats); + when(dm.getSystem()).thenReturn(system); + when(dm.getCancelCriterion()).thenReturn(stopper); + when(dm.getMembershipManager()).thenReturn(manager); + when(dm.getViewMembers()).thenReturn(members); + when(dm.getDistributionManagerIds()).thenReturn(new HashSet(members)); + when(dm.addMembershipListenerAndGetDistributionManagerIds(any(MembershipListener.class))).thenReturn(new HashSet(members)); + + manager.start(); + manager.started(); + manager.isJoining = true; + + List<InternalDistributedMember> viewmembers = Arrays.asList(new InternalDistributedMember[] {mockMembers[0], mockMembers[1], myMemberId}); + manager.installView(new NetView(myMemberId, 2, viewmembers, emptyMembersSet, emptyMembersSet)); + + List<InternalDistributedMember> mbrs = new ArrayList<>(1); + mbrs.add(mockMembers[0]); + ReplyProcessor21 rp = new ReplyProcessor21(dm, mbrs); + rp.enableSevereAlertProcessing(); + boolean result = rp.waitForReplies(2000); + assertFalse(result); // the wait should have timed out + verify(healthMonitor, atLeastOnce()).checkIfAvailable(isA(InternalDistributedMember.class), isA(String.class), isA(Boolean.class)); + } + } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a5906e5b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/DataSerializableJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/DataSerializableJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/DataSerializableJUnitTest.java index 91a3411..5f69393 100644 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/DataSerializableJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/DataSerializableJUnitTest.java @@ -3545,21 +3545,26 @@ public class DataSerializableJUnitTest extends TestCase } public void testObjectEnum() throws Exception { - System.getProperties().setProperty("DataSerializer.DEBUG", "true"); - DAY_OF_WEEK e = DAY_OF_WEEK.SUN; - MONTH m = MONTH.FEB; - DataOutputStream out = getDataOutput(); - DataSerializer.writeObject(e, out); - DataSerializer.writeObject(m, out); - out.flush(); - - DataInput in = getDataInput(); - DAY_OF_WEEK e2 = (DAY_OF_WEEK)DataSerializer.readObject(in); - MONTH m2 = (MONTH)DataSerializer.readObject(in); - assertEquals(e, e2); - assertEquals(m, m2); - // Make sure there's nothing left in the stream - assertEquals(0, in.skipBytes(1)); + final String propName = "DataSerializer.DEBUG"; + System.setProperty(propName, "true"); + try { + DAY_OF_WEEK e = DAY_OF_WEEK.SUN; + MONTH m = MONTH.FEB; + DataOutputStream out = getDataOutput(); + DataSerializer.writeObject(e, out); + DataSerializer.writeObject(m, out); + out.flush(); + + DataInput in = getDataInput(); + DAY_OF_WEEK e2 = (DAY_OF_WEEK)DataSerializer.readObject(in); + MONTH m2 = (MONTH)DataSerializer.readObject(in); + assertEquals(e, e2); + assertEquals(m, m2); + // Make sure there's nothing left in the stream + assertEquals(0, in.skipBytes(1)); + } finally { + System.getProperties().remove(propName); + } } /**
