GEODE-3052 Restarting locators causes potential locator split brain When restarting from a locatorView.dat file we should ignore any locator entries in the view. Recovery tries to get this state from other locators before resorting to using the persisted view so there we know all of the locator entries in the view are invalid. This allows the locators to quickly move into the concurrent-startup algorithm and find each other.
I removed the Flaky categorization of the test that I modified to reproduce the problem. A subclass's use of the test was reported as a Flaky failure but I found that the ticket was closed. Project: http://git-wip-us.apache.org/repos/asf/geode/repo Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/bad2dcdb Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/bad2dcdb Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/bad2dcdb Branch: refs/heads/feature/GEODE-3023 Commit: bad2dcdb1fa0638f958bd382852664be86138dfd Parents: e07591c Author: Bruce Schuchardt <[email protected]> Authored: Thu Jun 8 17:01:41 2017 -0700 Committer: Udo Kohlmeyer <[email protected]> Committed: Fri Jun 9 13:12:04 2017 -0700 ---------------------------------------------------------------------- .../internal/membership/gms/locator/GMSLocator.java | 9 +++++++++ .../java/org/apache/geode/distributed/LocatorDUnitTest.java | 4 +++- .../geode/distributed/LocatorUDPSecurityDUnitTest.java | 6 ------ 3 files changed, 12 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/geode/blob/bad2dcdb/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/locator/GMSLocator.java ---------------------------------------------------------------------- diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/locator/GMSLocator.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/locator/GMSLocator.java index e3635f2..93fa9da 100644 --- a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/locator/GMSLocator.java +++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/locator/GMSLocator.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.geode.InternalGemFireException; import org.apache.geode.distributed.internal.ClusterConfigurationService; +import org.apache.geode.distributed.internal.DistributionManager; import org.apache.logging.log4j.Logger; import org.apache.geode.DataSerializer; @@ -405,6 +406,14 @@ public class GMSLocator implements Locator, NetLocator { Object o = DataSerializer.readObject(ois2); this.view = (NetView) o; + List<InternalDistributedMember> members = new ArrayList<>(view.getMembers()); + // GEODE-3052 - remove locators from the view. Since we couldn't recover from an existing + // locator we know that all of the locators in the view are defunct + for (InternalDistributedMember member : members) { + if (member.getVmKind() == DistributionManager.LOCATOR_DM_TYPE) { + view.remove(member); + } + } logger.info("Peer locator initial membership is " + view); return true; http://git-wip-us.apache.org/repos/asf/geode/blob/bad2dcdb/geode-core/src/test/java/org/apache/geode/distributed/LocatorDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/distributed/LocatorDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/LocatorDUnitTest.java index 8ff9b67..c62a545 100644 --- a/geode-core/src/test/java/org/apache/geode/distributed/LocatorDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/distributed/LocatorDUnitTest.java @@ -278,7 +278,6 @@ public class LocatorDUnitTest extends JUnit4DistributedTestCase { * stagger the starting of locators. This test configures two locators to start up simultaneously * and shows that they find each other and form a single system. */ - @Category(FlakyTest.class) // GEODE-1931 @Test public void testStartTwoLocators() throws Exception { disconnectAllFromDS(); @@ -305,6 +304,9 @@ public class LocatorDUnitTest extends JUnit4DistributedTestCase { properties.put(ENABLE_CLUSTER_CONFIGURATION, "false"); addDSProps(properties); startVerifyAndStopLocator(loc1, loc2, port1, port2, properties); + // GEODE-3052 - split brain on restart from persistent view data + startVerifyAndStopLocator(loc1, loc2, port1, port2, properties); + startVerifyAndStopLocator(loc1, loc2, port1, port2, properties); } private Boolean startLocatorWithPortAndProperties(final int port, final Properties properties) http://git-wip-us.apache.org/repos/asf/geode/blob/bad2dcdb/geode-core/src/test/java/org/apache/geode/distributed/LocatorUDPSecurityDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-core/src/test/java/org/apache/geode/distributed/LocatorUDPSecurityDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/LocatorUDPSecurityDUnitTest.java index 9d49d30..df1d8d1 100644 --- a/geode-core/src/test/java/org/apache/geode/distributed/LocatorUDPSecurityDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/distributed/LocatorUDPSecurityDUnitTest.java @@ -45,12 +45,6 @@ public class LocatorUDPSecurityDUnitTest extends LocatorDUnitTest { p.setProperty(SECURITY_UDP_DHALGO, "AES:128"); } - @Override - @Test - public void testStartTwoLocators() throws Exception { - super.testStartTwoLocators(); - } - @Test public void testLocatorWithUDPSecurityButServer() throws Exception { disconnectAllFromDS();
