Persisting management plane ID test coverage
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/178588f3 Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/178588f3 Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/178588f3 Branch: refs/heads/master Commit: 178588f387c4c3528299e7484221ffc2bb1d995e Parents: e799ca9 Author: Svetoslav Neykov <svetoslav.ney...@cloudsoftcorp.com> Authored: Mon Apr 3 10:32:54 2017 +0300 Committer: Svetoslav Neykov <svetoslav.ney...@cloudsoftcorp.com> Committed: Tue Apr 11 17:58:59 2017 +0300 ---------------------------------------------------------------------- .../mgmt/ha/HighAvailabilityManagerImpl.java | 6 +- .../rebind/PeriodicDeltaChangeListener.java | 5 + .../core/mgmt/rebind/ManagementPlaneIdTest.java | 248 +++++++++++++++++++ .../core/mgmt/rebind/RebindTestUtils.java | 41 ++- .../transformer/CompoundTransformerTest.java | 1 + 5 files changed, 290 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/178588f3/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/HighAvailabilityManagerImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/HighAvailabilityManagerImpl.java index f968c96..db65c9b 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/HighAvailabilityManagerImpl.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/HighAvailabilityManagerImpl.java @@ -284,7 +284,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager { boolean weAreRecognisedAsMaster = existingMaster!=null && ownNodeId.equals(existingMaster.getNodeId()); boolean weAreMasterLocally = getInternalNodeState()==ManagementNodeState.MASTER; - updatePlaneId(planeRec); + updateLocalPlaneId(planeRec); // catch error in some tests where mgmt context has a different HA manager if (managementContext.getHighAvailabilityManager()!=this) @@ -460,7 +460,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager { registerPollTask(); } - protected void updatePlaneId(ManagementPlaneSyncRecord existingMaster) { + protected void updateLocalPlaneId(ManagementPlaneSyncRecord existingMaster) { if (existingMaster.getPlaneId() != null) { ((LocalManagementContext)managementContext).setManagementPlaneId(existingMaster.getPlaneId()); } @@ -722,7 +722,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager { return; } - updatePlaneId(memento); + updateLocalPlaneId(memento); String currMasterNodeId = memento.getMasterNodeId(); ManagementNodeSyncRecord currMasterNodeRecord = memento.getManagementNodes().get(currMasterNodeId); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/178588f3/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/PeriodicDeltaChangeListener.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/PeriodicDeltaChangeListener.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/PeriodicDeltaChangeListener.java index 68e47f7..e5d6a40 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/PeriodicDeltaChangeListener.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/rebind/PeriodicDeltaChangeListener.java @@ -401,6 +401,11 @@ public class PeriodicDeltaChangeListener implements ChangeListener { if (!alreadyHasMutex) persistingMutex.acquire(); if (!isActive() && state != ListenerState.STOPPING) return; + // Writes to the datastore are lossy. We'll just log failures and move on. + // (Most) entities will get updated multiple times in their lifecycle + // so not a huge deal. planeId does not get updated so if the first + // write fails it's not available to the HA cluster at all. That's why it + // gets periodically written to the datastore. updatePlaneIdIfTimedOut(); // Atomically switch the delta, so subsequent modifications will be done in the http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/178588f3/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/ManagementPlaneIdTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/ManagementPlaneIdTest.java b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/ManagementPlaneIdTest.java new file mode 100644 index 0000000..56ff7c0 --- /dev/null +++ b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/ManagementPlaneIdTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.mgmt.rebind; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.Callable; + +import org.apache.brooklyn.api.mgmt.ManagementContext; +import org.apache.brooklyn.api.mgmt.ha.HighAvailabilityMode; +import org.apache.brooklyn.api.mgmt.ha.ManagementNodeState; +import org.apache.brooklyn.core.entity.Entities; +import org.apache.brooklyn.core.internal.BrooklynProperties; +import org.apache.brooklyn.core.mgmt.internal.LocalManagementContext; +import org.apache.brooklyn.core.mgmt.persist.BrooklynMementoPersisterToObjectStore; +import org.apache.brooklyn.core.mgmt.persist.FileBasedObjectStore; +import org.apache.brooklyn.core.mgmt.persist.PersistMode; +import org.apache.brooklyn.core.server.BrooklynServerConfig; +import org.apache.brooklyn.core.server.BrooklynServerPaths; +import org.apache.brooklyn.test.Asserts; +import org.apache.brooklyn.util.os.Os; +import org.apache.brooklyn.util.text.Strings; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class ManagementPlaneIdTest { + private File mementoDir; + + protected ClassLoader classLoader = getClass().getClassLoader(); + + private Collection<ManagementContext> managementContextForTermination; + + @BeforeMethod + public void setUp() { + mementoDir = Os.newTempDir(getClass()); + managementContextForTermination = new ArrayList<>(); + } + + @AfterMethod(alwaysRun=true) + public void tearDown() throws Exception { + if (managementContextForTermination != null) { + for (ManagementContext mgmt : managementContextForTermination) { + Entities.destroyAll(mgmt); + } + } + if (mementoDir != null) FileBasedObjectStore.deleteCompletely(mementoDir); + } + + @Test + public void testUninitializedThrows() { + ManagementContext mgmt = new LocalManagementContext(BrooklynProperties.Factory.newEmpty()); + try { + mgmt.getManagementPlaneId(); + Asserts.shouldHaveFailedPreviously("managementPlaneId not initialized"); + } catch (NullPointerException e) { + Asserts.expectedFailureContains(e, "not initialized"); + } + } + + @Test + public void testPlaneIdPersists() throws Exception { + final ManagementContext mgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.DISABLED); + checkPlaneIdPersisted(mgmt); + } + + @Test + public void testPlaneIdRolledBack() throws Exception { + final LocalManagementContext mgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.AUTO); + + checkPlaneIdPersisted(mgmt); + final String oldPlaneId = mgmt.getOptionalManagementPlaneId().get(); + mgmt.setManagementPlaneId(Strings.makeRandomId(8)); + assertNotEquals(oldPlaneId, mgmt.getOptionalManagementPlaneId().get()); + Asserts.succeedsEventually(new Runnable() { + @Override + public void run() { + assertEquals(oldPlaneId, mgmt.getOptionalManagementPlaneId().get()); + } + }); + } + + @Test + public void testColdRebindInitialisesPlaneId() throws Exception { + final LocalManagementContext origMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.DISABLED); + checkPlaneIdPersisted(origMgmt); + Entities.destroyAll(origMgmt); + + LocalManagementContext rebindMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.DISABLED); + + assertEquals(origMgmt.getManagementPlaneId(), rebindMgmt.getManagementPlaneId()); + } + + + @DataProvider + public Object[][] haSlaveModes() { + return new Object[][] { + {HighAvailabilityMode.AUTO}, + {HighAvailabilityMode.STANDBY}, + {HighAvailabilityMode.HOT_STANDBY}, + {HighAvailabilityMode.HOT_BACKUP}, + }; + } + + @Test(dataProvider="haSlaveModes") + public void testHaRebindInitialisesPlaneId(HighAvailabilityMode slaveMode) throws Exception { + final LocalManagementContext origMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.AUTO); + final LocalManagementContext rebindMgmt = createManagementContext(PersistMode.AUTO, slaveMode); + + Asserts.succeedsEventually(new Runnable() { + @Override + public void run() { + assertEquals(origMgmt.getManagementPlaneId(), rebindMgmt.getManagementPlaneId()); + } + }); + } + + @Test + public void testHaFailoverKeepsPlaneId() throws Exception { + final LocalManagementContext origMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.MASTER); + final LocalManagementContext rebindMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.STANDBY); + + Asserts.succeedsEventually(new Runnable() { + @Override + public void run() { + assertEquals(origMgmt.getManagementPlaneId(), rebindMgmt.getManagementPlaneId()); + } + }); + + Entities.destroyAll(origMgmt); + + Asserts.succeedsEventually(new Runnable() { + @Override + public void run() { + assertEquals(rebindMgmt.getHighAvailabilityManager().getNodeState(), ManagementNodeState.MASTER); + if (rebindMgmt.getRebindManager().isAwaitingInitialRebind()) { + throw new IllegalStateException("still rebinding"); + } + } + }); + + assertEquals(origMgmt.getManagementPlaneId(), rebindMgmt.getManagementPlaneId()); + } + + @Test + public void testPlaneIdBackedUp() throws Exception { + final LocalManagementContext origMgmt = createManagementContext(PersistMode.AUTO, HighAvailabilityMode.AUTO); + checkPlaneIdPersisted(origMgmt); + Entities.destroyAll(origMgmt); + + LocalManagementContext rebindMgmt = createManagementContextWithBackups(PersistMode.AUTO, HighAvailabilityMode.AUTO); + + assertEquals(origMgmt.getManagementPlaneId(), rebindMgmt.getManagementPlaneId()); + + String backupContainer = BrooklynServerPaths.newBackupPersistencePathResolver(rebindMgmt).resolve(); + + File[] promotionFolders = new File(backupContainer).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.contains("promotion"); + } + }); + + assertEquals(promotionFolders.length, 1); + + File planeIdFile = new File(promotionFolders[0], BrooklynMementoPersisterToObjectStore.PLANE_ID_FILE_NAME); + String planeId = readFile(planeIdFile); + assertEquals(origMgmt.getManagementPlaneId(), planeId); + } + + @Test + public void testFullPersist() throws Exception { + final LocalManagementContext origMgmt = createManagementContext(PersistMode.DISABLED, HighAvailabilityMode.DISABLED); + origMgmt.getRebindManager().getPersister().enableWriteAccess(); + origMgmt.getRebindManager().forcePersistNow(true, null); + checkPlaneIdPersisted(origMgmt); + } + + protected LocalManagementContext createManagementContext(PersistMode persistMode, HighAvailabilityMode haMode) { + return createManagementContext(persistMode, haMode, false); + } + + protected LocalManagementContext createManagementContextWithBackups(PersistMode persistMode, HighAvailabilityMode haMode) { + return createManagementContext(persistMode, haMode, true); + } + + protected LocalManagementContext createManagementContext(PersistMode persistMode, HighAvailabilityMode haMode, boolean backedUp) { + BrooklynProperties props = BrooklynProperties.Factory.newEmpty(); + props.put(BrooklynServerConfig.PERSISTENCE_BACKUPS_DIR, mementoDir); + LocalManagementContext mgmt = RebindTestUtils.managementContextBuilder(mementoDir, classLoader) + .persistPeriodMillis(1) + .persistMode(persistMode) + .haMode(haMode) + .enablePersistenceBackups(backedUp) + .emptyCatalog(true) + .properties(props) + .enableOsgi(false) + .buildStarted(); + markForTermination(mgmt); + return mgmt; + } + + private void markForTermination(ManagementContext mgmt) { + managementContextForTermination.add(mgmt); + } + + protected static String readFile(File planeIdFile) throws IOException { + return new String(Files.readAllBytes(planeIdFile.toPath()), StandardCharsets.UTF_8); + } + + protected void checkPlaneIdPersisted(final ManagementContext mgmt) { + final File planeIdFile = new File(mementoDir, BrooklynMementoPersisterToObjectStore.PLANE_ID_FILE_NAME); + Asserts.succeedsEventually(new Callable<Void>() { + @Override + public Void call() throws Exception { + String planeId = readFile(planeIdFile); + assertEquals(mgmt.getManagementPlaneId(), planeId); + return null; + } + }); + } + +} http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/178588f3/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/RebindTestUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/RebindTestUtils.java b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/RebindTestUtils.java index 3f8e330..29ea68b 100644 --- a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/RebindTestUtils.java +++ b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/RebindTestUtils.java @@ -55,7 +55,6 @@ import org.apache.brooklyn.core.test.entity.LocalManagementContextForTests; import org.apache.brooklyn.util.io.FileUtil; import org.apache.brooklyn.util.javalang.Serializers; import org.apache.brooklyn.util.javalang.Serializers.ObjectReplacer; -import org.apache.brooklyn.util.text.Identifiers; import org.apache.brooklyn.util.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -151,7 +150,8 @@ public class RebindTestUtils { BrooklynProperties properties; PersistenceObjectStore objectStore; Duration persistPeriod = Duration.millis(100); - HighAvailabilityMode haMode; + PersistMode persistMode = PersistMode.AUTO; + HighAvailabilityMode haMode = HighAvailabilityMode.DISABLED; boolean forLive; boolean enableOsgi = false; boolean emptyCatalog; @@ -207,11 +207,25 @@ public class RebindTestUtils { return this; } + public ManagementContextBuilder persistMode(PersistMode val) { + checkNotNull(val, "persistMode"); + this.persistMode = val; + if (persistMode == PersistMode.DISABLED) { + haMode(HighAvailabilityMode.DISABLED); + } + return this; + } + public ManagementContextBuilder haMode(HighAvailabilityMode val) { + checkNotNull(val, "haMode"); this.haMode = val; return this; } + /** + * What you could actually want is {@link #buildStarted()} with builder properties set to + * {@code .persistMode(PersistMode.DISABLED).haMode(HighAvailabilityMode.DISABLED)} + */ public LocalManagementContext buildUnstarted() { LocalManagementContext unstarted; BrooklynProperties properties = this.properties != null @@ -227,20 +241,22 @@ public class RebindTestUtils { } if (forLive) { unstarted = new LocalManagementContext(properties); - unstarted.generateManagementPlaneId(); } else { - unstarted = LocalManagementContextForTests.builder(true).useProperties(properties).disableOsgi(!enableOsgi).build(); + unstarted = LocalManagementContextForTests.builder(true) + .useProperties(properties) + .disableOsgi(!enableOsgi) + .disablePersistenceBackups(!enablePersistenceBackups) + .build(); } objectStore.injectManagementContext(unstarted); - objectStore.prepareForSharedUse(PersistMode.AUTO, (haMode == null ? HighAvailabilityMode.DISABLED : haMode)); + objectStore.prepareForSharedUse(PersistMode.AUTO, haMode); BrooklynMementoPersisterToObjectStore newPersister = new BrooklynMementoPersisterToObjectStore( objectStore, unstarted.getBrooklynProperties(), classLoader); ((RebindManagerImpl) unstarted.getRebindManager()).setPeriodicPersistPeriod(persistPeriod); unstarted.getRebindManager().setPersister(newPersister, PersistenceExceptionHandlerImpl.builder().build()); - unstarted.getHighAvailabilityManager().disabled(); // set the HA persister, in case any children want to use HA unstarted.getHighAvailabilityManager().setPersister(new ManagementPlaneSyncRecordPersisterToObjectStore(unstarted, objectStore, classLoader)); return unstarted; @@ -248,8 +264,17 @@ public class RebindTestUtils { public LocalManagementContext buildStarted() { LocalManagementContext unstarted = buildUnstarted(); - unstarted.getHighAvailabilityManager().disabled(); - unstarted.getRebindManager().startPersistence(); + // Follows BasicLauncher logic for initialising persistence. + // TODO It should really be encapsulated in a common entry point + if (persistMode == PersistMode.DISABLED) { + unstarted.generateManagementPlaneId(); + } else if (haMode == HighAvailabilityMode.DISABLED) { + unstarted.getRebindManager().rebind(classLoader, null, ManagementNodeState.MASTER); + unstarted.getRebindManager().startPersistence(); + unstarted.getHighAvailabilityManager().disabled(); + } else { + unstarted.getHighAvailabilityManager().start(haMode); + } return unstarted; } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/178588f3/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java index 7187826..b742444 100644 --- a/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java +++ b/core/src/test/java/org/apache/brooklyn/core/mgmt/rebind/transformer/CompoundTransformerTest.java @@ -169,6 +169,7 @@ public class CompoundTransformerTest extends RebindTestFixtureWithApp { // Assert has expected config/fields assertEquals(newApp.getId(), origApp.getId()); + assertEquals(origManagementContext.getManagementPlaneId(), newManagementContext.getManagementPlaneId()); } @Test
