Repository: brooklyn-server Updated Branches: refs/heads/master d2fd128ca -> e79e0bb2e
BROOKLYN-299: fix LocationUsage mutex Previously, we sometimes tried to ssh to a VM when it was destroyed to get its metadata. Avoid that! Also calls toMetadataRecord() outside of the mutex. Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/dc0fd3c1 Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/dc0fd3c1 Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/dc0fd3c1 Branch: refs/heads/master Commit: dc0fd3c1a862a2a8e65cdc5f0d082716954b5c52 Parents: d2fd128 Author: Aled Sage <[email protected]> Authored: Tue Jun 14 14:22:45 2016 +0100 Committer: Aled Sage <[email protected]> Committed: Wed Jun 15 10:16:15 2016 +0100 ---------------------------------------------------------------------- .../core/mgmt/internal/LocalUsageManager.java | 93 +++-- .../brooklyn/core/mgmt/usage/LocationUsage.java | 7 + .../location/ssh/SshMachineLocation.java | 7 +- .../core/internal/ssh/RecordingSshTool.java | 25 +- .../jclouds/JcloudsSshMachineLocation.java | 6 +- .../usage/JcloudsLocationUsageTrackingTest.java | 356 +++++++++++++++++++ .../mgmt/usage/LocationUsageTrackingTest.java | 52 +++ 7 files changed, 500 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/LocalUsageManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/LocalUsageManager.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/LocalUsageManager.java index 363009e..7caf958 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/LocalUsageManager.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/LocalUsageManager.java @@ -48,12 +48,12 @@ import org.apache.brooklyn.core.mgmt.usage.ApplicationUsage; import org.apache.brooklyn.core.mgmt.usage.LocationUsage; import org.apache.brooklyn.core.mgmt.usage.UsageListener; import org.apache.brooklyn.core.mgmt.usage.UsageManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.brooklyn.util.core.flags.TypeCoercions; import org.apache.brooklyn.util.exceptions.Exceptions; import org.apache.brooklyn.util.javalang.Reflections; import org.apache.brooklyn.util.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; @@ -243,12 +243,25 @@ public class LocalUsageManager implements UsageManager { public void recordApplicationEvent(final Application app, final Lifecycle state) { log.debug("Storing application lifecycle usage event: application {} in state {}", new Object[] {app, state}); ConcurrentMap<String, ApplicationUsage> eventMap = managementContext.getStorage().getMap(APPLICATION_USAGE_KEY); + + // Don't call out to alien-code (i.e. app.toMetadataRecord()) while holding mutex. It might take a while. + // If we don't have a usage record, then generate one outside of the mutex. But then double-check while + // holding the mutex to see if another thread has created one. If it has, stick with that rather than + // overwriting it. + ApplicationUsage usage; + synchronized (mutex) { + usage = eventMap.get(app.getId()); + } + if (usage == null) { + usage = new ApplicationUsage(app.getId(), app.getDisplayName(), app.getEntityType().getName(), ((EntityInternal)app).toMetadataRecord()); + } + final ApplicationUsage.ApplicationEvent event = new ApplicationUsage.ApplicationEvent(state, getUser()); + synchronized (mutex) { - ApplicationUsage usage = eventMap.get(app.getId()); - if (usage == null) { - usage = new ApplicationUsage(app.getId(), app.getDisplayName(), app.getEntityType().getName(), ((EntityInternal)app).toMetadataRecord()); + ApplicationUsage otherUsage = eventMap.get(app.getId()); + if (otherUsage != null) { + usage = otherUsage; } - final ApplicationUsage.ApplicationEvent event = new ApplicationUsage.ApplicationEvent(state, getUser()); usage.addEvent(event); eventMap.put(app.getId(), usage); @@ -297,37 +310,55 @@ public class LocalUsageManager implements UsageManager { Object callerContext = loc.getConfig(LocationConfigKeys.CALLER_CONTEXT); if (callerContext != null && callerContext instanceof Entity) { - log.debug("Storing location lifecycle usage event: location {} in state {}; caller context {}", new Object[] {loc, state, callerContext}); - Entity caller = (Entity) callerContext; - String entityTypeName = caller.getEntityType().getName(); - String appId = caller.getApplicationId(); - - final LocationUsage.LocationEvent event = new LocationUsage.LocationEvent(state, caller.getId(), entityTypeName, appId, getUser()); - - ConcurrentMap<String, LocationUsage> usageMap = managementContext.getStorage().<String, LocationUsage>getMap(LOCATION_USAGE_KEY); - synchronized (mutex) { - LocationUsage usage = usageMap.get(loc.getId()); - if (usage == null) { - usage = new LocationUsage(loc.getId(), ((LocationInternal)loc).toMetadataRecord()); - } - usage.addEvent(event); - usageMap.put(loc.getId(), usage); - - execOnListeners(new Function<UsageListener, Void>() { - public Void apply(UsageListener listener) { - listener.onLocationEvent(new LocationMetadataImpl(loc), event); - return null; - } - public String toString() { - return "locationEvent("+loc+", "+state+")"; - }}); - } + recordLocationEvent(loc, caller, state); } else { // normal for high-level locations log.trace("Not recording location lifecycle usage event for {} in state {}, because no caller context", new Object[] {loc, state}); } } + + protected void recordLocationEvent(final Location loc, final Entity caller, final Lifecycle state) { + log.debug("Storing location lifecycle usage event: location {} in state {}; caller context {}", new Object[] {loc, state, caller}); + ConcurrentMap<String, LocationUsage> eventMap = managementContext.getStorage().<String, LocationUsage>getMap(LOCATION_USAGE_KEY); + + String entityTypeName = caller.getEntityType().getName(); + String appId = caller.getApplicationId(); + + final LocationUsage.LocationEvent event = new LocationUsage.LocationEvent(state, caller.getId(), entityTypeName, appId, getUser()); + + + // Don't call out to alien-code (i.e. loc.toMetadataRecord()) while holding mutex. It might take a while, + // e.g. ssh'ing to the machine! + // If we don't have a usage record, then generate one outside of the mutex. But then double-check while + // holding the mutex to see if another thread has created one. If it has, stick with that rather than + // overwriting it. + LocationUsage usage; + synchronized (mutex) { + usage = eventMap.get(loc.getId()); + } + if (usage == null) { + usage = new LocationUsage(loc.getId(), ((LocationInternal)loc).toMetadataRecord()); + } + + synchronized (mutex) { + LocationUsage otherUsage = eventMap.get(loc.getId()); + if (otherUsage != null) { + usage = otherUsage; + } + usage.addEvent(event); + eventMap.put(loc.getId(), usage); + + execOnListeners(new Function<UsageListener, Void>() { + public Void apply(UsageListener listener) { + listener.onLocationEvent(new LocationMetadataImpl(loc), event); + return null; + } + public String toString() { + return "locationEvent("+loc+", "+state+")"; + }}); + } + } /** * Returns the usage info for the location with the given id, or null if unknown. http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/core/src/main/java/org/apache/brooklyn/core/mgmt/usage/LocationUsage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/usage/LocationUsage.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/usage/LocationUsage.java index 4196186..7d4013f 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/usage/LocationUsage.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/usage/LocationUsage.java @@ -132,4 +132,11 @@ public class LocationUsage { public void addEvent(LocationEvent event) { events.add(checkNotNull(event, "event")); } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("locationId", locationId) + .toString(); + } } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java b/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java index 7ca672b..3a91845 100644 --- a/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java +++ b/core/src/main/java/org/apache/brooklyn/location/ssh/SshMachineLocation.java @@ -1062,9 +1062,12 @@ public class SshMachineLocation extends AbstractLocation implements MachineLocat protected MachineDetails inferMachineDetails() { boolean detectionEnabled = getConfig(DETECT_MACHINE_DETAILS); - if (!detectionEnabled) + if (!detectionEnabled) { return new BasicMachineDetails(new BasicHardwareDetails(-1, -1), new BasicOsDetails("UNKNOWN", "UNKNOWN", "UNKNOWN")); - + } else if (!isManaged()) { + return new BasicMachineDetails(new BasicHardwareDetails(-1, -1), new BasicOsDetails("UNKNOWN", "UNKNOWN", "UNKNOWN")); + } + Tasks.setBlockingDetails("Waiting for machine details"); try { return BasicMachineDetails.forSshMachineLocationLive(this); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/core/src/test/java/org/apache/brooklyn/util/core/internal/ssh/RecordingSshTool.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/brooklyn/util/core/internal/ssh/RecordingSshTool.java b/core/src/test/java/org/apache/brooklyn/util/core/internal/ssh/RecordingSshTool.java index a2a6764..e064916 100644 --- a/core/src/test/java/org/apache/brooklyn/util/core/internal/ssh/RecordingSshTool.java +++ b/core/src/test/java/org/apache/brooklyn/util/core/internal/ssh/RecordingSshTool.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.brooklyn.util.exceptions.Exceptions; import org.apache.brooklyn.util.text.Strings; @@ -85,8 +86,8 @@ public class RecordingSshTool implements SshTool { customResponses.clear(); } - public static void setCustomResponse(String cmd, CustomResponse response) { - customResponses.put(cmd, checkNotNull(response, "response")); + public static void setCustomResponse(String cmdRegex, CustomResponse response) { + customResponses.put(cmdRegex, checkNotNull(response, "response")); } public static ExecCmd getLastExecCmd() { @@ -111,10 +112,12 @@ public class RecordingSshTool implements SshTool { @Override public int execScript(Map<String, ?> props, List<String> commands, Map<String, ?> env) { execScriptCmds.add(new ExecCmd(props, "", commands, env)); for (String cmd : commands) { - if (customResponses.containsKey(cmd)) { - CustomResponse response = customResponses.get(cmd); - writeCustomResponseStreams(props, response); - return response.exitCode; + for (Entry<String, CustomResponse> entry : customResponses.entrySet()) { + if (cmd.matches(entry.getKey())) { + CustomResponse response = entry.getValue(); + writeCustomResponseStreams(props, response); + return response.exitCode; + } } } return 0; @@ -125,10 +128,12 @@ public class RecordingSshTool implements SshTool { @Override public int execCommands(Map<String, ?> props, List<String> commands, Map<String, ?> env) { execScriptCmds.add(new ExecCmd(props, "", commands, env)); for (String cmd : commands) { - if (customResponses.containsKey(cmd)) { - CustomResponse response = customResponses.get(cmd); - writeCustomResponseStreams(props, response); - return response.exitCode; + for (Entry<String, CustomResponse> entry : customResponses.entrySet()) { + if (cmd.matches(entry.getKey())) { + CustomResponse response = entry.getValue(); + writeCustomResponseStreams(props, response); + return response.exitCode; + } } } return 0; http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java ---------------------------------------------------------------------- diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java index e626c74..1066602 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsSshMachineLocation.java @@ -548,11 +548,11 @@ public class JcloudsSshMachineLocation extends SshMachineLocation implements Jcl OsDetails osD = new BasicOsDetails(name.get(), architecture.get(), version.get()); HardwareDetails hwD = new BasicHardwareDetails(cpus.get(), ram.get()); return new BasicMachineDetails(hwD, osD); - } else if ("false".equalsIgnoreCase(getConfig(JcloudsLocation.WAIT_FOR_SSHABLE))) { + } else if (!isManaged() || "false".equalsIgnoreCase(getConfig(JcloudsLocation.WAIT_FOR_SSHABLE))) { if (LOG.isTraceEnabled()) { - LOG.trace("Machine details for {} missing from Jclouds, but skipping SSH test because waitForSshable=false. name={}, version={}, " + + LOG.trace("Machine details for {} missing from Jclouds, but skipping SSH test because {}. name={}, version={}, " + "arch={}, ram={}, #cpus={}", - new Object[]{this, name, version, architecture, ram, cpus}); + new Object[]{this, (isManaged() ? "waitForSshable=false" : "unmanaged"), name, version, architecture, ram, cpus}); } OsDetails osD = new BasicOsDetails(name.orNull(), architecture.orNull(), version.orNull()); HardwareDetails hwD = new BasicHardwareDetails(cpus.orNull(), ram.orNull()); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/JcloudsLocationUsageTrackingTest.java ---------------------------------------------------------------------- diff --git a/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/JcloudsLocationUsageTrackingTest.java b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/JcloudsLocationUsageTrackingTest.java new file mode 100644 index 0000000..22d88b9 --- /dev/null +++ b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/JcloudsLocationUsageTrackingTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.entity.software.base.test.core.mgmt.usage; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + +import org.apache.brooklyn.api.entity.Entity; +import org.apache.brooklyn.api.entity.EntitySpec; +import org.apache.brooklyn.core.entity.BrooklynConfigKeys; +import org.apache.brooklyn.core.entity.lifecycle.Lifecycle; +import org.apache.brooklyn.core.location.Machines; +import org.apache.brooklyn.core.mgmt.internal.LocalUsageManager; +import org.apache.brooklyn.core.mgmt.usage.LocationUsage; +import org.apache.brooklyn.core.mgmt.usage.LocationUsage.LocationEvent; +import org.apache.brooklyn.core.test.entity.TestApplication; +import org.apache.brooklyn.entity.software.base.SoftwareProcessEntityTest; +import org.apache.brooklyn.location.jclouds.AbstractJcloudsStubbedLiveTest; +import org.apache.brooklyn.location.jclouds.JcloudsLocation; +import org.apache.brooklyn.location.jclouds.JcloudsLocationConfig; +import org.apache.brooklyn.location.jclouds.JcloudsSshMachineLocation; +import org.apache.brooklyn.location.jclouds.StubbedComputeServiceRegistry.AbstractNodeCreator; +import org.apache.brooklyn.location.jclouds.StubbedComputeServiceRegistry.NodeCreator; +import org.apache.brooklyn.util.core.internal.ssh.RecordingSshTool; +import org.apache.brooklyn.util.core.internal.ssh.RecordingSshTool.ExecCmd; +import org.apache.brooklyn.util.javalang.Reflections; +import org.apache.brooklyn.util.net.Networking; +import org.apache.brooklyn.util.time.Time; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.NodeMetadata.Status; +import org.jclouds.compute.domain.NodeMetadataBuilder; +import org.jclouds.compute.domain.Template; +import org.jclouds.domain.LoginCredentials; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; + +/** + * These tests confirm that we (correctly!) retrieve the metadata for jclouds VMs. + * + * They are live tests: they talk directly to the cloud to get the image details etc, + * but they do not provision VMs (that part is stubbed out). + * + * Some tests expect it to try to ssh. This leads to some unusual configuration! We open up a + * server socket, listening on a random high-number port. We return a jclouds NodeMetadata that + * claims its public IP + login port is that of the server-socket (thus hopefully avoiding the + * risk of risk of accidentally executing ssh commands on the local machine!). We also configure + * the location to use the RecordingSshTool, which stubs out the ssh execution. + */ +public class JcloudsLocationUsageTrackingTest extends AbstractJcloudsStubbedLiveTest { + + private static final Logger LOG = LoggerFactory.getLogger(JcloudsLocationUsageTrackingTest.class); + + private TestApplication app; + private SoftwareProcessEntityTest.MyService entity; + + /** + * A socket that is used to simulate an ssh endpoint. The JcloudsLocation code just waits for + * the port to be reachable, before then switching to the SshTool for executing commands. We + * therefore need a real socket (rather than just relying on {@link RecordingSshTool}. + */ + private ServerSocket serverSocket; + + /** + * If true, then real hardware metadata is included in the jclouds node. If false, we leave + * the hardware as null. This causes {@link JcloudsSshMachineLocation} to try executing an + * ssh command to find out the OS, architecture, etc. + */ + protected boolean includeNodeHardwareMetadata; + + @BeforeMethod(alwaysRun = true) + @Override + public void setUp() throws Exception { + super.setUp(); + RecordingSshTool.clear(); + + app = managementContext.getEntityManager().createEntity(EntitySpec.create(TestApplication.class) + .configure(BrooklynConfigKeys.SKIP_ON_BOX_BASE_DIR_RESOLUTION, true)); + entity = app.createAndManageChild(EntitySpec.create(SoftwareProcessEntityTest.MyService.class)); + + serverSocket = new ServerSocket(); + serverSocket.bind(new InetSocketAddress(Networking.getLocalHost(), 0), 0); + } + + @AfterMethod(alwaysRun=true) + @Override + public void tearDown() throws Exception { + try { + super.tearDown(); + } finally { + if (serverSocket != null) serverSocket.close(); + } + } + + @Override + protected NodeCreator newNodeCreator() { + return new AbstractNodeCreator() { + @Override protected NodeMetadata newNode(String group, Template template) { + NodeMetadata result = new NodeMetadataBuilder() + .id("myNodeId") + .credentials(LoginCredentials.builder().identity("myuser").credential("mypassword").build()) + .loginPort(serverSocket.getLocalPort()) + .status(Status.RUNNING) + .publicAddresses(ImmutableList.of(serverSocket.getInetAddress().getHostAddress())) + .privateAddresses(ImmutableList.of("1.2.3.4")) + .imageId(template.getImage().getId()) + .tags(template.getOptions().getTags()) + .hardware(includeNodeHardwareMetadata ? template.getHardware() : null) + .group(template.getOptions().getGroups().isEmpty() ? "myGroup" : Iterables.get(template.getOptions().getGroups(), 0)) + .build(); + return result; + } + }; + } + + @Test(groups={"Live", "Live-sanity"}) + public void testLocationEventGetsMetadataFromCloudProvider() throws Exception { + includeNodeHardwareMetadata = true; + jcloudsLocation = (JcloudsLocation) managementContext.getLocationRegistry().getLocationManaged( + getLocationSpec(), + jcloudsLocationConfig(ImmutableMap.<Object, Object>builder() + .put(JcloudsLocationConfig.COMPUTE_SERVICE_REGISTRY, computeServiceRegistry) + .put("sshToolClass", RecordingSshTool.class.getName()) + .put(JcloudsLocation.WAIT_FOR_SSHABLE.getName(), false) + .put(JcloudsLocation.IMAGE_ID.getName(), "UBUNTU_14_64") + .put(JcloudsLocation.MIN_RAM.getName(), 1024) + .put(JcloudsLocation.MIN_CORES.getName(), 1) + .build())); + + + // Start the app; expect record of location in use (along with metadata) + app.start(ImmutableList.of(jcloudsLocation)); + JcloudsSshMachineLocation machine = Machines.findUniqueMachineLocation(entity.getLocations(), JcloudsSshMachineLocation.class).get(); + + // Expect usage information, including metadata about machine (obtained by ssh'ing) + Set<LocationUsage> usages = managementContext.getUsageManager().getLocationUsage(Predicates.alwaysTrue()); + LocationUsage usage = findLocationUsage(usages, machine.getId()); + LOG.info("metadata="+usage.getMetadata()); + assertMetadata(usage.getMetadata(), ImmutableMap.<String, String>builder() + .put("displayName", machine.getDisplayName()) + .put("parentDisplayName", jcloudsLocation.getDisplayName()) + .put("provider", jcloudsLocation.getProvider()) + .put("account", jcloudsLocation.getIdentity()) + .put("region", jcloudsLocation.getRegion()) + .put("serverId", "myNodeId") + .put("imageId", "UBUNTU_14_64") + .put("instanceTypeId", "cpu=1,memory=1024,disk=25,type=LOCAL") + .put("ram", "1024") + .put("cpus", "1") + .put("osName", "ubuntu") + .put("osArch", "x86_64") + .put("is64bit", "true") + .build()); + } + + @Test(groups={"Live", "Live-sanity"}) + public void testLocationEventMetadataObtainedOverSsh() throws Exception { + runLocationEventTrackingSshCalls(false); + } + + /** + * Previously (see BROOKLYN-299), after a rebind we've lost the usage records, so if a location is + * subsequently unmanaged we'd attempt to obtain the os-details again (by executing an ssh command). + * But the machine had been terminated, so that ssh command would eventually timeout (e.g. after + * two minutes). + */ + @Test(groups={"Live", "Live-sanity"}) + public void testLocationEventMetadataNotObtainedOverSshOnStop() throws Exception { + runLocationEventTrackingSshCalls(true); + } + + protected void runLocationEventTrackingSshCalls(boolean simulateRebind) throws Exception { + includeNodeHardwareMetadata = false; + + String osDetailsResponse = Joiner.on("\n").join( + "name:Acme OS", + "version:10.11.5", + "architecture:x86_64", + "ram:16384", + "cpus:8"); + RecordingSshTool.setCustomResponse( + ".*os-release.*", + new RecordingSshTool.CustomResponse(0, osDetailsResponse, "")); + + jcloudsLocation = (JcloudsLocation) managementContext.getLocationRegistry().getLocationManaged( + getLocationSpec(), + jcloudsLocationConfig(ImmutableMap.<Object, Object>builder() + .put(JcloudsLocationConfig.COMPUTE_SERVICE_REGISTRY, computeServiceRegistry) + .put("sshToolClass", RecordingSshTool.class.getName()) + .put(JcloudsLocation.WAIT_FOR_SSHABLE.getName(), "1m") + .put(JcloudsLocation.IMAGE_ID.getName(), "UBUNTU_14_64") + .build())); + + SoftwareProcessEntityTest.MyService entity = app.createAndManageChild(EntitySpec.create(SoftwareProcessEntityTest.MyService.class)); + + // Start the app; expect record of location in use (along with metadata) + long preStart = System.currentTimeMillis(); + app.start(ImmutableList.of(jcloudsLocation)); + long postStart = System.currentTimeMillis(); + JcloudsSshMachineLocation machine = Machines.findUniqueMachineLocation(entity.getLocations(), JcloudsSshMachineLocation.class).get(); + + // imageId=myImageId, instanceTypeId=cpu=1,memory=1024,disk=25,type=LOCAL, ram=1024, cpus=1, osName=ubuntu, osArch=x86_64, is64bit=true} expected [Acme OS] but found [ubuntu] + + // Expect usage information, including metadata about machine (obtained by ssh'ing) + Set<LocationUsage> usages1 = managementContext.getUsageManager().getLocationUsage(Predicates.alwaysTrue()); + LocationUsage usage1 = findLocationUsage(usages1, machine.getId()); + assertEquals(usage1.getLocationId(), machine.getId(), "usage="+usage1); + LOG.info("metadata="+usage1.getMetadata()); + assertMetadata(usage1.getMetadata(), ImmutableMap.<String, String>builder() + .put("displayName", machine.getDisplayName()) + .put("parentDisplayName", jcloudsLocation.getDisplayName()) + .put("provider", jcloudsLocation.getProvider()) + .put("account", jcloudsLocation.getIdentity()) + .put("region", jcloudsLocation.getRegion()) + .put("serverId", "myNodeId") + .put("imageId", "UBUNTU_14_64") + .put("osName", "Acme OS") + .put("osArch", "x86_64") + .put("is64bit", "true") + .build()); + LocationEvent event1 = usage1.getEvents().get(0); + assertLocationEvent(event1, entity, Lifecycle.CREATED, preStart, postStart); + + assertCmdContains(RecordingSshTool.execScriptCmds, "os-release"); + + // Clear the ssh-history, so we can assert again + RecordingSshTool.clear(); + + if (simulateRebind) { + managementContext.getStorage().getMap(LocalUsageManager.APPLICATION_USAGE_KEY).clear(); + managementContext.getStorage().getMap(LocalUsageManager.LOCATION_USAGE_KEY).clear(); + Field machineDetailsField = Reflections.findField(JcloudsSshMachineLocation.class, "machineDetails"); + machineDetailsField.setAccessible(true); + machineDetailsField.set(machine, null); + } + + // Stop the app; expect location-event for "destroyed". + // Expect *not* to have exec'ed ssh command again. + long preStop = System.currentTimeMillis(); + app.stop(); + long postStop = System.currentTimeMillis(); + + Set<LocationUsage> usages2 = managementContext.getUsageManager().getLocationUsage(Predicates.alwaysTrue()); + LocationUsage usage2 = findLocationUsage(usages2, machine.getId()); + LOG.info("metadata="+usage2.getMetadata()); + assertEquals(usage2.getLocationId(), machine.getId(), "usage="+usage2); + if (simulateRebind) { + assertMetadata(usage2.getMetadata(), ImmutableMap.<String, String>builder() + .put("displayName", machine.getDisplayName()) + .put("parentDisplayName", jcloudsLocation.getDisplayName()) + .put("provider", jcloudsLocation.getProvider()) + .put("account", jcloudsLocation.getIdentity()) + .put("region", jcloudsLocation.getRegion()) + .put("serverId", "myNodeId") + .put("imageId", "UBUNTU_14_64") + .build()); + } else { + assertMetadata(usage2.getMetadata(), usage2.getMetadata()); + } + LocationEvent event2 = usage2.getEvents().get(usage2.getEvents().size()-1); + assertLocationEvent(event2, app.getApplicationId(), entity.getId(), entity.getEntityType().getName(), Lifecycle.DESTROYED, preStop, postStop); + + assertCmdNotContains(RecordingSshTool.execScriptCmds, "os-release"); + } + + // Assets everything in "expected" is in the metadata; allows additional values in the metadata + private void assertMetadata(Map<String, String> metadata, Map<String, String> expected) { + String errMsg = "metadata="+metadata; + for (Map.Entry<String, String> entry : expected.entrySet()) { + assertEquals(metadata.get(entry.getKey()), entry.getValue(), errMsg); + } + } + + private LocationUsage findLocationUsage(Iterable<? extends LocationUsage> usages, String locationId) { + for (LocationUsage usage : usages) { + if (locationId.equals(usage.getLocationId())) { + return usage; + } + } + throw new NoSuchElementException("No location-usage for "+locationId+": "+usages); + } + + private void assertCmdContains(Iterable<? extends ExecCmd> cmds, String expected) { + if (!doesCmdContain(cmds, expected)) { + fail("'"+expected+"' not found in executed commands: "+cmds); + } + } + + private void assertCmdNotContains(Iterable<? extends ExecCmd> cmds, String expected) { + if (doesCmdContain(cmds, expected)) { + fail("Expected '"+expected+"' not present, but found in executed commands: "+cmds); + } + } + + private boolean doesCmdContain(Iterable<? extends ExecCmd> cmds, String expected) { + for (ExecCmd cmd : cmds) { + for (String subCmd : cmd.commands) { + if (subCmd.contains(expected)) { + return true; + } + } + } + return false; + } + + private void assertLocationEvent(LocationEvent event, Entity expectedEntity, Lifecycle expectedState, long preEvent, long postEvent) { + assertLocationEvent(event, expectedEntity.getApplicationId(), expectedEntity.getId(), expectedEntity.getEntityType().getName(), expectedState, preEvent, postEvent); + } + + private void assertLocationEvent(LocationEvent event, String expectedAppId, String expectedEntityId, String expectedEntityType, Lifecycle expectedState, long preEvent, long postEvent) { + // Saw times differ by 1ms - perhaps different threads calling currentTimeMillis() can get out-of-order times?! + final int TIMING_GRACE = 5; + + assertEquals(event.getApplicationId(), expectedAppId); + assertEquals(event.getEntityId(), expectedEntityId); + assertEquals(event.getEntityType(), expectedEntityType); + assertEquals(event.getState(), expectedState); + long eventTime = event.getDate().getTime(); + if (eventTime < (preEvent - TIMING_GRACE) || eventTime > (postEvent + TIMING_GRACE)) { + fail("for "+expectedState+": event=" + Time.makeDateString(eventTime) + "("+eventTime + "); " + + "pre=" + Time.makeDateString(preEvent) + " ("+preEvent+ "); " + + "post=" + Time.makeDateString(postEvent) + " ("+postEvent + ")"); + } + } +} http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/dc0fd3c1/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/LocationUsageTrackingTest.java ---------------------------------------------------------------------- diff --git a/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/LocationUsageTrackingTest.java b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/LocationUsageTrackingTest.java index f877261..4419567 100644 --- a/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/LocationUsageTrackingTest.java +++ b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/test/core/mgmt/usage/LocationUsageTrackingTest.java @@ -30,16 +30,20 @@ import org.apache.brooklyn.api.entity.Entity; import org.apache.brooklyn.api.entity.EntitySpec; import org.apache.brooklyn.api.location.Location; import org.apache.brooklyn.api.location.LocationSpec; +import org.apache.brooklyn.api.location.MachineLocation; import org.apache.brooklyn.api.location.NoMachinesAvailableException; import org.apache.brooklyn.core.entity.lifecycle.Lifecycle; +import org.apache.brooklyn.core.location.Machines; import org.apache.brooklyn.core.mgmt.usage.LocationUsage; import org.apache.brooklyn.core.mgmt.usage.LocationUsage.LocationEvent; import org.apache.brooklyn.core.mgmt.usage.UsageListener.LocationMetadata; import org.apache.brooklyn.core.test.BrooklynAppUnitTestSupport; import org.apache.brooklyn.entity.software.base.SoftwareProcessEntityTest; +import org.apache.brooklyn.location.byon.FixedListMachineProvisioningLocation; import org.apache.brooklyn.location.localhost.LocalhostMachineProvisioningLocation; import org.apache.brooklyn.location.ssh.SshMachineLocation; import org.apache.brooklyn.test.Asserts; +import org.apache.brooklyn.util.core.internal.ssh.RecordingSshTool; import org.apache.brooklyn.util.time.Time; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -58,6 +62,7 @@ public class LocationUsageTrackingTest extends BrooklynAppUnitTestSupport { public void setUp() throws Exception { super.setUp(); loc = mgmt.getLocationManager().createLocation(LocationSpec.create(DynamicLocalhostMachineProvisioningLocation.class)); + RecordingSshTool.clear(); } @Test @@ -129,6 +134,53 @@ public class LocationUsageTrackingTest extends BrooklynAppUnitTestSupport { assertEquals(usage2.getEvents().size(), 2, "usage="+usage2); } + @Test + public void testFoo() throws Exception { + DynamicLocalhostMachineProvisioningLocation recordingLoc = mgmt.getLocationManager().createLocation(LocationSpec.create(DynamicLocalhostMachineProvisioningLocation.class) + .configure("sshToolClass", RecordingSshTool.class.getName())); + + SoftwareProcessEntityTest.MyService entity = app.createAndManageChild(EntitySpec.create(SoftwareProcessEntityTest.MyService.class)); + + // Start the app; expect record of location in use + long preStart = System.currentTimeMillis(); + app.start(ImmutableList.of(recordingLoc)); + long postStart = System.currentTimeMillis(); + SshMachineLocation machine = Machines.findUniqueMachineLocation(entity.getLocations(), SshMachineLocation.class).get(); + + Set<LocationUsage> usages1 = mgmt.getUsageManager().getLocationUsage(Predicates.alwaysTrue()); + LocationUsage usage1 = Iterables.getOnlyElement(usages1); + + assertEquals(usage1.getLocationId(), machine.getId(), "usage="+usage1); + assertNotNull(usage1.getMetadata(), "usage="+usage1); + + assertLocationEvent(usage1.getEvents().get(0), entity, Lifecycle.CREATED, preStart, postStart); + +// assertEquals(event.getApplicationId(), expectedAppId); +// assertEquals(event.getEntityId(), expectedEntityId); +// assertEquals(event.getEntityType(), expectedEntityType); +// assertEquals(event.getState(), expectedState); +// long eventTime = event.getDate().getTime(); +// if (eventTime < (preEvent - TIMING_GRACE) || eventTime > (postEvent + TIMING_GRACE)) { +// fail("for "+expectedState+": event=" + Time.makeDateString(eventTime) + "("+eventTime + "); " +// + "pre=" + Time.makeDateString(preEvent) + " ("+preEvent+ "); " +// + "post=" + Time.makeDateString(postEvent) + " ("+postEvent + ")"); +// } + + // Stop the app; expect record of location no longer in use + long preStop = System.currentTimeMillis(); + app.stop(); + long postStop = System.currentTimeMillis(); + + Set<LocationUsage> usages2 = mgmt.getUsageManager().getLocationUsage(Predicates.alwaysTrue()); + LocationUsage usage2 = Iterables.getOnlyElement(usages2); + assertLocationUsage(usage2, machine); + assertLocationEvent(usage2.getEvents().get(1), app.getApplicationId(), entity.getId(), entity.getEntityType().getName(), Lifecycle.DESTROYED, preStop, postStop); + + assertEquals(usage2.getEvents().size(), 2, "usage="+usage2); + + System.out.println(RecordingSshTool.execScriptCmds); + } + public static class DynamicLocalhostMachineProvisioningLocation extends LocalhostMachineProvisioningLocation { @Override public SshMachineLocation obtain(Map<?, ?> flags) throws NoMachinesAvailableException {
