rewrite of ServiceFailureDetector -- part of what it did before is now done by ServiceStateLogic of course, so now this extends that, and provides options for emitting notifications of ENTITY_FAILED, ENTITY_RECOVERED, as well as suppressing ON_FIRE
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/b2daedf8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/b2daedf8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/b2daedf8 Branch: refs/heads/master Commit: b2daedf8336a891167602f67e7f8e576256dc5e8 Parents: d3886a0 Author: Alex Heneveld <[email protected]> Authored: Mon Aug 25 15:17:53 2014 -0500 Committer: Alex Heneveld <[email protected]> Committed: Wed Aug 27 02:17:18 2014 -0400 ---------------------------------------------------------------------- api/src/main/java/brooklyn/entity/Entity.java | 2 + .../entity/basic/ServiceStateLogic.java | 33 +- .../internal/LocalSubscriptionManager.java | 6 +- .../util/task/BasicExecutionManager.java | 7 +- .../enricher/CustomAggregatingEnricherTest.java | 2 +- .../basic/MultiLocationResolverTest.java | 3 +- .../brooklyn/test/entity/TestClusterImpl.java | 9 + .../java/brooklyn/test/entity/TestEntity.java | 4 +- .../brooklyn/test/entity/TestEntityImpl.java | 10 +- .../brooklyn/demo/CumulusRDFApplication.java | 6 +- .../demo/HighAvailabilityCassandraCluster.java | 5 +- .../java/brooklyn/demo/ResilientMongoDbApp.java | 2 +- .../java/brooklyn/demo/RiakClusterExample.java | 9 +- .../brooklyn/demo/WideAreaCassandraCluster.java | 5 +- .../policy/ha/ServiceFailureDetector.java | 409 +++++-------------- .../entity/brooklyn/BrooklynMetricsTest.java | 20 +- .../autoscaling/AutoScalerPolicyMetricTest.java | 5 +- .../brooklyn/policy/ha/HaPolicyRebindTest.java | 9 +- ...ServiceFailureDetectorStabilizationTest.java | 31 +- .../policy/ha/ServiceFailureDetectorTest.java | 203 +++++---- .../brooklyn/policy/ha/ServiceReplacerTest.java | 2 +- .../brooklyn/entity/webapp/JBossExample.groovy | 48 --- .../test/entity/TestJavaWebAppEntity.groovy | 75 ---- .../test/entity/TestJavaWebAppEntity.java | 73 ++++ 24 files changed, 421 insertions(+), 557 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/api/src/main/java/brooklyn/entity/Entity.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/brooklyn/entity/Entity.java b/api/src/main/java/brooklyn/entity/Entity.java index 88a10fa..9a09392 100644 --- a/api/src/main/java/brooklyn/entity/Entity.java +++ b/api/src/main/java/brooklyn/entity/Entity.java @@ -25,6 +25,8 @@ import java.util.Set; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import com.google.common.annotations.Beta; + import brooklyn.basic.BrooklynObject; import brooklyn.config.ConfigKey; import brooklyn.config.ConfigKey.HasConfigKey; http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java b/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java index 9e66381..e3d5090 100644 --- a/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java +++ b/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java @@ -70,6 +70,12 @@ public class ServiceStateLogic { /** static only; not for instantiation */ private ServiceStateLogic() {} + public static <TKey,TVal> TVal getMapSensorEntry(EntityLocal entity, AttributeSensor<Map<TKey,TVal>> sensor, TKey key) { + Map<TKey, TVal> map = entity.getAttribute(sensor); + if (map==null) return null; + return map.get(key); + } + @SuppressWarnings("unchecked") public static <TKey,TVal> void clearMapSensorEntry(EntityLocal entity, AttributeSensor<Map<TKey,TVal>> sensor, TKey key) { updateMapSensorEntry(entity, sensor, key, (TVal)Entities.REMOVE); @@ -164,6 +170,15 @@ public class ServiceStateLogic { * {@link ServiceStateLogic#newEnricherForServiceState(Class)} and added to an entity. */ public static class ComputeServiceState extends AbstractEnricher implements SensorEventListener<Object> { + public ComputeServiceState() {} + public ComputeServiceState(Map<?,?> flags) { super(flags); } + + @Override + public void init() { + super.init(); + if (uniqueTag==null) uniqueTag = "service.state.actual"; + } + public void setEntity(EntityLocal entity) { super.setEntity(entity); if (suppressDuplicates==null) { @@ -206,7 +221,13 @@ public class ServiceStateLogic { } else if (problems!=null && !problems.isEmpty()) { return Lifecycle.ON_FIRE; } else { - return (up==null ? null : up ? Lifecycle.RUNNING : Lifecycle.STOPPED); + // no expected transition + // if the problems map is non-null, then infer, else leave unchanged + if (problems!=null) + return (up==null ? null /* remove if up is not set */ : + up ? Lifecycle.RUNNING : Lifecycle.STOPPED); + else + return entity.getAttribute(SERVICE_STATE_ACTUAL); } } @@ -220,7 +241,7 @@ public class ServiceStateLogic { return newEnricherForServiceState(ComputeServiceState.class); } public static final EnricherSpec<?> newEnricherForServiceState(Class<? extends Enricher> type) { - return EnricherSpec.create(type).uniqueTag("service.state.actual from service.state.expected and service.problems"); + return EnricherSpec.create(type); } public static class ServiceProblemsLogic { @@ -244,6 +265,14 @@ public class ServiceStateLogic { public static void clearProblemsIndicator(EntityLocal entity, Effector<?> eff) { clearMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, eff.getName()); } + /** as {@link #updateProblemsIndicator(EntityLocal, Sensor, Object)} */ + public static void updateProblemsIndicator(EntityLocal entity, String key, Object value) { + updateMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, key, value); + } + /** as {@link #clearProblemsIndicator(EntityLocal, Sensor)} */ + public static void clearProblemsIndicator(EntityLocal entity, String key) { + clearMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, key); + } } public static class ComputeServiceIndicatorsFromChildrenAndMembers extends AbstractMultipleSensorAggregator<Void> implements SensorEventListener<Object> { http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java b/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java index 0bd8578..83cec75 100644 --- a/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java +++ b/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java @@ -185,7 +185,11 @@ public class LocalSubscriptionManager extends AbstractSubscriptionManager { return "LSM.publish("+event+")"; } public void run() { - sAtClosureCreation.listener.onEvent(event); + try { + sAtClosureCreation.listener.onEvent(event); + } catch (Throwable t) { + LOG.warn("Error in "+this+": "+t, t); + } }}); totalEventsDeliveredCount.incrementAndGet(); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java b/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java index da8c456..2f6e396 100644 --- a/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java +++ b/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java @@ -397,8 +397,13 @@ public class BasicExecutionManager implements ExecutionManager { afterEnd(flags, task); } if (error!=null) { + /* we throw, after logging debug. + * the throw means the error is available for task submitters to monitor. + * however it is possible no one is monitoring it, in which case we will have debug logging only for errors. + * (the alternative, of warn-level logging in lots of places where we don't want it, seems worse!) + */ if (log.isDebugEnabled()) { - // debug only here, because we rethrow + // debug only here, because most submitters will handle failures log.debug("Exception running task "+task+" (rethrowing): "+error.getMessage(), error); if (log.isTraceEnabled()) log.trace("Trace for exception running task "+task+" (rethrowing): "+error.getMessage(), error); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java b/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java index f3e06f6..70d0c10 100644 --- a/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java +++ b/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java @@ -48,7 +48,7 @@ public class CustomAggregatingEnricherTest extends BrooklynAppUnitTestSupport { public static final Logger log = LoggerFactory.getLogger(CustomAggregatingEnricherTest.class); private static final long TIMEOUT_MS = 10*1000; - private static final long SHORT_WAIT_MS = 250; + private static final long SHORT_WAIT_MS = 50; TestEntity entity; SimulatedLocation loc; http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java b/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java index 937679d..85bcfbd 100644 --- a/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java +++ b/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java @@ -44,6 +44,7 @@ import brooklyn.location.MachineProvisioningLocation; import brooklyn.location.NoMachinesAvailableException; import brooklyn.location.cloud.AvailabilityZoneExtension; import brooklyn.management.internal.LocalManagementContext; +import brooklyn.test.entity.LocalManagementContextForTests; import brooklyn.util.collections.MutableList; import brooklyn.util.collections.MutableMap; import brooklyn.util.exceptions.Exceptions; @@ -63,7 +64,7 @@ public class MultiLocationResolverTest { @BeforeMethod(alwaysRun=true) public void setUp() throws Exception { - managementContext = new LocalManagementContext(BrooklynProperties.Factory.newEmpty()); + managementContext = LocalManagementContextForTests.newInstance(); brooklynProperties = managementContext.getBrooklynProperties(); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java b/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java index 0bd2521..3663520 100644 --- a/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java +++ b/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java @@ -18,6 +18,7 @@ */ package brooklyn.test.entity; +import brooklyn.entity.basic.QuorumCheck.QuorumChecks; import brooklyn.entity.group.DynamicClusterImpl; import brooklyn.entity.trait.Startable; @@ -38,6 +39,14 @@ public class TestClusterImpl extends DynamicClusterImpl implements TestCluster { } @Override + protected void initEnrichers() { + // say this is up if it has no children + setConfig(UP_QUORUM_CHECK, QuorumChecks.atLeastOneUnlessEmpty()); + + super.initEnrichers(); + } + + @Override public Integer resize(Integer desiredSize) { this.size = desiredSize; return size; http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestEntity.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/test/entity/TestEntity.java b/core/src/test/java/brooklyn/test/entity/TestEntity.java index ac88acd..c3ca9c2 100644 --- a/core/src/test/java/brooklyn/test/entity/TestEntity.java +++ b/core/src/test/java/brooklyn/test/entity/TestEntity.java @@ -76,7 +76,9 @@ public interface TestEntity extends Entity, Startable, EntityLocal, EntityIntern public static final AttributeSensor<String> NAME = Sensors.newStringSensor("test.name", "Test name"); public static final BasicNotificationSensor<Integer> MY_NOTIF = new BasicNotificationSensor<Integer>(Integer.class, "test.myNotif", "Test notification"); - public static final AttributeSensor<Lifecycle> SERVICE_STATE = Attributes.SERVICE_STATE; + public static final AttributeSensor<Lifecycle> SERVICE_STATE_ACTUAL = Attributes.SERVICE_STATE_ACTUAL; + @Deprecated + public static final AttributeSensor<Lifecycle> SERVICE_STATE = Attributes.SERVICE_STATE_ACTUAL; public static final MethodEffector<Void> MY_EFFECTOR = new MethodEffector<Void>(TestEntity.class, "myEffector"); public static final MethodEffector<Object> IDENTITY_EFFECTOR = new MethodEffector<Object>(TestEntity.class, "identityEffector"); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java b/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java index c2a3884..3ccf614 100644 --- a/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java +++ b/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import brooklyn.entity.Entity; import brooklyn.entity.basic.AbstractEntity; import brooklyn.entity.basic.Lifecycle; +import brooklyn.entity.basic.ServiceStateLogic; import brooklyn.entity.proxying.EntitySpec; import brooklyn.location.Location; import brooklyn.util.collections.MutableMap; @@ -122,19 +123,20 @@ public class TestEntityImpl extends AbstractEntity implements TestEntity { public void start(Collection<? extends Location> locs) { LOG.trace("Starting {}", this); callHistory.add("start"); - setAttribute(SERVICE_STATE, Lifecycle.STARTING); + ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING); counter.incrementAndGet(); addLocations(locs); - setAttribute(SERVICE_STATE, Lifecycle.RUNNING); + ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING); + setAttribute(SERVICE_UP, true); } @Override public void stop() { LOG.trace("Stopping {}", this); callHistory.add("stop"); - setAttribute(SERVICE_STATE, Lifecycle.STOPPING); + ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPING); counter.decrementAndGet(); - setAttribute(SERVICE_STATE, Lifecycle.STOPPED); + ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPED); } @Override http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java ---------------------------------------------------------------------- diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java index a54ef58..eaabaff 100644 --- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java +++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java @@ -31,15 +31,12 @@ import brooklyn.config.ConfigKey; import brooklyn.entity.Effector; import brooklyn.entity.Entity; import brooklyn.entity.basic.AbstractApplication; -import brooklyn.entity.basic.Attributes; import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.EntityInternal; import brooklyn.entity.basic.Lifecycle; import brooklyn.entity.basic.ServiceStateLogic; -import brooklyn.entity.basic.SoftwareProcess; import brooklyn.entity.basic.StartableApplication; -import brooklyn.entity.basic.ServiceStateLogic.ServiceNotUpLogic; import brooklyn.entity.effector.EffectorBody; import brooklyn.entity.effector.Effectors; import brooklyn.entity.java.UsesJava; @@ -58,6 +55,7 @@ import brooklyn.event.basic.DependentConfiguration; import brooklyn.launcher.BrooklynLauncher; import brooklyn.location.Location; import brooklyn.location.basic.PortRanges; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.PolicySpec; import brooklyn.policy.ha.ServiceFailureDetector; import brooklyn.policy.ha.ServiceReplacer; @@ -127,7 +125,7 @@ public class CumulusRDFApplication extends AbstractApplication { .configure(UsesJmx.JMX_PORT, PortRanges.fromString("11099+")) .configure(UsesJmx.RMI_REGISTRY_PORT, PortRanges.fromString("9001+")) .configure(CassandraNode.THRIFT_PORT, PortRanges.fromInteger(getConfig(CASSANDRA_THRIFT_PORT))) - .policy(PolicySpec.create(ServiceFailureDetector.class)) + .enricher(EnricherSpec.create(ServiceFailureDetector.class)) .policy(PolicySpec.create(ServiceRestarter.class) .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED))) .policy(PolicySpec.create(ServiceReplacer.class) http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java ---------------------------------------------------------------------- diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java index 2dce1ea..c21c4cf 100644 --- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java +++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java @@ -23,14 +23,15 @@ import java.util.List; import brooklyn.catalog.Catalog; import brooklyn.catalog.CatalogConfig; import brooklyn.config.ConfigKey; -import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.AbstractApplication; +import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.StartableApplication; import brooklyn.entity.nosql.cassandra.CassandraDatacenter; import brooklyn.entity.nosql.cassandra.CassandraNode; import brooklyn.entity.proxying.EntitySpec; import brooklyn.launcher.BrooklynLauncher; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.PolicySpec; import brooklyn.policy.ha.ServiceFailureDetector; import brooklyn.policy.ha.ServiceReplacer; @@ -64,7 +65,7 @@ public class HighAvailabilityCassandraCluster extends AbstractApplication { //.configure(CassandraCluster.AVAILABILITY_ZONE_NAMES, ImmutableList.of("us-east-1b", "us-east-1c", "us-east-1e")) .configure(CassandraDatacenter.ENDPOINT_SNITCH_NAME, "GossipingPropertyFileSnitch") .configure(CassandraDatacenter.MEMBER_SPEC, EntitySpec.create(CassandraNode.class) - .policy(PolicySpec.create(ServiceFailureDetector.class)) + .enricher(EnricherSpec.create(ServiceFailureDetector.class)) .policy(PolicySpec.create(ServiceRestarter.class) .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED))) .policy(PolicySpec.create(ServiceReplacer.class) http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java ---------------------------------------------------------------------- diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java index 8506b27..290f25e 100644 --- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java +++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java @@ -84,7 +84,7 @@ public class ResilientMongoDbApp extends AbstractApplication implements Startabl /** invoked whenever a new MongoDB server is added (the server may not be started yet) */ protected void initSoftwareProcess(SoftwareProcess p) { - p.addPolicy(new ServiceFailureDetector()); + p.addEnricher(new ServiceFailureDetector()); p.addPolicy(new ServiceRestarter(ServiceFailureDetector.ENTITY_FAILED)); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java ---------------------------------------------------------------------- diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java index d53ce86..0134e27 100644 --- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java +++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java @@ -20,9 +20,6 @@ package brooklyn.demo; import java.util.List; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - import brooklyn.catalog.Catalog; import brooklyn.catalog.CatalogConfig; import brooklyn.config.ConfigKey; @@ -34,11 +31,15 @@ import brooklyn.entity.nosql.riak.RiakCluster; import brooklyn.entity.nosql.riak.RiakNode; import brooklyn.entity.proxying.EntitySpec; import brooklyn.launcher.BrooklynLauncher; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.PolicySpec; import brooklyn.policy.ha.ServiceFailureDetector; import brooklyn.policy.ha.ServiceRestarter; import brooklyn.util.CommandLineUtil; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + @Catalog(name = "Riak Cluster Application", description = "Riak ring deployment blueprint") public class RiakClusterExample extends AbstractApplication { @@ -67,7 +68,7 @@ public class RiakClusterExample extends AbstractApplication { addChild(EntitySpec.create(RiakCluster.class) .configure(RiakCluster.INITIAL_SIZE, getConfig(RIAK_RING_SIZE)) .configure(RiakCluster.MEMBER_SPEC, EntitySpec.create(RiakNode.class) - .policy(PolicySpec.create(ServiceFailureDetector.class)) + .enricher(EnricherSpec.create(ServiceFailureDetector.class)) .policy(PolicySpec.create(ServiceRestarter.class) .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED)))); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java ---------------------------------------------------------------------- diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java index c40eb27..8e30fc4 100644 --- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java +++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java @@ -24,8 +24,8 @@ import java.util.List; import brooklyn.catalog.Catalog; import brooklyn.catalog.CatalogConfig; import brooklyn.config.ConfigKey; -import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.AbstractApplication; +import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.StartableApplication; import brooklyn.entity.nosql.cassandra.CassandraDatacenter; @@ -33,6 +33,7 @@ import brooklyn.entity.nosql.cassandra.CassandraFabric; import brooklyn.entity.nosql.cassandra.CassandraNode; import brooklyn.entity.proxying.EntitySpec; import brooklyn.launcher.BrooklynLauncher; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.PolicySpec; import brooklyn.policy.ha.ServiceFailureDetector; import brooklyn.policy.ha.ServiceReplacer; @@ -61,7 +62,7 @@ public class WideAreaCassandraCluster extends AbstractApplication { .configure(CassandraNode.CUSTOM_SNITCH_JAR_URL, "classpath://brooklyn/entity/nosql/cassandra/cassandra-multicloud-snitch.jar") .configure(CassandraFabric.MEMBER_SPEC, EntitySpec.create(CassandraDatacenter.class) .configure(CassandraDatacenter.MEMBER_SPEC, EntitySpec.create(CassandraNode.class) - .policy(PolicySpec.create(ServiceFailureDetector.class)) + .enricher(EnricherSpec.create(ServiceFailureDetector.class)) .policy(PolicySpec.create(ServiceRestarter.class) .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED))) .policy(PolicySpec.create(ServiceReplacer.class) http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java ---------------------------------------------------------------------- diff --git a/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java b/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java index 5efad2b..c523aba 100644 --- a/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java +++ b/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java @@ -18,13 +18,10 @@ */ package brooklyn.policy.ha; -import static brooklyn.util.time.Time.makeTimeStringRounded; - -import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,15 +30,12 @@ import brooklyn.config.ConfigKey; import brooklyn.entity.basic.Attributes; import brooklyn.entity.basic.ConfigKeys; import brooklyn.entity.basic.EntityInternal; -import brooklyn.entity.basic.EntityLocal; import brooklyn.entity.basic.Lifecycle; -import brooklyn.entity.trait.Startable; +import brooklyn.entity.basic.ServiceStateLogic; +import brooklyn.entity.basic.ServiceStateLogic.ComputeServiceState; import brooklyn.event.SensorEvent; -import brooklyn.event.SensorEventListener; import brooklyn.event.basic.BasicConfigKey; import brooklyn.event.basic.BasicNotificationSensor; -import brooklyn.management.SubscriptionHandle; -import brooklyn.policy.basic.AbstractPolicy; import brooklyn.policy.ha.HASensors.FailureDescriptor; import brooklyn.util.collections.MutableMap; import brooklyn.util.config.ConfigBag; @@ -50,26 +44,17 @@ import brooklyn.util.flags.SetFromFlag; import brooklyn.util.task.BasicTask; import brooklyn.util.task.ScheduledTask; import brooklyn.util.time.Duration; -import brooklyn.util.time.Time; - -import com.google.common.base.Objects; -import com.google.common.collect.Lists; -/** attaches to a SoftwareProcess (or anything emitting SERVICE_UP and SERVICE_STATE) - * and emits HASensors.ENTITY_FAILED and ENTITY_RECOVERED as appropriate - * @see MemberFailureDetectionPolicy +/** + * emits {@link HASensors#ENTITY_FAILED} whenever the parent's default logic ({@link ComputeServiceState}) would detect a problem, + * and similarly {@link HASensors#ENTITY_RECOVERED} when recovered. + * <p> + * gives more control over suppressing {@link Lifecycle#ON_FIRE}, + * for some period of time + * (or until another process manually sets {@link Attributes#SERVICE_STATE_ACTUAL} to {@value Lifecycle#ON_FIRE}, + * which this enricher will not clear until all problems have gone away) */ -public class ServiceFailureDetector extends AbstractPolicy { - - // TODO Remove duplication between this and MemberFailureDetectionPolicy. - // The latter could be re-written to use this. Or could even be deprecated - // in favour of this. - - public enum LastPublished { - NONE, - FAILED, - RECOVERED; - } +public class ServiceFailureDetector extends ServiceStateLogic.ComputeServiceState { private static final Logger LOG = LoggerFactory.getLogger(ServiceFailureDetector.class); @@ -77,47 +62,45 @@ public class ServiceFailureDetector extends AbstractPolicy { public static final BasicNotificationSensor<FailureDescriptor> ENTITY_FAILED = HASensors.ENTITY_FAILED; - // TODO delay before reporting failure (give it time to fix itself, e.g. transient failures) - @SetFromFlag("onlyReportIfPreviouslyUp") - public static final ConfigKey<Boolean> ONLY_REPORT_IF_PREVIOUSLY_UP = ConfigKeys.newBooleanConfigKey("onlyReportIfPreviouslyUp", "", true); + public static final ConfigKey<Boolean> ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP = ConfigKeys.newBooleanConfigKey("onlyReportIfPreviouslyUp", + "Prevents the policy from emitting ENTITY_FAILED if the entity fails on startup (ie has never been up)", true); - @SetFromFlag("useServiceStateRunning") - public static final ConfigKey<Boolean> USE_SERVICE_STATE_RUNNING = ConfigKeys.newBooleanConfigKey("useServiceStateRunning", "", true); + public static final ConfigKey<Boolean> MONITOR_SERVICE_PROBLEMS = ConfigKeys.newBooleanConfigKey("monitorServiceProblems", + "Whether to monitor service problems, and emit on failures there (if set to false, this monitors only service up)", true); - @SetFromFlag("setOnFireOnFailure") - public static final ConfigKey<Boolean> SET_ON_FIRE_ON_FAILURE = ConfigKeys.newBooleanConfigKey("setOnFireOnFailure", "", true); + @SetFromFlag("serviceOnFireStabilizationDelay") + public static final ConfigKey<Duration> SERVICE_ON_FIRE_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class) + .name("serviceOnFire.stabilizationDelay") + .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before concluding ON_FIRE") + .defaultValue(Duration.ZERO) + .build(); - @SetFromFlag("serviceFailedStabilizationDelay") - public static final ConfigKey<Duration> SERVICE_FAILED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class) - .name("serviceRestarter.serviceFailedStabilizationDelay") - .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before concluding failure") + @SetFromFlag("entityFailedStabilizationDelay") + public static final ConfigKey<Duration> ENTITY_FAILED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class) + .name("entityFailed.stabilizationDelay") + .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before emitting ENTITY_FAILED") .defaultValue(Duration.ZERO) .build(); - @SetFromFlag("serviceRecoveredStabilizationDelay") - public static final ConfigKey<Duration> SERVICE_RECOVERED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class) - .name("serviceRestarter.serviceRecoveredStabilizationDelay") - .description("For a failed entity, time period for which the service must be consistently up for (e.g. doesn't report up-down-up) before concluding recovered") + @SetFromFlag("entityRecoveredStabilizationDelay") + public static final ConfigKey<Duration> ENTITY_RECOVERED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class) + .name("entityRecovered.stabilizationDelay") + .description("For a failed entity, time period for which the service must be consistently up for (e.g. doesn't report up-down-up) before emitting ENTITY_RECOVERED") .defaultValue(Duration.ZERO) .build(); - protected final AtomicReference<Boolean> serviceIsUp = new AtomicReference<Boolean>(); - protected final AtomicReference<Lifecycle> serviceState = new AtomicReference<Lifecycle>(); - protected final AtomicReference<Long> serviceLastUp = new AtomicReference<Long>(); - protected final AtomicReference<Long> serviceLastDown = new AtomicReference<Long>(); + protected Long firstUpTime; protected Long currentFailureStartTime = null; protected Long currentRecoveryStartTime = null; - - protected LastPublished lastPublished = LastPublished.NONE; - protected boolean weSetItOnFire = false; + + protected Long publishEntityFailedTime = null; + protected Long publishEntityRecoveredTime = null; private final AtomicBoolean executorQueued = new AtomicBoolean(false); private volatile long executorTime = 0; - private List<SubscriptionHandle> subscriptionHandles = Lists.newCopyOnWriteArrayList(); - public ServiceFailureDetector() { this(new ConfigBag()); } @@ -130,161 +113,98 @@ public class ServiceFailureDetector extends AbstractPolicy { // TODO hierarchy should use ConfigBag, and not change flags super(configBag.getAllConfigMutable()); } - - @Override - public void setEntity(EntityLocal entity) { - super.setEntity(entity); - doSubscribe(); - onMemberAdded(); - } - + @Override - public void suspend() { - super.suspend(); - doUnsubscribe(); + public void onEvent(SensorEvent<Object> event) { + if (firstUpTime==null && event!=null && Attributes.SERVICE_UP.equals(event.getSensor()) && Boolean.TRUE.equals(event.getValue())) { + firstUpTime = event.getTimestamp(); + } + + super.onEvent(event); } @Override - public void resume() { - serviceIsUp.set(null); - serviceState.set(null); - serviceLastUp.set(null); - serviceLastDown.set(null); - currentFailureStartTime = null; - currentRecoveryStartTime = null; - lastPublished = LastPublished.NONE; - weSetItOnFire = false; - executorQueued.set(false); - executorTime = 0; - - super.resume(); - doSubscribe(); - onMemberAdded(); - } - - protected void doSubscribe() { - if (subscriptionHandles.isEmpty()) { - if (getConfig(USE_SERVICE_STATE_RUNNING)) { - SubscriptionHandle handle = subscribe(entity, Attributes.SERVICE_STATE, new SensorEventListener<Lifecycle>() { - @Override public void onEvent(SensorEvent<Lifecycle> event) { - onServiceState(event.getValue()); - } - }); - subscriptionHandles.add(handle); + protected void setActualState(Lifecycle state) { + if (state==Lifecycle.ON_FIRE) { + if (currentFailureStartTime==null) { + currentFailureStartTime = System.currentTimeMillis(); + publishEntityFailedTime = currentFailureStartTime + getConfig(ENTITY_FAILED_STABILIZATION_DELAY).toMilliseconds(); } + // cancel any existing recovery + currentRecoveryStartTime = null; + publishEntityRecoveredTime = null; - SubscriptionHandle handle = subscribe(entity, Startable.SERVICE_UP, new SensorEventListener<Boolean>() { - @Override public void onEvent(SensorEvent<Boolean> event) { - onServiceUp(event.getValue()); - } - }); - subscriptionHandles.add(handle); - } - } - - protected void doUnsubscribe() { - // TODO Could be more defensive with synchronization, but things shouldn't be calling resume + suspend concurrently - for (SubscriptionHandle handle : subscriptionHandles) { - unsubscribe(entity, handle); - } - subscriptionHandles.clear(); - } - - private Duration getServiceFailedStabilizationDelay() { - return getConfig(SERVICE_FAILED_STABILIZATION_DELAY); - } - - private Duration getServiceRecoveredStabilizationDelay() { - return getConfig(SERVICE_RECOVERED_STABILIZATION_DELAY); - } - - private synchronized void onServiceUp(Boolean isNowUp) { - if (isNowUp != null) { - Boolean old = serviceIsUp.getAndSet(isNowUp); - if (isNowUp) { - serviceLastUp.set(System.currentTimeMillis()); + long now = System.currentTimeMillis(); + + long delayBeforeCheck = currentFailureStartTime+getConfig(SERVICE_ON_FIRE_STABILIZATION_DELAY).toMilliseconds() - now; + if (delayBeforeCheck<=0) { + super.setActualState(state); } else { - serviceLastDown.set(System.currentTimeMillis()); - } - if (!Objects.equal(old, serviceIsUp)) { - checkHealth(); + recomputeAfterDelay(delayBeforeCheck); } - } - } - - private synchronized void onServiceState(Lifecycle status) { - if (status != null) { - Lifecycle old = serviceState.getAndSet(status); - if (!Objects.equal(old, status)) { - checkHealth(); - } - } - } - - private synchronized void onMemberAdded() { - if (getConfig(USE_SERVICE_STATE_RUNNING)) { - Lifecycle status = entity.getAttribute(Attributes.SERVICE_STATE); - onServiceState(status); - } - - Boolean isUp = entity.getAttribute(Startable.SERVICE_UP); - onServiceUp(isUp); - } - - private synchronized void checkHealth() { - CalculatedStatus status = calculateStatus(); - boolean failed = status.failed; - boolean healthy = status.healthy; - long now = System.currentTimeMillis(); - - if (healthy) { - if (lastPublished == LastPublished.FAILED) { - if (currentRecoveryStartTime == null) { - LOG.info("{} health-check for {}, component now recovering: {}", new Object[] {this, entity, status.getDescription()}); - currentRecoveryStartTime = now; - schedulePublish(); + + if (publishEntityFailedTime!=null) { + delayBeforeCheck = publishEntityFailedTime - now; + if (firstUpTime==null && getConfig(ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP)) { + // suppress + publishEntityFailedTime = null; + } else if (delayBeforeCheck<=0) { + publishEntityFailedTime = null; + entity.emit(HASensors.ENTITY_FAILED, new HASensors.FailureDescriptor(entity, getFailureDescription(now))); } else { - if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component continuing recovering: {}", new Object[] {this, entity, status.getDescription()}); + recomputeAfterDelay(delayBeforeCheck); } - } else { - if (currentFailureStartTime != null) { - LOG.info("{} health-check for {}, component now healthy: {}", new Object[] {this, entity, status.getDescription()}); + } + + } else { + if (state == Lifecycle.RUNNING) { + if (currentFailureStartTime!=null) { currentFailureStartTime = null; - } else { - if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component still healthy: {}", new Object[] {this, entity, status.getDescription()}); + publishEntityFailedTime = null; + + currentRecoveryStartTime = System.currentTimeMillis(); + publishEntityRecoveredTime = currentRecoveryStartTime + getConfig(ENTITY_RECOVERED_STABILIZATION_DELAY).toMilliseconds(); } } - } else if (failed) { - if (lastPublished != LastPublished.FAILED) { - if (currentFailureStartTime == null) { - LOG.info("{} health-check for {}, component now failing: {}", new Object[] {this, entity, status.getDescription()}); - currentFailureStartTime = now; - schedulePublish(); - } else { - if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component continuing failing: {}", new Object[] {this, entity, status.getDescription()}); - } - } else { - if (currentRecoveryStartTime != null) { - LOG.info("{} health-check for {}, component now failing: {}", new Object[] {this, entity, status.getDescription()}); - currentRecoveryStartTime = null; + + super.setActualState(state); + + if (publishEntityRecoveredTime!=null) { + long now = System.currentTimeMillis(); + long delayBeforeCheck = publishEntityRecoveredTime - now; + if (delayBeforeCheck<=0) { + entity.emit(HASensors.ENTITY_RECOVERED, new HASensors.FailureDescriptor(entity, null)); + publishEntityRecoveredTime = null; } else { - if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component still failed: {}", new Object[] {this, entity, status.getDescription()}); + recomputeAfterDelay(delayBeforeCheck); } } - } else { - if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, in unconfirmed sate: {}", new Object[] {this, entity, status.getDescription()}); } } - - protected CalculatedStatus calculateStatus() { - return new CalculatedStatus(); - } - protected void schedulePublish() { - schedulePublish(0); + private String getFailureDescription(long now) { + String description = null; + Map<String, Object> serviceProblems = entity.getAttribute(Attributes.SERVICE_PROBLEMS); + if (serviceProblems!=null && !serviceProblems.isEmpty()) { + Entry<String, Object> problem = serviceProblems.entrySet().iterator().next(); + description = problem.getKey()+": "+problem.getValue(); + if (serviceProblems.size()>1) { + description = serviceProblems.size()+" service problems, including "+description; + } else { + description = "service problem: "+description; + } + } else if (Boolean.FALSE.equals(entity.getAttribute(Attributes.SERVICE_UP))) { + description = "service not up"; + } else { + description = "service failure detected"; + } + if (publishEntityFailedTime!=null && currentFailureStartTime!=null && publishEntityFailedTime > currentFailureStartTime) + description = " (stabilized for "+Duration.of(now - currentFailureStartTime, TimeUnit.MILLISECONDS)+")"; + return description; } - protected void schedulePublish(long delay) { + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected void recomputeAfterDelay(long delay) { if (isRunning() && executorQueued.compareAndSet(false, true)) { long now = System.currentTimeMillis(); delay = Math.max(0, Math.max(delay, (executorTime + MIN_PERIOD_BETWEEN_EXECS_MILLIS) - now)); @@ -296,16 +216,16 @@ public class ServiceFailureDetector extends AbstractPolicy { executorTime = System.currentTimeMillis(); executorQueued.set(false); - publishNow(); + onEvent(null); } catch (Exception e) { if (isRunning()) { - LOG.error("Error resizing: "+e, e); + LOG.error("Error in enricher "+this+": "+e, e); } else { - if (LOG.isDebugEnabled()) LOG.debug("Error resizing, but no longer running: "+e, e); + if (LOG.isDebugEnabled()) LOG.debug("Error in enricher "+this+" (but no longer running): "+e, e); } } catch (Throwable t) { - LOG.error("Error in service-failure-detector: "+t, t); + LOG.error("Error in enricher "+this+": "+t, t); throw Exceptions.propagate(t); } } @@ -316,115 +236,4 @@ public class ServiceFailureDetector extends AbstractPolicy { } } - private synchronized void publishNow() { - if (!isRunning()) return; - - CalculatedStatus calculatedStatus = calculateStatus(); - - Long lastUpTime = serviceLastUp.get(); - Long lastDownTime = serviceLastDown.get(); - Boolean isUp = serviceIsUp.get(); - Lifecycle status = serviceState.get(); - boolean failed = calculatedStatus.failed; - boolean healthy = calculatedStatus.healthy; - long serviceFailedStabilizationDelay = getServiceFailedStabilizationDelay().toMilliseconds(); - long serviceRecoveredStabilizationDelay = getServiceRecoveredStabilizationDelay().toMilliseconds(); - long now = System.currentTimeMillis(); - - if (failed) { - if (lastPublished != LastPublished.FAILED) { - // only publish if consistently down for serviceFailedStabilizationDelay - long currentFailurePeriod = getTimeDiff(now, currentFailureStartTime); - long sinceLastUpPeriod = getTimeDiff(now, lastUpTime); - if (currentFailurePeriod > serviceFailedStabilizationDelay && sinceLastUpPeriod > serviceFailedStabilizationDelay) { - String description = calculatedStatus.getDescription(); - LOG.warn("{} health-check for {}, publishing component failed: {}", new Object[] {this, entity, description}); - if (getConfig(USE_SERVICE_STATE_RUNNING) && getConfig(SET_ON_FIRE_ON_FAILURE) && status != Lifecycle.ON_FIRE) { - weSetItOnFire = true; - entity.setAttribute(Attributes.SERVICE_STATE, Lifecycle.ON_FIRE); - } - entity.emit(HASensors.ENTITY_FAILED, new HASensors.FailureDescriptor(entity, description)); - lastPublished = LastPublished.FAILED; - currentRecoveryStartTime = null; - } else { - long nextAttemptTime = Math.max(serviceFailedStabilizationDelay - currentFailurePeriod, serviceFailedStabilizationDelay - sinceLastUpPeriod); - schedulePublish(nextAttemptTime); - } - } - } else if (healthy) { - if (lastPublished == LastPublished.FAILED) { - // only publish if consistently up for serviceRecoveredStabilizationDelay - long currentRecoveryPeriod = getTimeDiff(now, currentRecoveryStartTime); - long sinceLastDownPeriod = getTimeDiff(now, lastDownTime); - if (currentRecoveryPeriod > serviceRecoveredStabilizationDelay && sinceLastDownPeriod > serviceRecoveredStabilizationDelay) { - String description = calculatedStatus.getDescription(); - LOG.warn("{} health-check for {}, publishing component recovered: {}", new Object[] {this, entity, description}); - if (weSetItOnFire) { - if (status == Lifecycle.ON_FIRE) { - entity.setAttribute(Attributes.SERVICE_STATE, Lifecycle.RUNNING); - } - weSetItOnFire = false; - } - entity.emit(HASensors.ENTITY_RECOVERED, new HASensors.FailureDescriptor(entity, description)); - lastPublished = LastPublished.RECOVERED; - currentFailureStartTime = null; - } else { - long nextAttemptTime = Math.max(serviceRecoveredStabilizationDelay - currentRecoveryPeriod, serviceRecoveredStabilizationDelay - sinceLastDownPeriod); - schedulePublish(nextAttemptTime); - } - } - } - } - - public class CalculatedStatus { - public final boolean failed; - public final boolean healthy; - - public CalculatedStatus() { - Long lastUpTime = serviceLastUp.get(); - Boolean isUp = serviceIsUp.get(); - Lifecycle status = serviceState.get(); - - failed = - (getConfig(USE_SERVICE_STATE_RUNNING) && status == Lifecycle.ON_FIRE && !weSetItOnFire) || - (Boolean.FALSE.equals(isUp) && - (getConfig(USE_SERVICE_STATE_RUNNING) ? status == Lifecycle.RUNNING : true) && - (getConfig(ONLY_REPORT_IF_PREVIOUSLY_UP) ? lastUpTime != null : true)); - healthy = - (getConfig(USE_SERVICE_STATE_RUNNING) ? (status == Lifecycle.RUNNING || (weSetItOnFire && status == Lifecycle.ON_FIRE)) : - true) && - Boolean.TRUE.equals(isUp); - } - - public String getDescription() { - Long lastUpTime = serviceLastUp.get(); - Boolean isUp = serviceIsUp.get(); - Lifecycle status = serviceState.get(); - Duration serviceFailedStabilizationDelay = getServiceFailedStabilizationDelay(); - Duration serviceRecoveredStabilizationDelay = getServiceRecoveredStabilizationDelay(); - - return String.format("location=%s; isUp=%s; status=%s; timeNow=%s; lastReportedUp=%s; lastPublished=%s; "+ - "currentFailurePeriod=%s; currentRecoveryPeriod=%s", - entity.getLocations(), - (isUp != null ? isUp : "<unreported>"), - (status != null ? status : "<unreported>"), - Time.makeDateString(System.currentTimeMillis()), - (lastUpTime != null ? Time.makeDateString(lastUpTime) : "<never>"), - lastPublished, - (currentFailureStartTime != null ? getTimeStringSince(currentFailureStartTime) : "<none>") + " (stabilization "+makeTimeStringRounded(serviceFailedStabilizationDelay) + ")", - (currentRecoveryStartTime != null ? getTimeStringSince(currentRecoveryStartTime) : "<none>") + " (stabilization "+makeTimeStringRounded(serviceRecoveredStabilizationDelay) + ")"); - } - } - - private long getTimeDiff(Long recent, Long previous) { - return (previous == null) ? recent : (recent - previous); - } - - private String getTimeStringSince(Long time) { - return time == null ? null : Time.makeTimeStringRounded(System.currentTimeMillis() - time); - } - - private String getTimeStringSince(AtomicReference<Long> timeRef) { - return getTimeStringSince(timeRef.get()); - } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java b/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java index 39be9d4..319b345 100644 --- a/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java +++ b/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java @@ -26,22 +26,24 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import brooklyn.entity.Entity; -import brooklyn.entity.basic.ApplicationBuilder; import brooklyn.entity.basic.Entities; import brooklyn.entity.proxying.EntitySpec; import brooklyn.event.AttributeSensor; import brooklyn.event.SensorEventListener; import brooklyn.location.basic.SimulatedLocation; import brooklyn.test.Asserts; -import brooklyn.test.entity.LocalManagementContextForTests; import brooklyn.test.entity.TestApplication; import brooklyn.test.entity.TestEntity; +import brooklyn.test.entity.TestEntityNoEnrichersImpl; +import brooklyn.util.collections.MutableMap; +import brooklyn.util.time.Duration; import com.google.common.collect.ImmutableList; public class BrooklynMetricsTest { private static final long TIMEOUT_MS = 2*1000; + private final static int DEFAULT_SUBSCRIPTIONS_PER_ENTITY = 2; TestApplication app; SimulatedLocation loc; @@ -52,7 +54,6 @@ public class BrooklynMetricsTest { loc = new SimulatedLocation(); app = TestApplication.Factory.newManagedInstanceForTests(); brooklynMetrics = app.createAndManageChild(EntitySpec.create(BrooklynMetrics.class).configure("updatePeriod", 10L)); - Entities.manage(brooklynMetrics); } @AfterMethod(alwaysRun=true) @@ -64,7 +65,7 @@ public class BrooklynMetricsTest { public void testInitialBrooklynMetrics() { app.start(ImmutableList.of(loc)); - Asserts.succeedsEventually(new Runnable() { + Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() { public void run() { assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EFFECTORS_INVOKED), (Long)1L); assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_TASKS_SUBMITTED) > 0); @@ -72,16 +73,16 @@ public class BrooklynMetricsTest { assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_ACTIVE_TASKS), (Long)0L); assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_PUBLISHED) > 0); assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_DELIVERED), (Long)0L); - assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)0L); + assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)(2L*DEFAULT_SUBSCRIPTIONS_PER_ENTITY)); }}); } @Test public void testBrooklynMetricsIncremented() { - TestEntity e = app.createAndManageChild(EntitySpec.create(TestEntity.class)); + TestEntity e = app.createAndManageChild(EntitySpec.create(TestEntity.class, TestEntityNoEnrichersImpl.class)); app.start(ImmutableList.of(loc)); - Asserts.succeedsEventually(new Runnable() { + Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() { public void run() { assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EFFECTORS_INVOKED), (Long)2L); // for app and testEntity's start }}); @@ -106,11 +107,12 @@ public class BrooklynMetricsTest { app.subscribe(e, TestEntity.SEQUENCE, SensorEventListener.NOOP); e.setAttribute(TestEntity.SEQUENCE, 1); - Asserts.succeedsEventually(new Runnable() { + Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() { public void run() { assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_PUBLISHED) > eventsPublished); assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_DELIVERED) > eventsDelivered); - assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)1L); + assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long) + (1L + 2*DEFAULT_SUBSCRIPTIONS_PER_ENTITY)); }}); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java b/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java index 2290a33..7cec2ed 100644 --- a/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java +++ b/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java @@ -31,7 +31,6 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import brooklyn.entity.basic.ApplicationBuilder; import brooklyn.entity.basic.Entities; import brooklyn.entity.proxying.EntitySpec; import brooklyn.event.AttributeSensor; @@ -49,7 +48,7 @@ import com.google.common.collect.Lists; public class AutoScalerPolicyMetricTest { private static long TIMEOUT_MS = 10000; - private static long SHORT_WAIT_MS = 250; + private static long SHORT_WAIT_MS = 50; private static final AttributeSensor<Integer> MY_ATTRIBUTE = Sensors.newIntegerSensor("autoscaler.test.intAttrib"); TestApplication app; @@ -57,7 +56,7 @@ public class AutoScalerPolicyMetricTest { @BeforeMethod(alwaysRun=true) public void before() { - app = ApplicationBuilder.newManagedApp(TestApplication.class); + app = TestApplication.Factory.newManagedInstanceForTests(); tc = app.createAndManageChild(EntitySpec.create(TestCluster.class) .configure("initialSize", 1)); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java b/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java index 9d8f427..0cccf14 100644 --- a/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java +++ b/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java @@ -31,6 +31,7 @@ import org.testng.annotations.Test; import brooklyn.entity.Entity; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.Lifecycle; +import brooklyn.entity.basic.ServiceStateLogic; import brooklyn.entity.group.DynamicCluster; import brooklyn.entity.proxying.EntitySpec; import brooklyn.entity.rebind.RebindTestFixtureWithApp; @@ -40,6 +41,7 @@ import brooklyn.event.SensorEventListener; import brooklyn.location.Location; import brooklyn.location.LocationSpec; import brooklyn.location.basic.SimulatedLocation; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.PolicySpec; import brooklyn.policy.ha.HASensors.FailureDescriptor; import brooklyn.test.Asserts; @@ -131,7 +133,7 @@ public class HaPolicyRebindTest extends RebindTestFixtureWithApp { @Test public void testServiceFailureDetectorWorksAfterRebind() throws Exception { - origEntity.addPolicy(PolicySpec.create(ServiceFailureDetector.class)); + origEntity.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); // rebind TestApplication newApp = rebind(); @@ -139,9 +141,10 @@ public class HaPolicyRebindTest extends RebindTestFixtureWithApp { newApp.getManagementContext().getSubscriptionManager().subscribe(newEntity, HASensors.ENTITY_FAILED, eventListener); - // stimulate the policy - newEntity.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); newEntity.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(newEntity, Lifecycle.RUNNING); + + // trigger the failure newEntity.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(newEntity), null); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java index 3426598..4649b36 100644 --- a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java +++ b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java @@ -31,12 +31,14 @@ import org.testng.annotations.Test; import brooklyn.entity.basic.ApplicationBuilder; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.Lifecycle; +import brooklyn.entity.basic.ServiceStateLogic; +import brooklyn.entity.basic.ServiceStateLogicTest; import brooklyn.entity.proxying.EntitySpec; import brooklyn.event.Sensor; import brooklyn.event.SensorEvent; import brooklyn.event.SensorEventListener; import brooklyn.management.ManagementContext; -import brooklyn.policy.PolicySpec; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.ha.HASensors.FailureDescriptor; import brooklyn.test.Asserts; import brooklyn.test.entity.LocalManagementContextForTests; @@ -49,6 +51,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableMap; +/** also see more primitive tests in {@link ServiceStateLogicTest} */ public class ServiceFailureDetectorStabilizationTest { private static final int TIMEOUT_MS = 10*1000; @@ -67,8 +70,8 @@ public class ServiceFailureDetectorStabilizationTest { managementContext = new LocalManagementContextForTests(); app = ApplicationBuilder.newManagedApp(TestApplication.class, managementContext); e1 = app.createAndManageChild(EntitySpec.create(TestEntity.class)); - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); app.getManagementContext().getSubscriptionManager().subscribe( e1, @@ -95,8 +98,8 @@ public class ServiceFailureDetectorStabilizationTest { @Test(groups="Integration") // Because slow public void testNotNotifiedOfTemporaryFailuresDuringStabilisationDelay() throws Exception { - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.ONE_MINUTE)); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.ONE_MINUTE)); e1.setAttribute(TestEntity.SERVICE_UP, false); Thread.sleep(100); @@ -109,8 +112,8 @@ public class ServiceFailureDetectorStabilizationTest { public void testNotifiedOfFailureAfterStabilisationDelay() throws Exception { final int stabilisationDelay = 1000; - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); e1.setAttribute(TestEntity.SERVICE_UP, false); @@ -122,8 +125,8 @@ public class ServiceFailureDetectorStabilizationTest { public void testFailuresThenUpDownResetsStabilisationCount() throws Exception { final long stabilisationDelay = 1000; - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); e1.setAttribute(TestEntity.SERVICE_UP, false); assertNoEventsContinually(Duration.of(stabilisationDelay - OVERHEAD)); @@ -139,8 +142,8 @@ public class ServiceFailureDetectorStabilizationTest { public void testNotNotifiedOfTemporaryRecoveryDuringStabilisationDelay() throws Exception { final long stabilisationDelay = 1000; - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); e1.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); @@ -157,8 +160,8 @@ public class ServiceFailureDetectorStabilizationTest { public void testNotifiedOfRecoveryAfterStabilisationDelay() throws Exception { final int stabilisationDelay = 1000; - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); e1.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); @@ -173,8 +176,8 @@ public class ServiceFailureDetectorStabilizationTest { public void testRecoversThenDownUpResetsStabilisationCount() throws Exception { final long stabilisationDelay = 1000; - e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class) - .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay))); e1.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java index 98c28de..5c9dd66 100644 --- a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java +++ b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java @@ -32,11 +32,14 @@ import org.testng.annotations.Test; import brooklyn.entity.basic.ApplicationBuilder; import brooklyn.entity.basic.Entities; import brooklyn.entity.basic.Lifecycle; +import brooklyn.entity.basic.ServiceStateLogic; +import brooklyn.entity.basic.ServiceStateLogic.ServiceProblemsLogic; import brooklyn.entity.proxying.EntitySpec; import brooklyn.event.Sensor; import brooklyn.event.SensorEvent; import brooklyn.event.SensorEventListener; import brooklyn.management.ManagementContext; +import brooklyn.policy.EnricherSpec; import brooklyn.policy.ha.HASensors.FailureDescriptor; import brooklyn.test.Asserts; import brooklyn.test.EntityTestUtils; @@ -44,19 +47,19 @@ import brooklyn.test.entity.LocalManagementContextForTests; import brooklyn.test.entity.TestApplication; import brooklyn.test.entity.TestEntity; import brooklyn.util.collections.MutableMap; +import brooklyn.util.time.Duration; +import brooklyn.util.time.Time; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableMap; public class ServiceFailureDetectorTest { - private static final int TIMEOUT_MS = 10*1000; + private static final int TIMEOUT_MS = 1*1000; private ManagementContext managementContext; private TestApplication app; private TestEntity e1; - private ServiceFailureDetector policy; private List<SensorEvent<FailureDescriptor>> events; private SensorEventListener<FailureDescriptor> eventListener; @@ -73,6 +76,7 @@ public class ServiceFailureDetectorTest { managementContext = new LocalManagementContextForTests(); app = ApplicationBuilder.newManagedApp(TestApplication.class, managementContext); e1 = app.createAndManageChild(EntitySpec.create(TestEntity.class)); + e1.addEnricher(ServiceStateLogic.newEnricherForServiceStateFromProblemsAndUp()); app.getManagementContext().getSubscriptionManager().subscribe(e1, HASensors.ENTITY_FAILED, eventListener); app.getManagementContext().getSubscriptionManager().subscribe(e1, HASensors.ENTITY_RECOVERED, eventListener); @@ -86,181 +90,220 @@ public class ServiceFailureDetectorTest { @Test(groups="Integration") // Has a 1 second wait public void testNotNotifiedOfFailuresForHealthy() throws Exception { // Create members before and after the policy is registered, to test both scenarios - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); assertNoEventsContinually(); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); } @Test public void testNotifiedOfFailure() throws Exception { - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); + + assertEquals(events.size(), 0, "events="+events); + e1.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); assertEquals(events.size(), 1, "events="+events); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); } @Test - public void testNotifiedOfFailureOnStateOnFire() throws Exception { - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + public void testNotifiedOfFailureOnProblem() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); + + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE); + assertEquals(events.size(), 0, "events="+events); + + ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo"); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); assertEquals(events.size(), 1, "events="+events); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); + } + + @Test + public void testNotifiedOfFailureOnStateOnFire() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.ON_FIRE); + + assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + assertEquals(events.size(), 1, "events="+events); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); } @Test public void testNotifiedOfRecovery() throws Exception { - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); - // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); + // Make the entity fail e1.setAttribute(TestEntity.SERVICE_UP, false); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); // And make the entity recover e1.setAttribute(TestEntity.SERVICE_UP, true); assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null); assertEquals(events.size(), 2, "events="+events); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING); } - @Test(groups="Integration") // Has a 1 second wait - public void testOnlyReportsFailureIfPreviouslyUp() throws Exception { - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + @Test + public void testNotifiedOfRecoveryFromProblems() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); - e1.setAttribute(TestEntity.SERVICE_UP, false); + ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo"); - assertNoEventsContinually(); + assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); + + // And make the entity recover + ServiceProblemsLogic.clearProblemsIndicator(e1, "test"); + assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null); + assertEquals(events.size(), 2, "events="+events); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING); } - @Test - public void testDisablingOnlyReportsFailureIfPreviouslyUp() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + + @Test(groups="Integration") // Has a 1 second wait + public void testEmitsEntityFailureOnlyIfPreviouslyUp() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, false); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); - assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); + assertNoEventsContinually(); } @Test - public void testSetsOnFireOnFailure() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + public void testDisablingPreviouslyUpRequirementForEntityFailed() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false)); - // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, false); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); - EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); + assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); } @Test public void testDisablingSetsOnFireOnFailure() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("setOnFireOnFailure", false, "onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.PRACTICALLY_FOREVER)); // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, false); - EntityTestUtils.assertAttributeEqualsContinually(e1, TestEntity.SERVICE_STATE, Lifecycle.RUNNING); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); } @Test(groups="Integration") // Has a 1 second wait - public void testUsesServiceStateRunning() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + public void testOnFireAfterDelay() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.ONE_SECOND)); - // entity no counted as failed, because serviceState != running || onfire + // Make the entity fail + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, false); - assertNoEventsContinually(); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); + Time.sleep(Duration.millis(100)); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); } - - @Test - public void testDisablingUsesServiceStateRunning() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("useServiceStateRunning", false, "onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + + @Test(groups="Integration") // Has a 1 second wait + public void testOnFailureDelayFromProblemAndRecover() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.ONE_SECOND) + .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.ONE_SECOND)); // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_UP, false); + e1.setAttribute(TestEntity.SERVICE_UP, true); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); + ServiceStateLogic.ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo"); + + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); + Time.sleep(Duration.millis(100)); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING); + + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); + + // Now recover + ServiceStateLogic.ServiceProblemsLogic.clearProblemsIndicator(e1, "test"); + EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING); + + assertEquals(events.size(), 1, "events="+events); + + assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null); + assertEquals(events.size(), 2, "events="+events); } - + @Test(groups="Integration") // Has a 1 second wait - public void testOnlyReportsFailureIfRunning() throws Exception { - policy = new ServiceFailureDetector(); - e1.addPolicy(policy); + public void testAttendsToServiceState() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); - // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.STARTING); e1.setAttribute(TestEntity.SERVICE_UP, true); + // not counted as failed because not expected to be running e1.setAttribute(TestEntity.SERVICE_UP, false); assertNoEventsContinually(); } - - @Test - public void testReportsFailureWhenNotPreviouslyUp() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); - - // Make the entity fail - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); - e1.setAttribute(TestEntity.SERVICE_UP, false); - assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); - } - - @Test - public void testReportsFailureWhenNoServiceState() throws Exception { - policy = new ServiceFailureDetector(ImmutableMap.of("useServiceStateRunning", false)); - e1.addPolicy(policy); + @Test(groups="Integration") // Has a 1 second wait + public void testOnlyReportsFailureIfRunning() throws Exception { + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)); // Make the entity fail + ServiceStateLogic.setExpectedState(e1, Lifecycle.STARTING); e1.setAttribute(TestEntity.SERVICE_UP, true); e1.setAttribute(TestEntity.SERVICE_UP, false); - assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); + assertNoEventsContinually(); } @Test public void testReportsFailureWhenAlreadyDownOnRegisteringPolicy() throws Exception { - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING); + ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING); e1.setAttribute(TestEntity.SERVICE_UP, false); - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false)); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); } @Test public void testReportsFailureWhenAlreadyOnFireOnRegisteringPolicy() throws Exception { - e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE); + ServiceStateLogic.setExpectedState(e1, Lifecycle.ON_FIRE); - policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false)); - e1.addPolicy(policy); + e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class) + .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false)); assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null); } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java ---------------------------------------------------------------------- diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java index 4b5a0d9..8d1a683 100644 --- a/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java +++ b/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java @@ -138,7 +138,7 @@ public class ServiceReplacerTest { e1.emit(HASensors.ENTITY_FAILED, new FailureDescriptor(e1, "simulate failure")); // Expect cluster to go on-fire when fails to start replacement - EntityTestUtils.assertAttributeEqualsEventually(cluster, Attributes.SERVICE_STATE, Lifecycle.ON_FIRE); + EntityTestUtils.assertAttributeEqualsEventually(cluster, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE); // And expect to have the second failed entity still kicking around as proof (in quarantine) Iterable<Entity> members = Iterables.filter(managementContext.getEntityManager().getEntities(), Predicates.instanceOf(FailingEntity.class));
