This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new ace6761 CAMEL-17143: camel-health - Add success threshold
ace6761 is described below
commit ace6761c2b2f482c297cf16f32320742c2eb660d
Author: Claus Ibsen <[email protected]>
AuthorDate: Fri Oct 29 15:29:53 2021 +0200
CAMEL-17143: camel-health - Add success threshold
---
.../java/org/apache/camel/health/HealthCheck.java | 1 +
.../camel/health/HealthCheckConfiguration.java | 24 ++++++++++++++++
.../apache/camel/impl/health/HealthCheckTest.java | 22 ++++++++++++++-
.../camel/impl/health/AbstractHealthCheck.java | 33 ++++++++++++++++++----
.../impl/health/HealthCheckRegistryRepository.java | 1 +
...althCheckConfigurationPropertiesConfigurer.java | 6 ++++
.../org/apache/camel/main/BaseMainSupport.java | 1 +
.../api/management/mbean/CamelOpenMBeanTypes.java | 6 ++--
.../camel/management/mbean/ManagedCamelHealth.java | 2 ++
.../modules/ROOT/pages/health-check.adoc | 7 ++++-
10 files changed, 93 insertions(+), 10 deletions(-)
diff --git
a/core/camel-api/src/main/java/org/apache/camel/health/HealthCheck.java
b/core/camel-api/src/main/java/org/apache/camel/health/HealthCheck.java
index 9f06c9e..bb9fd2a 100644
--- a/core/camel-api/src/main/java/org/apache/camel/health/HealthCheck.java
+++ b/core/camel-api/src/main/java/org/apache/camel/health/HealthCheck.java
@@ -38,6 +38,7 @@ public interface HealthCheck extends HasGroup, HasId, Ordered
{
String FAILURE_COUNT = "failure.count";
String FAILURE_ENDPOINT_URI = "failure.endpoint.uri";
String FAILURE_ERROR_COUNT = "failure.error.count";
+ String SUCCESS_COUNT = "success.count";
enum State {
UP,
diff --git
a/core/camel-api/src/main/java/org/apache/camel/health/HealthCheckConfiguration.java
b/core/camel-api/src/main/java/org/apache/camel/health/HealthCheckConfiguration.java
index 143dbc1..22eaeb9 100644
---
a/core/camel-api/src/main/java/org/apache/camel/health/HealthCheckConfiguration.java
+++
b/core/camel-api/src/main/java/org/apache/camel/health/HealthCheckConfiguration.java
@@ -31,6 +31,7 @@ public class HealthCheckConfiguration implements Cloneable {
private boolean enabled = true;
private long interval;
private int failureThreshold;
+ private int successThreshold;
// *************************************************
// Properties
@@ -82,6 +83,17 @@ public class HealthCheckConfiguration implements Cloneable {
this.failureThreshold = failureThreshold;
}
+ public int getSuccessThreshold() {
+ return successThreshold;
+ }
+
+ /**
+ * Set the number of success before reporting the service as healthy.
+ */
+ public void setSuccessThreshold(int successThreshold) {
+ this.successThreshold = successThreshold;
+ }
+
public HealthCheckConfiguration copy() {
try {
return (HealthCheckConfiguration) super.clone();
@@ -103,6 +115,7 @@ public class HealthCheckConfiguration implements Cloneable {
private Boolean enabled;
private Long interval;
private Integer failureThreshold;
+ private Integer successThreshold;
private Builder() {
}
@@ -121,6 +134,9 @@ public class HealthCheckConfiguration implements Cloneable {
if (this.failureThreshold == null) {
this.failureThreshold = template.failureThreshold;
}
+ if (this.successThreshold == null) {
+ this.successThreshold = template.successThreshold;
+ }
}
return this;
@@ -157,6 +173,11 @@ public class HealthCheckConfiguration implements Cloneable
{
return this;
}
+ public Builder successThreshold(int successThreshold) {
+ this.successThreshold = successThreshold;
+ return this;
+ }
+
@Override
public HealthCheckConfiguration build() {
HealthCheckConfiguration conf = new HealthCheckConfiguration();
@@ -172,6 +193,9 @@ public class HealthCheckConfiguration implements Cloneable {
if (failureThreshold != null) {
conf.setFailureThreshold(failureThreshold);
}
+ if (successThreshold != null) {
+ conf.setSuccessThreshold(successThreshold);
+ }
return conf;
}
}
diff --git
a/core/camel-core/src/test/java/org/apache/camel/impl/health/HealthCheckTest.java
b/core/camel-core/src/test/java/org/apache/camel/impl/health/HealthCheckTest.java
index 096325d..1fe145d 100644
---
a/core/camel-core/src/test/java/org/apache/camel/impl/health/HealthCheckTest.java
+++
b/core/camel-core/src/test/java/org/apache/camel/impl/health/HealthCheckTest.java
@@ -88,7 +88,7 @@ public class HealthCheckTest {
}
@Test
- public void testThreshold() throws Exception {
+ public void testFailureThreshold() throws Exception {
MyHealthCheck check = new MyHealthCheck();
check.setState(HealthCheck.State.DOWN);
check.getConfiguration().setEnabled(true);
@@ -108,6 +108,26 @@ public class HealthCheckTest {
}
@Test
+ public void testSuccessThreshold() throws Exception {
+ MyHealthCheck check = new MyHealthCheck();
+ check.setState(HealthCheck.State.UP);
+ check.getConfiguration().setEnabled(true);
+ check.getConfiguration().setSuccessThreshold(2);
+
+ HealthCheck.Result result;
+
+ for (int i = 0; i < check.getConfiguration().getSuccessThreshold();
i++) {
+ result = check.call();
+
+ assertEquals(HealthCheck.State.DOWN, result.getState());
+ assertEquals(i + 1,
result.getDetails().get(AbstractHealthCheck.INVOCATION_COUNT));
+ assertEquals(i + 1,
result.getDetails().get(AbstractHealthCheck.SUCCESS_COUNT));
+ }
+
+ assertEquals(HealthCheck.State.UP, check.call().getState());
+ }
+
+ @Test
public void testIntervalThreshold() throws Exception {
MyHealthCheck check = new MyHealthCheck();
check.setState(HealthCheck.State.DOWN);
diff --git
a/core/camel-health/src/main/java/org/apache/camel/impl/health/AbstractHealthCheck.java
b/core/camel-health/src/main/java/org/apache/camel/impl/health/AbstractHealthCheck.java
index cef9605..24fdd64 100644
---
a/core/camel-health/src/main/java/org/apache/camel/impl/health/AbstractHealthCheck.java
+++
b/core/camel-health/src/main/java/org/apache/camel/impl/health/AbstractHealthCheck.java
@@ -109,11 +109,13 @@ public abstract class AbstractHealthCheck implements
HealthCheck {
final ZonedDateTime now = ZonedDateTime.now();
final boolean enabled = conf.isEnabled();
final long interval = conf.getInterval();
- final int threshold = conf.getFailureThreshold();
+ final int failureThreshold = conf.getFailureThreshold();
+ final int successThreshold = conf.getSuccessThreshold();
// Extract relevant information from meta data.
int invocationCount = (Integer)
meta.getOrDefault(INVOCATION_COUNT, 0);
int failureCount = (Integer) meta.getOrDefault(FAILURE_COUNT, 0);
+ int successCount = (Integer) meta.getOrDefault(SUCCESS_COUNT, 0);
String invocationTime =
now.format(DateTimeFormatter.ISO_ZONED_DATE_TIME);
boolean call = true;
@@ -156,33 +158,54 @@ public abstract class AbstractHealthCheck implements
HealthCheck {
ObjectHelper.notNull(builder.state(), "Response State");
if (builder.state() == State.DOWN) {
+ // reset success since it failed
+ successCount = 0;
+
// If the service is un-healthy but the number of time it
// has been consecutively reported in this state is less
// than the threshold configured, mark it as UP. This is
// used to avoid false positive in case of glitches.
- if (failureCount++ < threshold) {
+ if (failureCount++ < failureThreshold) {
LOGGER.debug(
"Health-check {}/{} has status DOWN but
failure count ({}) is less than configured threshold ({})",
getGroup(),
getId(),
failureCount,
- threshold);
+ failureThreshold);
builder.up();
}
- } else {
+ } else if (builder.state() == State.UP) {
+ // reset failure since it ok
failureCount = 0;
+
+ // If the service is healthy but the number of time it
+ // has been consecutively reported in this state is less
+ // than the threshold configured, mark it as DOWN. This is
+ // used to avoid false positive in case of glitches.
+ if (successCount++ < successThreshold) {
+ LOGGER.debug(
+ "Health-check {}/{} has status UP but success
count ({}) is less than configured threshold ({})",
+ getGroup(),
+ getId(),
+ successCount,
+ successThreshold);
+
+ builder.down();
+ }
}
meta.put(INVOCATION_TIME, invocationTime);
- meta.put(FAILURE_COUNT, failureCount);
meta.put(INVOCATION_COUNT, ++invocationCount);
+ meta.put(FAILURE_COUNT, failureCount);
+ meta.put(SUCCESS_COUNT, successCount);
// Copy some of the meta-data bits to the response attributes
so the
// response caches the health-check state at the time of the
invocation.
builder.detail(INVOCATION_TIME, meta.get(INVOCATION_TIME));
builder.detail(INVOCATION_COUNT, meta.get(INVOCATION_COUNT));
builder.detail(FAILURE_COUNT, meta.get(FAILURE_COUNT));
+ builder.detail(SUCCESS_COUNT, meta.get(SUCCESS_COUNT));
// update last invocation time.
lastInvocation = now;
diff --git
a/core/camel-health/src/main/java/org/apache/camel/impl/health/HealthCheckRegistryRepository.java
b/core/camel-health/src/main/java/org/apache/camel/impl/health/HealthCheckRegistryRepository.java
index 26e7e9e..516581e 100644
---
a/core/camel-health/src/main/java/org/apache/camel/impl/health/HealthCheckRegistryRepository.java
+++
b/core/camel-health/src/main/java/org/apache/camel/impl/health/HealthCheckRegistryRepository.java
@@ -103,6 +103,7 @@ public class HealthCheckRegistryRepository implements
CamelContextAware, HealthC
hc.getConfiguration().setEnabled(hcc.isEnabled());
hc.getConfiguration().setInterval(hcc.getInterval());
hc.getConfiguration().setFailureThreshold(hcc.getFailureThreshold());
+
hc.getConfiguration().setSuccessThreshold(hcc.getSuccessThreshold());
}
}
return hc;
diff --git
a/core/camel-main/src/generated/java/org/apache/camel/main/HealthCheckConfigurationPropertiesConfigurer.java
b/core/camel-main/src/generated/java/org/apache/camel/main/HealthCheckConfigurationPropertiesConfigurer.java
index b4c0af2..53c42cd 100644
---
a/core/camel-main/src/generated/java/org/apache/camel/main/HealthCheckConfigurationPropertiesConfigurer.java
+++
b/core/camel-main/src/generated/java/org/apache/camel/main/HealthCheckConfigurationPropertiesConfigurer.java
@@ -29,6 +29,8 @@ public class HealthCheckConfigurationPropertiesConfigurer
extends org.apache.cam
case "Interval": target.setInterval(property(camelContext, long.class,
value)); return true;
case "parent":
case "Parent": target.setParent(property(camelContext,
java.lang.String.class, value)); return true;
+ case "successthreshold":
+ case "SuccessThreshold":
target.setSuccessThreshold(property(camelContext, int.class, value)); return
true;
default: return false;
}
}
@@ -44,6 +46,8 @@ public class HealthCheckConfigurationPropertiesConfigurer
extends org.apache.cam
case "Interval": return long.class;
case "parent":
case "Parent": return java.lang.String.class;
+ case "successthreshold":
+ case "SuccessThreshold": return int.class;
default: return null;
}
}
@@ -60,6 +64,8 @@ public class HealthCheckConfigurationPropertiesConfigurer
extends org.apache.cam
case "Interval": return target.getInterval();
case "parent":
case "Parent": return target.getParent();
+ case "successthreshold":
+ case "SuccessThreshold": return target.getSuccessThreshold();
default: return null;
}
}
diff --git
a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
index cf30b0f..b7e4782 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/BaseMainSupport.java
@@ -1005,6 +1005,7 @@ public abstract class BaseMainSupport extends BaseService
{
((HealthCheck)
hc).getConfiguration().setParent(hcc.getParent());
((HealthCheck)
hc).getConfiguration().setEnabled(hcc.isEnabled());
((HealthCheck)
hc).getConfiguration().setFailureThreshold(hcc.getFailureThreshold());
+ ((HealthCheck)
hc).getConfiguration().setSuccessThreshold(hcc.getSuccessThreshold());
((HealthCheck)
hc).getConfiguration().setInterval(hcc.getInterval());
} else if (hc instanceof HealthCheckRepository) {
((HealthCheckRepository) hc).addConfiguration(id, hcc);
diff --git
a/core/camel-management-api/src/main/java/org/apache/camel/api/management/mbean/CamelOpenMBeanTypes.java
b/core/camel-management-api/src/main/java/org/apache/camel/api/management/mbean/CamelOpenMBeanTypes.java
index 53a6620..8b18b24 100644
---
a/core/camel-management-api/src/main/java/org/apache/camel/api/management/mbean/CamelOpenMBeanTypes.java
+++
b/core/camel-management-api/src/main/java/org/apache/camel/api/management/mbean/CamelOpenMBeanTypes.java
@@ -237,15 +237,15 @@ public final class CamelOpenMBeanTypes {
new String[] {
"id", "group", "state", "enabled", "message",
"failureUri", "failureCount", "failureStackTrace",
"readiness", "liveness",
- "interval", "failureThreshold" },
+ "interval", "successThreshold", "failureThreshold" },
new String[] {
"ID", "Group", "State", "Enabled", "Message", "Failure
Uri", "Failure Count", "Failure StackTrace",
"Readiness", "Liveness",
- "Interval", "Failure Threshold" },
+ "Interval", "Success Threshold", "Failure Threshold" },
new OpenType[] {
SimpleType.STRING, SimpleType.STRING,
SimpleType.STRING, SimpleType.BOOLEAN, SimpleType.STRING,
SimpleType.STRING, SimpleType.INTEGER,
SimpleType.STRING, SimpleType.BOOLEAN, SimpleType.BOOLEAN,
- SimpleType.LONG, SimpleType.INTEGER });
+ SimpleType.LONG, SimpleType.INTEGER,
SimpleType.INTEGER });
}
public static TabularType camelHealthDetailsTabularType() throws
OpenDataException {
diff --git
a/core/camel-management/src/main/java/org/apache/camel/management/mbean/ManagedCamelHealth.java
b/core/camel-management/src/main/java/org/apache/camel/management/mbean/ManagedCamelHealth.java
index 4fa4361..6f198ff 100644
---
a/core/camel-management/src/main/java/org/apache/camel/management/mbean/ManagedCamelHealth.java
+++
b/core/camel-management/src/main/java/org/apache/camel/management/mbean/ManagedCamelHealth.java
@@ -134,6 +134,7 @@ public class ManagedCamelHealth implements
ManagedCamelHealthMBean {
"readiness",
"liveness",
"interval",
+ "successThreshold",
"failureThreshold"
},
new Object[] {
@@ -148,6 +149,7 @@ public class ManagedCamelHealth implements
ManagedCamelHealthMBean {
result.getCheck().isReadiness(),
result.getCheck().isLiveness(),
result.getCheck().getConfiguration().getInterval(),
+
result.getCheck().getConfiguration().getSuccessThreshold(),
result.getCheck().getConfiguration().getFailureThreshold()
});
diff --git a/docs/user-manual/modules/ROOT/pages/health-check.adoc
b/docs/user-manual/modules/ROOT/pages/health-check.adoc
index ba929c8..4b73510 100644
--- a/docs/user-manual/modules/ROOT/pages/health-check.adoc
+++ b/docs/user-manual/modules/ROOT/pages/health-check.adoc
@@ -56,15 +56,19 @@ The `HealthCheckConfiguration` has the following options:
| Name | Default | Description
| enabled | true | Set if the check associated to this configuration is
enabled or not.
| interval | | Set the check interval in milliseconds.
+| successThreshold | | Set the number of success before reporting the service
as healthy.
| failureThreshold | | Set the number of failure before reporting the service
as un-healthy.
|====
-The `interval` and `failureThreshold` are used for configuring health checks
to deal with flaky checks.
+The `interval`, `successThreshold`, and `failureThreshold` are used for
configuring health checks to deal with flaky checks.
For example assume you do not want to regard a check as immediately DOWN when
a check for the first time returns a response as DOWN.
So you can specify the `interval=10000` and `failureThreshold=5`, which means
that the check has slacks, and will
only report it as DOWN when there has been 5 failures in a row, with a minimum
of 10 second of interval between these checks.
So in other words after minimum 50 seconds and 5 calls then it can be reported
as DOWN.
+The `successThreshold` is used for number of successful checks in a row before
the check is regarded as UP.
+By default, this is not in use, and the first successful check will result in
UP.
+
== Configuring health-check
Camel supports via `camel-main` to configure health-checks from configuration
files. This is possible for runtimes that leverage `camel-main`
@@ -107,6 +111,7 @@ camel.health.config[kafka*].parent=routes
camel.health.config[kafka*].enabled=false
camel.health.config[*].parent=routes
camel.health.config[*].enabled=true
+camel.health.config[*].success-threshold=3
camel.health.config[*].failure-threshold=10
----