This is an automated email from the ASF dual-hosted git repository.
frankgh pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2eb3474 CASSANDRASC-86 Add Random Delays Between Retry Attempts for
Health Checks
2eb3474 is described below
commit 2eb3474d7037a2887bcd9dee1f64c2a36a7e8d26
Author: Yuriy Semchyshyn <[email protected]>
AuthorDate: Wed Nov 29 17:46:43 2023 -0600
CASSANDRASC-86 Add Random Delays Between Retry Attempts for Health Checks
Patch by Yuriy Semchyshyn; Reviewed by Yifan Cai and Francisco Guerrero for
CASSANDRASC-86
---
CHANGES.txt | 1 +
.../cassandra/sidecar/client/SidecarClient.java | 7 ++-
.../sidecar/client/SidecarClientConfig.java | 12 +++++
.../sidecar/client/SidecarClientConfigImpl.java | 50 +++++++++++++++++
.../client/retry/OncePerInstanceRetryPolicy.java | 35 +++++++++++-
.../cassandra/sidecar/common/utils/TimeUtils.java | 56 +++++++++++++++++++
.../sidecar/common/utils/TimeUtilsTest.java | 62 ++++++++++++++++++++++
7 files changed, 219 insertions(+), 4 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 3f1b4cc..27fba7e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
1.0.0
-----
+ * Startup Validation Failures when Checking Sidecar Connectivity
(CASSANDRASC-86)
* Add support for additional digest validation during SSTable upload
(CASSANDRASC-97)
* Add sidecar client changes for restore from S3 (CASSANDRASC-95)
* Add restore SSTables from S3 into Cassandra feature to Cassandra Sidecar
(CASSANDRASC-92)
diff --git
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java
index 42b7093..c89fc9a 100644
---
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java
+++
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java
@@ -69,6 +69,7 @@ public class SidecarClient implements AutoCloseable,
SidecarClientBlobRestoreExt
protected RequestExecutor executor;
protected final RetryPolicy defaultRetryPolicy;
protected final RetryPolicy ignoreConflictRetryPolicy;
+ protected final RetryPolicy oncePerInstanceRetryPolicy;
protected RequestContext.Builder baseBuilder;
public SidecarClient(SidecarInstancesProvider instancesProvider,
@@ -80,6 +81,8 @@ public class SidecarClient implements AutoCloseable,
SidecarClientBlobRestoreExt
ignoreConflictRetryPolicy = new
IgnoreConflictRetryPolicy(sidecarClientConfig.maxRetries(),
sidecarClientConfig.retryDelayMillis(),
sidecarClientConfig.maxRetryDelayMillis());
+ oncePerInstanceRetryPolicy = new
OncePerInstanceRetryPolicy(sidecarClientConfig.minimumHealthRetryDelay(),
+
sidecarClientConfig.maximumHealthRetryDelay());
baseBuilder = new RequestContext.Builder()
.instanceSelectionPolicy(new
RandomInstanceSelectionPolicy(instancesProvider))
.retryPolicy(defaultRetryPolicy);
@@ -95,7 +98,7 @@ public class SidecarClient implements AutoCloseable,
SidecarClientBlobRestoreExt
{
return executor.executeRequestAsync(requestBuilder()
.sidecarHealthRequest()
- .retryPolicy(new
OncePerInstanceRetryPolicy())
+
.retryPolicy(oncePerInstanceRetryPolicy)
.build());
}
@@ -110,7 +113,7 @@ public class SidecarClient implements AutoCloseable,
SidecarClientBlobRestoreExt
{
return executor.executeRequestAsync(requestBuilder()
.cassandraHealthRequest()
- .retryPolicy(new
OncePerInstanceRetryPolicy())
+
.retryPolicy(oncePerInstanceRetryPolicy)
.build());
}
diff --git
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfig.java
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfig.java
index fd138d1..a3d9fc9 100644
---
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfig.java
+++
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfig.java
@@ -19,6 +19,8 @@
package org.apache.cassandra.sidecar.client;
+import java.time.Duration;
+
/**
* Encapsulates configurations for the {@link SidecarClient}
*/
@@ -38,4 +40,14 @@ public interface SidecarClientConfig
* @return the maximum amount of time to wait before retrying a failed
request
*/
long maxRetryDelayMillis();
+
+ /**
+ * @return the minimum amount of time to wait before retrying a failed
health check
+ */
+ Duration minimumHealthRetryDelay();
+
+ /**
+ * @return the maximum amount of time to wait before retrying a failed
health check
+ */
+ Duration maximumHealthRetryDelay();
}
diff --git
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfigImpl.java
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfigImpl.java
index 3a3a4c9..5c4a6ce 100644
---
a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfigImpl.java
+++
b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClientConfigImpl.java
@@ -19,6 +19,8 @@
package org.apache.cassandra.sidecar.client;
+import java.time.Duration;
+
import org.apache.cassandra.sidecar.common.DataObjectBuilder;
/**
@@ -29,16 +31,22 @@ public class SidecarClientConfigImpl implements
SidecarClientConfig
public static final int DEFAULT_MAX_RETRIES = 3;
public static final long DEFAULT_RETRY_DELAY_MILLIS = 500L;
public static final long DEFAULT_MAX_RETRY_DELAY_MILLIS = 60_000L;
+ public static final Duration DEFAULT_MINIMUM_HEALTH_RETRY_DELAY =
Duration.ofSeconds(1L);
+ public static final Duration DEFAULT_MAXIMUM_HEALTH_RETRY_DELAY =
Duration.ofSeconds(5L);
protected final int maxRetries;
protected final long retryDelayMillis;
protected final long maxRetryDelayMillis;
+ protected final Duration minimumHealthRetryDelay;
+ protected final Duration maximumHealthRetryDelay;
private SidecarClientConfigImpl(Builder builder)
{
maxRetries = builder.maxRetries;
retryDelayMillis = builder.retryDelayMillis;
maxRetryDelayMillis = builder.maxRetryDelayMillis;
+ minimumHealthRetryDelay = builder.minimumHealthRetryDelay;
+ maximumHealthRetryDelay = builder.maximumHealthRetryDelay;
}
/**
@@ -68,6 +76,24 @@ public class SidecarClientConfigImpl implements
SidecarClientConfig
return maxRetryDelayMillis;
}
+ /**
+ * @return the minimum amount of time to wait before retrying a failed
health check
+ */
+ @Override
+ public Duration minimumHealthRetryDelay()
+ {
+ return minimumHealthRetryDelay;
+ }
+
+ /**
+ * @return the maximum amount of time to wait before retrying a failed
health check
+ */
+ @Override
+ public Duration maximumHealthRetryDelay()
+ {
+ return maximumHealthRetryDelay;
+ }
+
public static Builder builder()
{
return new Builder();
@@ -81,6 +107,8 @@ public class SidecarClientConfigImpl implements
SidecarClientConfig
protected int maxRetries = DEFAULT_MAX_RETRIES;
protected long retryDelayMillis = DEFAULT_RETRY_DELAY_MILLIS;
protected long maxRetryDelayMillis = DEFAULT_MAX_RETRY_DELAY_MILLIS;
+ protected Duration minimumHealthRetryDelay =
DEFAULT_MINIMUM_HEALTH_RETRY_DELAY;
+ protected Duration maximumHealthRetryDelay =
DEFAULT_MAXIMUM_HEALTH_RETRY_DELAY;
protected Builder()
{
@@ -125,6 +153,28 @@ public class SidecarClientConfigImpl implements
SidecarClientConfig
return update(b -> b.maxRetryDelayMillis = maxRetryDelayMillis);
}
+ /**
+ * Sets the {@code minimumHealthRetryDelay} and returns a reference to
this Builder enabling method chaining
+ *
+ * @param minimumHealthRetryDelay the {@code minimumHealthRetryDelay}
to set
+ * @return a reference to this Builder
+ */
+ public Builder minimumHealthRetryDelay(Duration
minimumHealthRetryDelay)
+ {
+ return update(builder -> builder.minimumHealthRetryDelay =
minimumHealthRetryDelay);
+ }
+
+ /**
+ * Sets the {@code maximumHealthRetryDelay} and returns a reference to
this Builder enabling method chaining
+ *
+ * @param maximumHealthRetryDelay the {@code maximumHealthRetryDelay}
to set
+ * @return a reference to this Builder
+ */
+ public Builder maximumHealthRetryDelay(Duration
maximumHealthRetryDelay)
+ {
+ return update(builder -> builder.maximumHealthRetryDelay =
maximumHealthRetryDelay);
+ }
+
/**
* Returns a {@code SidecarConfig} built from the parameters
previously set.
*
diff --git
a/client/src/main/java/org/apache/cassandra/sidecar/client/retry/OncePerInstanceRetryPolicy.java
b/client/src/main/java/org/apache/cassandra/sidecar/client/retry/OncePerInstanceRetryPolicy.java
index cb14429..7c229f4 100644
---
a/client/src/main/java/org/apache/cassandra/sidecar/client/retry/OncePerInstanceRetryPolicy.java
+++
b/client/src/main/java/org/apache/cassandra/sidecar/client/retry/OncePerInstanceRetryPolicy.java
@@ -18,19 +18,50 @@
package org.apache.cassandra.sidecar.client.retry;
+import java.time.Duration;
import java.util.concurrent.CompletableFuture;
import io.netty.handler.codec.http.HttpResponseStatus;
import org.apache.cassandra.sidecar.client.HttpResponse;
import org.apache.cassandra.sidecar.client.exception.RetriesExhaustedException;
import org.apache.cassandra.sidecar.client.request.Request;
+import org.apache.cassandra.sidecar.common.utils.TimeUtils;
/**
* A retry policy that attempts to execute the request once on each instance
- * until the first successful response is received, or fails if none were
successful
+ * until the first successful response is received, or fails if none were
successful.
+ *
+ * Accepts optional minimum and maximum durations used to calculate random
delay
+ * before each retry attempt in order to avoid the thundering herd problem.
+ *
+ * Retries immediately without delay if minimum and maximum durations are not
specified.
*/
public class OncePerInstanceRetryPolicy extends RetryPolicy
{
+ private final Duration minimumDelay;
+ private final Duration maximumDelay;
+
+ /**
+ * Instantiates {@link OncePerInstanceRetryPolicy} with no delay between
retry attempts
+ */
+ public OncePerInstanceRetryPolicy()
+ {
+ this(Duration.ZERO, Duration.ZERO);
+ }
+
+ /**
+ * Instantiates {@link OncePerInstanceRetryPolicy} with random delays
between retry attempts
+ *
+ * @param minimumDelay duration of minimum possible retry delay, inclusive
+ * @param maximumDelay duration of maximum possible retry delay, inclusive
+ */
+ public OncePerInstanceRetryPolicy(Duration minimumDelay, Duration
maximumDelay)
+ {
+ super();
+ this.minimumDelay = minimumDelay;
+ this.maximumDelay = maximumDelay;
+ }
+
/**
* {@inheritDoc}
*/
@@ -49,7 +80,7 @@ public class OncePerInstanceRetryPolicy extends RetryPolicy
}
else if (canRetryOnADifferentHost)
{
- retryAction.retry(attempts + 1, 0L);
+ retryAction.retry(attempts + 1,
TimeUtils.randomDuration(minimumDelay, maximumDelay).toMillis());
}
else
{
diff --git
a/common/src/main/java/org/apache/cassandra/sidecar/common/utils/TimeUtils.java
b/common/src/main/java/org/apache/cassandra/sidecar/common/utils/TimeUtils.java
new file mode 100644
index 0000000..a44f1f6
--- /dev/null
+++
b/common/src/main/java/org/apache/cassandra/sidecar/common/utils/TimeUtils.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cassandra.sidecar.common.utils;
+
+import java.time.Duration;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Utility class for manipulating dates, times, and durations
+ */
+public final class TimeUtils
+{
+ /**
+ * Private constructor that prevents unnecessary instantiation
+ *
+ * @throws IllegalStateException when called
+ */
+ private TimeUtils()
+ {
+ throw new IllegalStateException(getClass() + " is a static utility
class and shall not be instantiated");
+ }
+
+ /**
+ * Returns a random duration with millisecond precision that is uniformly
distributed between two provided durations
+ *
+ * @param minimum minimum possible duration, inclusive
+ * @param maximum maximum possible duration, inclusive
+ *
+ * @return random duration uniformly distributed between two provided
durations, with millisecond precision
+ *
+ * @throws IllegalArgumentException if minimum duration is greater than
maximum duration
+ */
+ public static Duration randomDuration(Duration minimum, Duration maximum)
+ {
+ Preconditions.checkArgument(minimum.compareTo(maximum) <= 0,
+ "Minimum duration must be less or equal to
maximum duration");
+ return
Duration.ofMillis(ThreadLocalRandom.current().nextLong(minimum.toMillis(),
maximum.toMillis() + 1L));
+ }
+}
diff --git
a/common/src/test/java/org/apache/cassandra/sidecar/common/utils/TimeUtilsTest.java
b/common/src/test/java/org/apache/cassandra/sidecar/common/utils/TimeUtilsTest.java
new file mode 100644
index 0000000..0f8d5bb
--- /dev/null
+++
b/common/src/test/java/org/apache/cassandra/sidecar/common/utils/TimeUtilsTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cassandra.sidecar.common.utils;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Unit tests for {@link TimeUtils} class
+ */
+public class TimeUtilsTest
+{
+ @Test
+ public void testRandomDurationWithMinimumAboveMaximum()
+ {
+ assertThrows(IllegalArgumentException.class,
+ () -> TimeUtils.randomDuration(Duration.ofSeconds(2L),
Duration.ofSeconds(1L)));
+ }
+
+ @Test
+ public void testRandomDurationWithMinimumEqualToMaximum()
+ {
+ assertEquals(Duration.ofSeconds(1L),
+ TimeUtils.randomDuration(Duration.ofSeconds(1L),
Duration.ofSeconds(1L)));
+ }
+
+ @Test
+ public void testRandomDurationWithMinimumBelowMaximum()
+ {
+ for (int test = 0; test < 3600; test++)
+ {
+ Duration minimum = Duration.ofSeconds(test);
+ Duration maximum = Duration.ofSeconds(test + 1);
+ Duration random = TimeUtils.randomDuration(minimum, maximum);
+
+ assertTrue(minimum.compareTo(random) <= 0);
+ assertTrue(random.compareTo(maximum) <= 0);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
