This is an automated email from the ASF dual-hosted git repository.
liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-java-chassis.git
The following commit(s) were added to refs/heads/master by this push:
new 25631cd1e [SCB-2743]edge service support retry governance (#4038)
25631cd1e is described below
commit 25631cd1e52f8e5e217617cbfdf0501d36fdb7ff
Author: liubao68 <[email protected]>
AuthorDate: Fri Nov 17 11:01:02 2023 +0800
[SCB-2743]edge service support retry governance (#4038)
---
.../core/filter/CoreFilterConfiguration.java | 7 +
.../servicecomb/core/filter/impl/RetryFilter.java | 135 ++++++++++++++
.../servicecomb/core/governance/RetryContext.java | 1 +
.../ServiceCombCircuitBreakerExtension.java | 13 +-
.../ServiceCombInstanceIsolationExtension.java | 8 +-
.../core/governance/ServiceCombRetryExtension.java | 13 +-
.../core/provider/consumer/InvokerUtils.java | 196 +--------------------
.../core/provider/consumer/TestInvokerUtils.java | 7 +-
.../demo/filter/client/RetryClientSchema.java | 25 +++
.../src/main/resources/microservice.yaml | 8 +
.../demo/filter/tests/TestRetrySchemaFromEdge.java | 13 ++
.../src/main/resources/microservice.yaml | 10 --
.../filterEdge/RetryClientSchema.yaml | 23 ++-
.../src/main/resources/microservice.yaml | 10 +-
.../governance/handler/CircuitBreakerHandler.java | 2 +-
.../handler/InstanceIsolationHandler.java | 2 +-
.../governance/handler/RetryHandler.java | 2 +-
.../governance/handler/ext/FailurePredictor.java | 4 +-
.../governance/AbstractFailurePredictorTest.java | 4 +-
.../servicecomb/governance/MockRetryExtension.java | 2 +-
.../servicecomb/governance/RetryHandlerTest.java | 6 +-
21 files changed, 250 insertions(+), 241 deletions(-)
diff --git
a/core/src/main/java/org/apache/servicecomb/core/filter/CoreFilterConfiguration.java
b/core/src/main/java/org/apache/servicecomb/core/filter/CoreFilterConfiguration.java
index 7359fdc55..d5a828a18 100644
---
a/core/src/main/java/org/apache/servicecomb/core/filter/CoreFilterConfiguration.java
+++
b/core/src/main/java/org/apache/servicecomb/core/filter/CoreFilterConfiguration.java
@@ -18,7 +18,9 @@ package org.apache.servicecomb.core.filter;
import org.apache.servicecomb.core.filter.impl.ParameterValidatorFilter;
import org.apache.servicecomb.core.filter.impl.ProviderOperationFilter;
+import org.apache.servicecomb.core.filter.impl.RetryFilter;
import org.apache.servicecomb.core.filter.impl.ScheduleFilter;
+import org.apache.servicecomb.governance.handler.RetryHandler;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -34,6 +36,11 @@ public class CoreFilterConfiguration {
return new ScheduleFilter();
}
+ @Bean
+ public RetryFilter retryFilter(RetryHandler retryHandler) {
+ return new RetryFilter(retryHandler);
+ }
+
@Bean
public FilterChainsManager filterChainsManager() {
return new FilterChainsManager();
diff --git
a/core/src/main/java/org/apache/servicecomb/core/filter/impl/RetryFilter.java
b/core/src/main/java/org/apache/servicecomb/core/filter/impl/RetryFilter.java
new file mode 100644
index 000000000..e423b8f7e
--- /dev/null
+++
b/core/src/main/java/org/apache/servicecomb/core/filter/impl/RetryFilter.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.core.filter.impl;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.core.filter.AbstractFilter;
+import org.apache.servicecomb.core.filter.ConsumerFilter;
+import org.apache.servicecomb.core.filter.EdgeFilter;
+import org.apache.servicecomb.core.filter.Filter;
+import org.apache.servicecomb.core.filter.FilterNode;
+import org.apache.servicecomb.core.governance.GovernanceConfiguration;
+import org.apache.servicecomb.core.governance.MatchType;
+import org.apache.servicecomb.core.governance.RetryContext;
+import org.apache.servicecomb.governance.handler.RetryHandler;
+import org.apache.servicecomb.governance.marker.GovernanceRequestExtractor;
+import org.apache.servicecomb.swagger.invocation.Response;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import io.github.resilience4j.decorators.Decorators;
+import io.github.resilience4j.decorators.Decorators.DecorateCompletionStage;
+import io.github.resilience4j.retry.Retry;
+
+public class RetryFilter extends AbstractFilter implements ConsumerFilter,
EdgeFilter {
+ private static final Object LOCK = new Object();
+
+ private static volatile ScheduledExecutorService reactiveRetryPool;
+
+ private static ScheduledExecutorService getOrCreateRetryPool() {
+ if (reactiveRetryPool == null) {
+ synchronized (LOCK) {
+ if (reactiveRetryPool == null) {
+ reactiveRetryPool = Executors.newScheduledThreadPool(2, new
ThreadFactory() {
+ private final AtomicInteger count = new AtomicInteger(0);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread thread = new Thread(r, "reactive-retry-pool-thread-" +
count.getAndIncrement());
+ // avoid block shutdown
+ thread.setDaemon(true);
+ return thread;
+ }
+ });
+ }
+ }
+ }
+ return reactiveRetryPool;
+ }
+
+ private final RetryHandler retryHandler;
+
+ @Autowired
+ public RetryFilter(RetryHandler retryHandler) {
+ this.retryHandler = retryHandler;
+ }
+
+ @Override
+ public String getName() {
+ return "retry";
+ }
+
+ @Override
+ public CompletableFuture<Response> onFilter(Invocation invocation,
FilterNode nextNode) {
+ GovernanceRequestExtractor request =
MatchType.createGovHttpRequest(invocation);
+ Retry retry = retryHandler.getActuator(request);
+ if (retry == null) {
+ return nextNode.onFilter(invocation);
+ }
+
+ Supplier<CompletionStage<Response>> next =
createBusinessCompletionStageSupplier(invocation, nextNode);
+ DecorateCompletionStage<Response> dcs = Decorators.ofCompletionStage(next);
+ dcs.withRetry(retry, getOrCreateRetryPool());
+ CompletableFuture<Response> future = new CompletableFuture<>();
+ dcs.get().whenComplete((r, e) -> {
+ if (e == null) {
+ future.complete(r);
+ return;
+ }
+
+ future.completeExceptionally(e);
+ });
+
+ return future;
+ }
+
+ private Supplier<CompletionStage<Response>>
createBusinessCompletionStageSupplier(Invocation invocation,
+ FilterNode nextNode) {
+ return () -> {
+ updateRetryStatus(invocation);
+ return nextNode.onFilter(invocation);
+ };
+ }
+
+ private static void updateRetryStatus(Invocation invocation) {
+ if (invocation.getLocalContext(RetryContext.RETRY_CONTEXT) != null) {
+ if (invocation.getLocalContext(RetryContext.RETRY_LOAD_BALANCE) != null
+ && (boolean)
invocation.getLocalContext(RetryContext.RETRY_LOAD_BALANCE)) {
+ // clear last server to avoid using user defined endpoint
+ invocation.setEndpoint(null);
+ }
+ RetryContext retryContext =
invocation.getLocalContext(RetryContext.RETRY_CONTEXT);
+ retryContext.incrementRetry();
+ return;
+ }
+
+ invocation.addLocalContext(RetryContext.RETRY_CONTEXT,
+ new
RetryContext(GovernanceConfiguration.getRetrySameServer(invocation.getMicroserviceName())));
+ }
+
+ @Override
+ public int getOrder() {
+ return Filter.CONSUMER_LOAD_BALANCE_ORDER - 1990;
+ }
+}
diff --git
a/core/src/main/java/org/apache/servicecomb/core/governance/RetryContext.java
b/core/src/main/java/org/apache/servicecomb/core/governance/RetryContext.java
index 058982a42..47792792e 100644
---
a/core/src/main/java/org/apache/servicecomb/core/governance/RetryContext.java
+++
b/core/src/main/java/org/apache/servicecomb/core/governance/RetryContext.java
@@ -19,6 +19,7 @@ package org.apache.servicecomb.core.governance;
public class RetryContext {
public static final String RETRY_CONTEXT = "x-context-retry";
+ // weather need reset Endpoint in retry
public static final String RETRY_LOAD_BALANCE =
"x-context-retry-loadbalance";
private boolean retry;
diff --git
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombCircuitBreakerExtension.java
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombCircuitBreakerExtension.java
index cb85ecfe1..9177ca718 100644
---
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombCircuitBreakerExtension.java
+++
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombCircuitBreakerExtension.java
@@ -17,11 +17,10 @@
package org.apache.servicecomb.core.governance;
-import jakarta.ws.rs.core.Response.Status;
+import java.util.List;
import
org.apache.servicecomb.governance.handler.ext.AbstractCircuitBreakerExtension;
import org.apache.servicecomb.swagger.invocation.Response;
-import org.apache.servicecomb.swagger.invocation.exception.ExceptionFactory;
import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
public class ServiceCombCircuitBreakerExtension extends
AbstractCircuitBreakerExtension {
@@ -41,15 +40,11 @@ public class ServiceCombCircuitBreakerExtension extends
AbstractCircuitBreakerEx
}
@Override
- public boolean isFailedResult(Throwable e) {
+ public boolean isFailedResult(List<String> statusList, Throwable e) {
if (e instanceof InvocationException) {
InvocationException invocationException = (InvocationException) e;
- if (invocationException.getStatusCode() ==
Status.SERVICE_UNAVAILABLE.getStatusCode() ||
- invocationException.getStatusCode() ==
Status.BAD_GATEWAY.getStatusCode() ||
- invocationException.getStatusCode() ==
ExceptionFactory.PRODUCER_INNER_STATUS_CODE) {
- return true;
- }
+ return
statusList.contains(String.valueOf(invocationException.getStatusCode()));
}
- return super.isFailedResult(e);
+ return super.isFailedResult(statusList, e);
}
}
diff --git
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombInstanceIsolationExtension.java
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombInstanceIsolationExtension.java
index f3b9075ea..2bbf6d3c8 100644
---
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombInstanceIsolationExtension.java
+++
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombInstanceIsolationExtension.java
@@ -17,10 +17,12 @@
package org.apache.servicecomb.core.governance;
+import java.util.List;
+
import
org.apache.servicecomb.governance.handler.ext.AbstractInstanceIsolationExtension;
public class ServiceCombInstanceIsolationExtension extends
AbstractInstanceIsolationExtension {
- private ServiceCombRetryExtension retryExtension = new
ServiceCombRetryExtension();
+ private final ServiceCombRetryExtension retryExtension = new
ServiceCombRetryExtension();
@Override
protected String extractStatusCode(Object result) {
@@ -28,7 +30,7 @@ public class ServiceCombInstanceIsolationExtension extends
AbstractInstanceIsola
}
@Override
- public boolean isFailedResult(Throwable e) {
- return retryExtension.isFailedResult(e);
+ public boolean isFailedResult(List<String> statusList, Throwable e) {
+ return retryExtension.isFailedResult(statusList, e);
}
}
diff --git
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombRetryExtension.java
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombRetryExtension.java
index ccd7f6f84..871a03bfa 100644
---
a/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombRetryExtension.java
+++
b/core/src/main/java/org/apache/servicecomb/core/governance/ServiceCombRetryExtension.java
@@ -17,11 +17,10 @@
package org.apache.servicecomb.core.governance;
-import jakarta.ws.rs.core.Response.Status;
+import java.util.List;
import org.apache.servicecomb.governance.handler.ext.AbstractRetryExtension;
import org.apache.servicecomb.swagger.invocation.Response;
-import org.apache.servicecomb.swagger.invocation.exception.ExceptionFactory;
import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
public class ServiceCombRetryExtension extends AbstractRetryExtension {
@@ -41,15 +40,11 @@ public class ServiceCombRetryExtension extends
AbstractRetryExtension {
}
@Override
- public boolean isFailedResult(Throwable e) {
+ public boolean isFailedResult(List<String> statusList, Throwable e) {
if (e instanceof InvocationException) {
InvocationException invocationException = (InvocationException) e;
- if (invocationException.getStatusCode() ==
Status.SERVICE_UNAVAILABLE.getStatusCode() ||
- invocationException.getStatusCode() ==
Status.BAD_GATEWAY.getStatusCode() ||
- invocationException.getStatusCode() ==
ExceptionFactory.CONSUMER_INNER_STATUS_CODE) {
- return true;
- }
+ return
statusList.contains(String.valueOf(invocationException.getStatusCode()));
}
- return super.isFailedResult(e);
+ return super.isFailedResult(statusList, e);
}
}
diff --git
a/core/src/main/java/org/apache/servicecomb/core/provider/consumer/InvokerUtils.java
b/core/src/main/java/org/apache/servicecomb/core/provider/consumer/InvokerUtils.java
index 5c2a012ae..d850cf6b6 100644
---
a/core/src/main/java/org/apache/servicecomb/core/provider/consumer/InvokerUtils.java
+++
b/core/src/main/java/org/apache/servicecomb/core/provider/consumer/InvokerUtils.java
@@ -18,22 +18,14 @@
package org.apache.servicecomb.core.provider.consumer;
import static
org.apache.servicecomb.core.exception.Exceptions.toConsumerResponse;
-import static
org.apache.servicecomb.swagger.invocation.exception.ExceptionFactory.CONSUMER_INNER_STATUS_CODE;
import java.lang.reflect.Method;
import java.lang.reflect.Type;
-import java.time.Duration;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Supplier;
import org.apache.servicecomb.core.Invocation;
import org.apache.servicecomb.core.SCBEngine;
@@ -41,62 +33,19 @@ import
org.apache.servicecomb.core.definition.InvocationRuntimeType;
import org.apache.servicecomb.core.definition.MicroserviceMeta;
import org.apache.servicecomb.core.definition.OperationMeta;
import org.apache.servicecomb.core.definition.SchemaMeta;
-import org.apache.servicecomb.core.governance.GovernanceConfiguration;
-import org.apache.servicecomb.core.governance.MatchType;
-import org.apache.servicecomb.core.governance.RetryContext;
import org.apache.servicecomb.core.invocation.InvocationFactory;
import org.apache.servicecomb.foundation.common.utils.AsyncUtils;
-import org.apache.servicecomb.foundation.common.utils.BeanUtils;
-import org.apache.servicecomb.governance.handler.RetryHandler;
-import org.apache.servicecomb.governance.handler.ext.FailurePredictor;
-import org.apache.servicecomb.governance.marker.GovernanceRequestExtractor;
import org.apache.servicecomb.swagger.invocation.AsyncResponse;
import org.apache.servicecomb.swagger.invocation.Response;
import org.apache.servicecomb.swagger.invocation.context.ContextUtils;
import org.apache.servicecomb.swagger.invocation.exception.CommonExceptionData;
import org.apache.servicecomb.swagger.invocation.exception.ExceptionFactory;
import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
-import io.github.resilience4j.decorators.Decorators;
-import io.github.resilience4j.decorators.Decorators.DecorateCompletionStage;
-import io.github.resilience4j.retry.Retry;
-import io.github.resilience4j.retry.RetryConfig;
-import io.github.resilience4j.retry.RetryRegistry;
import io.vertx.core.Context;
import jakarta.ws.rs.core.Response.Status;
public final class InvokerUtils {
- private static final Logger LOGGER =
LoggerFactory.getLogger(InvokerUtils.class);
-
- private static final Object LOCK = new Object();
-
- private static volatile ScheduledExecutorService reactiveRetryPool;
-
- private static ScheduledExecutorService getOrCreateRetryPool() {
- if (reactiveRetryPool == null) {
- synchronized (LOCK) {
- if (reactiveRetryPool == null) {
- reactiveRetryPool = Executors.newScheduledThreadPool(2, new
ThreadFactory() {
- private final AtomicInteger count = new AtomicInteger(0);
-
- @Override
- public Thread newThread(Runnable r) {
- Thread thread = new Thread(r, "reactive-retry-pool-thread-" +
count.getAndIncrement());
- // avoid block shutdown
- thread.setDaemon(true);
- return thread;
- }
- });
- }
- }
- }
- return reactiveRetryPool;
- }
-
@SuppressWarnings({"unchecked"})
public static <T> T syncInvoke(String microserviceName, String transport,
String schemaId, String operationId, Map<String, Object>
swaggerArguments, Type responseType) {
@@ -191,43 +140,6 @@ public final class InvokerUtils {
return toSync(invoke(invocation), invocation.getWaitTime());
}
- private static void updateRetryStatus(Invocation invocation) {
- if (invocation.isFinished()) {
- invocation.reset();
- if (invocation.getLocalContext(RetryContext.RETRY_LOAD_BALANCE) != null
- && (boolean)
invocation.getLocalContext(RetryContext.RETRY_LOAD_BALANCE)) {
- // clear last server to avoid using user defined endpoint
- invocation.setEndpoint(null);
- }
- RetryContext retryContext =
invocation.getLocalContext(RetryContext.RETRY_CONTEXT);
- retryContext.incrementRetry();
- return;
- }
-
- invocation.addLocalContext(RetryContext.RETRY_CONTEXT,
- new
RetryContext(GovernanceConfiguration.getRetrySameServer(invocation.getMicroserviceName())));
- }
-
- private static boolean isCompatibleRetryEnabled(Invocation invocation) {
- // maxAttempts must be greater than or equal to 1
- return
GovernanceConfiguration.isRetryEnabled(invocation.getMicroserviceName())
- &&
GovernanceConfiguration.getRetryNextServer(invocation.getMicroserviceName())
- +
GovernanceConfiguration.getRetrySameServer(invocation.getMicroserviceName()) >
0;
- }
-
- private static Retry getOrCreateCompatibleRetry(Invocation invocation) {
- RetryConfig retryConfig = RetryConfig.custom()
- // max attempts include the first call
-
.maxAttempts(GovernanceConfiguration.getRetryNextServer(invocation.getMicroserviceName())
- +
GovernanceConfiguration.getRetrySameServer(invocation.getMicroserviceName()) +
1)
- .retryOnResult(InvokerUtils::canRetryForStatusCode)
- .retryOnException(InvokerUtils::canRetryForException)
- .waitDuration(Duration.ofMillis(1))
- .build();
- RetryRegistry retryRegistry = RetryRegistry.of(retryConfig);
- return retryRegistry.retry(invocation.getMicroserviceName());
- }
-
/**
* This is an internal API, caller make sure already invoked
SCBEngine.ensureStatusUp
*/
@@ -241,22 +153,6 @@ public final class InvokerUtils {
});
}
- private static void decorateReactiveRetry(Invocation invocation,
DecorateCompletionStage<Response> dcs,
- GovernanceRequestExtractor request) {
- // governance implementations.
- RetryHandler retryHandler = BeanUtils.getBean(RetryHandler.class);
- Retry retry = retryHandler.getActuator(request);
- if (retry != null) {
- dcs.withRetry(retry, getOrCreateRetryPool());
- }
-
- if (isCompatibleRetryEnabled(invocation)) {
- // compatible implementation for retry in load balance module in old
versions.
- retry = getOrCreateCompatibleRetry(invocation);
- dcs.withRetry(retry, getOrCreateRetryPool());
- }
- }
-
public static boolean isSyncMethod(Method method) {
return !isAsyncMethod(method);
}
@@ -287,89 +183,13 @@ public final class InvokerUtils {
* NOTE: this method should never throw exception directly
*/
public static CompletableFuture<Response> invoke(Invocation invocation) {
- Supplier<CompletionStage<Response>> next = invokeImpl(invocation);
- DecorateCompletionStage<Response> dcs = Decorators.ofCompletionStage(next);
- GovernanceRequestExtractor request =
MatchType.createGovHttpRequest(invocation);
-
- decorateReactiveRetry(invocation, dcs, request);
-
- CompletableFuture<Response> result = new CompletableFuture<>();
- dcs.get().whenComplete((r, e) -> {
- ContextUtils.setInvocationContext(invocation.getParentContext());
-
- if (e == null) {
- result.complete(r);
- return;
- }
-
- String message = String.format("invoke failed, operation %s, trace id
%s",
- invocation.getMicroserviceQualifiedName(),
- invocation.getTraceId());
- LOGGER.error(message, e);
- Response response = Response.createConsumerFail(e, message);
- invocation.onFinish(response);
- result.complete(response);
- });
- return result;
- }
-
- private static Supplier<CompletionStage<Response>> invokeImpl(Invocation
invocation) {
- return () -> {
- invocation.onStart(null);
- updateRetryStatus(invocation);
- if (invocation.isEdge()) {
- return invocation.getMicroserviceMeta().getEdgeFilterChain()
- .onFilter(invocation)
- .exceptionally(throwable -> toConsumerResponse(invocation,
throwable))
- .whenComplete((response, throwable) ->
finishInvocation(invocation, response));
- }
- return invocation.getMicroserviceMeta().getConsumerFilterChain()
- .onFilter(invocation)
- .exceptionally(throwable -> toConsumerResponse(invocation,
throwable))
- .whenComplete((response, throwable) -> finishInvocation(invocation,
response));
- };
- }
-
- private static void finishInvocation(Invocation invocation, Response ar) {
- invocation.onFinish(ar);
-
- if (ar.isFailed()) {
- // re-throw exception to make sure retry based on exception
- // for InvocationException, users can configure status code for retry
- // for 490, details error are wrapped, need re-throw
-
- if (!(ar.getResult() instanceof InvocationException)) {
- throw AsyncUtils.rethrow(ar.getResult());
- }
-
- if (((InvocationException) ar.getResult()).getStatusCode() ==
CONSUMER_INNER_STATUS_CODE) {
- throw AsyncUtils.rethrow(ar.getResult());
- }
- }
- }
-
- @VisibleForTesting
- static boolean canRetryForException(Throwable e) {
- if (e instanceof InvocationException && ((InvocationException)
e).getStatusCode() == Status.SERVICE_UNAVAILABLE
- .getStatusCode()) {
- return true;
- }
- return
FailurePredictor.canRetryForException(FailurePredictor.STRICT_RETRIABLE, e);
- }
-
- @VisibleForTesting
- static boolean canRetryForStatusCode(Object response) {
- // retry on status code 503
- if (!(response instanceof Response resp)) {
- return false;
- }
- if (!resp.isFailed()) {
- return false;
- }
- if (resp.getResult() instanceof InvocationException) {
- InvocationException e = resp.getResult();
- return e.getStatusCode() == 503;
- }
- return false;
+ invocation.onStart(null);
+ return invocation.getMicroserviceMeta().getConsumerFilterChain()
+ .onFilter(invocation)
+ .exceptionally(throwable -> toConsumerResponse(invocation, throwable))
+ .whenComplete((response, throwable) -> {
+ ContextUtils.setInvocationContext(invocation.getParentContext());
+ invocation.onFinish(response);
+ });
}
}
diff --git
a/core/src/test/java/org/apache/servicecomb/core/provider/consumer/TestInvokerUtils.java
b/core/src/test/java/org/apache/servicecomb/core/provider/consumer/TestInvokerUtils.java
index d526219c5..3b3e56092 100644
---
a/core/src/test/java/org/apache/servicecomb/core/provider/consumer/TestInvokerUtils.java
+++
b/core/src/test/java/org/apache/servicecomb/core/provider/consumer/TestInvokerUtils.java
@@ -17,16 +17,11 @@
package org.apache.servicecomb.core.provider.consumer;
-import org.apache.servicecomb.swagger.invocation.Response;
-import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class TestInvokerUtils {
@Test
public void testRetryInvocation503() {
- InvocationException root = new InvocationException(503, "Service
Unavailable", "Error");
- boolean canRetry =
InvokerUtils.canRetryForStatusCode(Response.failResp(root));
- Assertions.assertTrue(canRetry);
+
}
}
diff --git
a/demo/demo-filter/filter-client/src/main/java/org/apache/servicecomb/demo/filter/client/RetryClientSchema.java
b/demo/demo-filter/filter-client/src/main/java/org/apache/servicecomb/demo/filter/client/RetryClientSchema.java
index e101db5a8..b5951557a 100644
---
a/demo/demo-filter/filter-client/src/main/java/org/apache/servicecomb/demo/filter/client/RetryClientSchema.java
+++
b/demo/demo-filter/filter-client/src/main/java/org/apache/servicecomb/demo/filter/client/RetryClientSchema.java
@@ -17,15 +17,19 @@
package org.apache.servicecomb.demo.filter.client;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.servicecomb.provider.pojo.RpcReference;
import org.apache.servicecomb.provider.rest.common.RestSchema;
import org.apache.servicecomb.provider.springmvc.reference.RestTemplateBuilder;
+import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.client.RestOperations;
+import jakarta.ws.rs.core.Response.Status;
+
@RestSchema(schemaId = "RetryClientSchema")
@RequestMapping(path = "/retry", produces = MediaType.APPLICATION_JSON_VALUE)
public class RetryClientSchema {
@@ -38,6 +42,27 @@ public class RetryClientSchema {
RestOperations restTemplate = RestTemplateBuilder.create();
+ private AtomicLong counter = new AtomicLong(0);
+
+ @GetMapping(path = "/governance/edgeSuccessWhenRetry")
+ public boolean edgeSuccessWhenRetry() {
+ if (counter.getAndIncrement() % 3 != 0) {
+ throw new InvocationException(Status.INTERNAL_SERVER_ERROR, "try again
later.");
+ }
+ return true;
+ }
+
+ @GetMapping(path = "/governance/edgeSuccessWhenRetryAsync")
+ public CompletableFuture<Boolean> edgeSuccessWhenRetryAsync() {
+ CompletableFuture<Boolean> result = new CompletableFuture<>();
+ if (counter.getAndIncrement() % 2 == 0) {
+ result.completeExceptionally(new
InvocationException(Status.INTERNAL_SERVER_ERROR, "try again later."));
+ } else {
+ result.complete(true);
+ }
+ return result;
+ }
+
@GetMapping(path = "/governance/successWhenRetry")
public boolean successWhenRetry() {
return
restTemplate.getForObject("servicecomb://filterServer/retry/governance/successWhenRetry",
diff --git a/demo/demo-filter/filter-edge/src/main/resources/microservice.yaml
b/demo/demo-filter/filter-edge/src/main/resources/microservice.yaml
index b70747606..034b8cd04 100644
--- a/demo/demo-filter/filter-edge/src/main/resources/microservice.yaml
+++ b/demo/demo-filter/filter-edge/src/main/resources/microservice.yaml
@@ -58,8 +58,16 @@ servicecomb:
matches:
- apiPath:
exact: "/govern/edgeFlowControl"
+ retry-governance: |
+ matches:
+ - apiPath:
+ prefix: "/retry/governance/"
rateLimiting:
edgeFlowControl: |
timeoutDuration: 0
limitRefreshPeriod: 1000
rate: 1
+ retry:
+ retry-governance: |
+ maxAttempts: 2
+ retryOnResponseStatus: [500]
diff --git
a/demo/demo-filter/filter-tests/src/main/java/org/apache/servicecomb/demo/filter/tests/TestRetrySchemaFromEdge.java
b/demo/demo-filter/filter-tests/src/main/java/org/apache/servicecomb/demo/filter/tests/TestRetrySchemaFromEdge.java
index 422297296..d637a8b2a 100644
---
a/demo/demo-filter/filter-tests/src/main/java/org/apache/servicecomb/demo/filter/tests/TestRetrySchemaFromEdge.java
+++
b/demo/demo-filter/filter-tests/src/main/java/org/apache/servicecomb/demo/filter/tests/TestRetrySchemaFromEdge.java
@@ -29,8 +29,12 @@ import org.springframework.web.client.RestTemplate;
@Component
public class TestRetrySchemaFromEdge implements CategorizedTestCase {
interface RetrySchemaInf {
+ boolean edgeSuccessWhenRetry();
+
boolean successWhenRetry();
+ CompletableFuture<Boolean> edgeSuccessWhenRetryAsync();
+
CompletableFuture<Boolean> successWhenRetryAsync();
}
@@ -55,6 +59,7 @@ public class TestRetrySchemaFromEdge implements
CategorizedTestCase {
testRetryGovernanceFromEdgeDefaultDispatcher();
testRetryGovernanceRestTemplate();
testRetryGovernanceRpc();
+ testEdgeRetryGovernanceRpc();
}
private void testRetryGovernanceRpc() throws Exception {
@@ -65,6 +70,14 @@ public class TestRetrySchemaFromEdge implements
CategorizedTestCase {
TestMgr.check(retrySchemaInf.successWhenRetryAsync().get(), true);
}
+ private void testEdgeRetryGovernanceRpc() throws Exception {
+ TestMgr.check(retrySchemaInf.edgeSuccessWhenRetry(), true);
+ TestMgr.check(retrySchemaInf.edgeSuccessWhenRetry(), true);
+
+ TestMgr.check(retrySchemaInf.edgeSuccessWhenRetryAsync().get(), true);
+ TestMgr.check(retrySchemaInf.edgeSuccessWhenRetryAsync().get(), true);
+ }
+
private void testRetryGovernanceRestTemplate() {
TestMgr.check(restTemplate.getForObject(
SERVER + "/retry/governance/successWhenRetry", boolean.class), true);
diff --git a/demo/demo-filter/filter-tests/src/main/resources/microservice.yaml
b/demo/demo-filter/filter-tests/src/main/resources/microservice.yaml
index 75a286db4..c6b2ea0fd 100644
--- a/demo/demo-filter/filter-tests/src/main/resources/microservice.yaml
+++ b/demo/demo-filter/filter-tests/src/main/resources/microservice.yaml
@@ -28,13 +28,3 @@ servicecomb:
invocation:
exception:
print-stack-trace: true
- # test governance retry
- matchGroup:
- retry-governance: |
- matches:
- - apiPath:
- prefix: "/retry/governance/"
- retry:
- retry-governance: |
- maxAttempts: 2
- retryOnResponseStatus: [500]
diff --git
a/demo/demo-filter/filter-tests/src/main/resources/microservices/filterEdge/RetryClientSchema.yaml
b/demo/demo-filter/filter-tests/src/main/resources/microservices/filterEdge/RetryClientSchema.yaml
index 3b8376002..77517becf 100644
---
a/demo/demo-filter/filter-tests/src/main/resources/microservices/filterEdge/RetryClientSchema.yaml
+++
b/demo/demo-filter/filter-tests/src/main/resources/microservices/filterEdge/RetryClientSchema.yaml
@@ -22,6 +22,26 @@ info:
servers:
- url: /retry
paths:
+ /governance/edgeSuccessWhenRetry:
+ get:
+ operationId: edgeSuccessWhenRetry
+ responses:
+ "200":
+ description: response of 200
+ content:
+ application/json:
+ schema:
+ type: boolean
+ /governance/edgeSuccessWhenRetryAsync:
+ get:
+ operationId: edgeSuccessWhenRetryAsync
+ responses:
+ "200":
+ description: response of 200
+ content:
+ application/json:
+ schema:
+ type: boolean
/governance/successWhenRetry:
get:
operationId: successWhenRetry
@@ -42,5 +62,4 @@ paths:
application/json:
schema:
type: boolean
-components:
- schemas: {}
\ No newline at end of file
+components: {}
diff --git
a/demo/demo-springmvc/springmvc-client/src/main/resources/microservice.yaml
b/demo/demo-springmvc/springmvc-client/src/main/resources/microservice.yaml
index eec6b301c..2fe0f881f 100644
--- a/demo/demo-springmvc/springmvc-client/src/main/resources/microservice.yaml
+++ b/demo/demo-springmvc/springmvc-client/src/main/resources/microservice.yaml
@@ -54,9 +54,6 @@ servicecomb:
userDefinedEndpoint.enabled: true
strategy:
name: WeightedResponse
- retryEnabled: true
- retryOnSame: 1
- retryOnNext: 1
filter.status.enabled: false
fallbackpolicy:
Consumer:
@@ -113,10 +110,17 @@ servicecomb:
matches:
- apiPath:
prefix: "/retry/governance/"
+ retry-success: |
+ matches:
+ - apiPath:
+ prefix: "/codeFirstSpringmvc/retrySuccess"
retry:
retry-governance: |
maxAttempts: 2
retryOnResponseStatus: [500]
+ retry-success: |
+ maxAttempts: 2
+ retryOnResponseStatus: [503]
#########SSL options
# open jdk 8 now TLSv1.3 not available
diff --git
a/governance/src/main/java/org/apache/servicecomb/governance/handler/CircuitBreakerHandler.java
b/governance/src/main/java/org/apache/servicecomb/governance/handler/CircuitBreakerHandler.java
index fdeb10a5a..39082661b 100644
---
a/governance/src/main/java/org/apache/servicecomb/governance/handler/CircuitBreakerHandler.java
+++
b/governance/src/main/java/org/apache/servicecomb/governance/handler/CircuitBreakerHandler.java
@@ -72,7 +72,7 @@ public class CircuitBreakerHandler extends
AbstractGovernanceHandler<CircuitBrea
.minimumNumberOfCalls(policy.getMinimumNumberOfCalls())
.slidingWindowType(policy.getSlidingWindowTypeEnum())
.slidingWindowSize(Integer.parseInt(policy.getSlidingWindowSize()))
- .recordException(circuitBreakerExtension::isFailedResult)
+ .recordException(e ->
circuitBreakerExtension.isFailedResult(policy.getRecordFailureStatus(), e))
.recordResult(r ->
circuitBreakerExtension.isFailedResult(policy.getRecordFailureStatus(), r))
.build();
CircuitBreakerRegistry circuitBreakerRegistry =
CircuitBreakerRegistry.of(circuitBreakerConfig);
diff --git
a/governance/src/main/java/org/apache/servicecomb/governance/handler/InstanceIsolationHandler.java
b/governance/src/main/java/org/apache/servicecomb/governance/handler/InstanceIsolationHandler.java
index 5be1978dd..b37967b5a 100644
---
a/governance/src/main/java/org/apache/servicecomb/governance/handler/InstanceIsolationHandler.java
+++
b/governance/src/main/java/org/apache/servicecomb/governance/handler/InstanceIsolationHandler.java
@@ -103,7 +103,7 @@ public class InstanceIsolationHandler extends
AbstractGovernanceHandler<CircuitB
.minimumNumberOfCalls(policy.getMinimumNumberOfCalls())
.slidingWindowType(policy.getSlidingWindowTypeEnum())
.slidingWindowSize(Integer.parseInt(policy.getSlidingWindowSize()))
- .recordException(isolationExtension::isFailedResult)
+ .recordException(e ->
isolationExtension.isFailedResult(policy.getRecordFailureStatus(), e))
.recordResult(r ->
isolationExtension.isFailedResult(policy.getRecordFailureStatus(), r))
.build();
CircuitBreakerRegistry circuitBreakerRegistry =
CircuitBreakerRegistry.of(circuitBreakerConfig);
diff --git
a/governance/src/main/java/org/apache/servicecomb/governance/handler/RetryHandler.java
b/governance/src/main/java/org/apache/servicecomb/governance/handler/RetryHandler.java
index 8b4c819b0..f7f2476f6 100644
---
a/governance/src/main/java/org/apache/servicecomb/governance/handler/RetryHandler.java
+++
b/governance/src/main/java/org/apache/servicecomb/governance/handler/RetryHandler.java
@@ -68,7 +68,7 @@ public class RetryHandler extends
AbstractGovernanceHandler<Retry, RetryPolicy>
RetryConfig config = RetryConfig.custom()
.maxAttempts(retryPolicy.getMaxAttempts() + 1)
.retryOnResult(response ->
retryExtension.isFailedResult(retryPolicy.getRetryOnResponseStatus(), response))
- .retryOnException(retryExtension::isFailedResult)
+ .retryOnException(exception ->
retryExtension.isFailedResult(retryPolicy.getRetryOnResponseStatus(),
exception))
.intervalFunction(getIntervalFunction(retryPolicy))
.failAfterMaxAttempts(retryPolicy.isFailAfterMaxAttempts())
.build();
diff --git
a/governance/src/main/java/org/apache/servicecomb/governance/handler/ext/FailurePredictor.java
b/governance/src/main/java/org/apache/servicecomb/governance/handler/ext/FailurePredictor.java
index 31ae8eca1..c07ebe39b 100644
---
a/governance/src/main/java/org/apache/servicecomb/governance/handler/ext/FailurePredictor.java
+++
b/governance/src/main/java/org/apache/servicecomb/governance/handler/ext/FailurePredictor.java
@@ -29,8 +29,8 @@ import javax.net.ssl.SSLHandshakeException;
import com.google.common.collect.ImmutableMap;
-import io.vertx.core.VertxException;
import io.netty.handler.ssl.SslHandshakeTimeoutException;
+import io.vertx.core.VertxException;
public interface FailurePredictor {
Map<Class<? extends Throwable>, List<String>> STRICT_RETRIABLE =
@@ -49,7 +49,7 @@ public interface FailurePredictor {
boolean isFailedResult(List<String> statusList, Object result);
- default boolean isFailedResult(Throwable e) {
+ default boolean isFailedResult(List<String> statusList, Throwable e) {
return canRetryForException(STRICT_RETRIABLE, e);
}
diff --git
a/governance/src/test/java/org/apache/servicecomb/governance/AbstractFailurePredictorTest.java
b/governance/src/test/java/org/apache/servicecomb/governance/AbstractFailurePredictorTest.java
index 9c84287ac..dedeba800 100644
---
a/governance/src/test/java/org/apache/servicecomb/governance/AbstractFailurePredictorTest.java
+++
b/governance/src/test/java/org/apache/servicecomb/governance/AbstractFailurePredictorTest.java
@@ -35,8 +35,8 @@ public class AbstractFailurePredictorTest {
}
@Override
- public boolean isFailedResult(Throwable e) {
- return super.isFailedResult(e);
+ public boolean isFailedResult(List<String> statusList, Throwable e) {
+ return super.isFailedResult(statusList, e);
}
}
diff --git
a/governance/src/test/java/org/apache/servicecomb/governance/MockRetryExtension.java
b/governance/src/test/java/org/apache/servicecomb/governance/MockRetryExtension.java
index 1dd272ca1..2badc9c15 100644
---
a/governance/src/test/java/org/apache/servicecomb/governance/MockRetryExtension.java
+++
b/governance/src/test/java/org/apache/servicecomb/governance/MockRetryExtension.java
@@ -34,7 +34,7 @@ public class MockRetryExtension extends
AbstractRetryExtension {
}
@Override
- public boolean isFailedResult(Throwable e) {
+ public boolean isFailedResult(List<String> statusList, Throwable e) {
return false;
}
}
diff --git
a/governance/src/test/java/org/apache/servicecomb/governance/RetryHandlerTest.java
b/governance/src/test/java/org/apache/servicecomb/governance/RetryHandlerTest.java
index 1223bfe74..073cfc445 100644
---
a/governance/src/test/java/org/apache/servicecomb/governance/RetryHandlerTest.java
+++
b/governance/src/test/java/org/apache/servicecomb/governance/RetryHandlerTest.java
@@ -35,7 +35,7 @@ public class RetryHandlerTest {
AbstractRetryExtension retryExtension =
Mockito.mock(AbstractRetryExtension.class);
RetryProperties retryProperties = Mockito.mock(RetryProperties.class);
GovernanceRequest governanceRequest =
Mockito.mock(GovernanceRequest.class);
-
Mockito.when(retryExtension.isFailedResult(Mockito.any())).thenReturn(true);
+ Mockito.when(retryExtension.isFailedResult(Mockito.any(),
Mockito.any())).thenReturn(true);
RetryPolicy retryPolicy = new RetryPolicy();
retryPolicy.setName("test");
@@ -53,7 +53,7 @@ public class RetryHandlerTest {
AbstractRetryExtension retryExtension =
Mockito.mock(AbstractRetryExtension.class);
RetryProperties retryProperties = Mockito.mock(RetryProperties.class);
GovernanceRequest governanceRequest =
Mockito.mock(GovernanceRequest.class);
-
Mockito.when(retryExtension.isFailedResult(Mockito.any())).thenReturn(true);
+ Mockito.when(retryExtension.isFailedResult(Mockito.any(),
Mockito.any())).thenReturn(true);
RetryPolicy retryPolicy = new RetryPolicy();
retryPolicy.setName("test");
@@ -71,7 +71,7 @@ public class RetryHandlerTest {
AbstractRetryExtension retryExtension =
Mockito.mock(AbstractRetryExtension.class);
RetryProperties retryProperties = Mockito.mock(RetryProperties.class);
GovernanceRequest governanceRequest =
Mockito.mock(GovernanceRequest.class);
- Mockito.when(retryExtension.isFailedResult(Mockito.any(),
Mockito.any())).thenReturn(true);
+ Mockito.when(retryExtension.isFailedResult(Mockito.any(), (Object)
Mockito.any())).thenReturn(true);
RetryPolicy retryPolicy = new RetryPolicy();
retryPolicy.setName("test");
