moremind commented on code in PR #6341:
URL: https://github.com/apache/shenyu/pull/6341#discussion_r3362970976
##########
shenyu-plugin/shenyu-plugin-ai/shenyu-plugin-ai-proxy/src/main/java/org/apache/shenyu/plugin/ai/proxy/enhanced/service/AiProxyExecutorService.java:
##########
@@ -39,78 +45,111 @@ public class AiProxyExecutorService {
private static final Logger LOG =
LoggerFactory.getLogger(AiProxyExecutorService.class);
/**
- * Execute the AI call with retry and fallback.
+ * Execute a streaming AI call directly via {@link OpenAiApi}, bypassing
Spring AI's
+ * {@code createRequest()} which loses fields like {@code
reasoning_content}.
*
- * @param mainClient the main chat client
- * @param fallbackClientOpt the optional fallback chat client
- * @param requestBody the request body
- * @return a Mono containing the ChatResponse
+ * @param mainApi the main OpenAiApi
+ * @param fallbackCtxOpt the optional fallback context (api + config)
+ * @param request the ChatCompletionRequest with all fields
preserved
+ * @param requestBody the original request body for rebuilding fallback
request
+ * @param stream whether this is a streaming request
+ * @return a Flux of ChatCompletionChunk
*/
- public Mono<ChatResponse> execute(final ChatClient mainClient, final
Optional<ChatClient> fallbackClientOpt, final String requestBody) {
- final Mono<ChatResponse> mainCall = doChatCall(mainClient,
requestBody);
+ public Flux<ChatCompletionChunk> executeDirectStream(final OpenAiApi
mainApi,
+ final Optional<FallbackContext> fallbackCtxOpt, final
ChatCompletionRequest request,
+ final String requestBody, final boolean stream) {
+ return mainApi.chatCompletionStream(request)
+ .doOnError(e -> UpstreamErrorLogger.logUpstreamError(LOG, e,
"direct stream"))
+ .retryWhen(Retry.max(1)
+ .filter(AiProxyExecutorService::isRetryable)
+ .onRetryExhaustedThrow((retryBackoffSpec, retrySignal)
-> {
+ LOG.warn("Direct stream retry exhausted.
Triggering fallback.",
+ retrySignal.failure());
+ return new NonTransientAiException(
+ "Direct stream failed after 1 retry.
Triggering fallback.",
+ retrySignal.failure());
+ }))
+ .onErrorResume(e -> handleDirectFallbackStream(e,
fallbackCtxOpt, requestBody, stream));
+ }
- return mainCall
+ /**
+ * Execute a non-streaming AI call directly via {@link OpenAiApi}.
+ *
+ * @param mainApi the main OpenAiApi
+ * @param fallbackCtxOpt the optional fallback context (api + config)
+ * @param request the ChatCompletionRequest with all fields
preserved
+ * @param requestBody the original request body for rebuilding fallback
request
+ * @return a Mono of ResponseEntity containing ChatCompletion
+ */
+ public Mono<ResponseEntity<ChatCompletion>> executeDirectCall(final
OpenAiApi mainApi,
+ final Optional<FallbackContext> fallbackCtxOpt, final
ChatCompletionRequest request,
+ final String requestBody) {
+ return Mono.fromCallable(() -> mainApi.chatCompletionEntity(request))
+ .subscribeOn(Schedulers.boundedElastic())
+ .doOnError(e -> UpstreamErrorLogger.logUpstreamError(LOG, e,
"direct call"))
.retryWhen(Retry.backoff(3, Duration.ofSeconds(1))
- .filter(throwable -> !(throwable instanceof
NonTransientAiException))
+ .filter(AiProxyExecutorService::isRetryable)
.onRetryExhaustedThrow((retryBackoffSpec, retrySignal)
-> {
- LOG.warn("Retries exhausted for AI call after {}
attempts.",
+ LOG.warn("Direct call retries exhausted after {}
attempts. Triggering fallback.",
retrySignal.totalRetries(),
retrySignal.failure());
- return new NonTransientAiException("Retries
exhausted. Triggering fallback.",
+ return new NonTransientAiException("Direct call
retries exhausted. Triggering fallback.",
retrySignal.failure());
}))
- .onErrorResume(NonTransientAiException.class,
- throwable -> handleFallback(throwable,
fallbackClientOpt, requestBody));
- }
-
- protected Mono<ChatResponse> doChatCall(final ChatClient client, final
String requestBody) {
- return Mono.fromCallable(() ->
client.prompt().user(requestBody).call().chatResponse())
- .subscribeOn(Schedulers.boundedElastic());
+ .onErrorResume(e -> handleDirectFallbackCall(e,
fallbackCtxOpt, requestBody));
}
- private Mono<ChatResponse> handleFallback(final Throwable throwable, final
Optional<ChatClient> fallbackClientOpt, final String requestBody) {
- LOG.warn("AI main call failed or retries exhausted, attempting to
fallback...", throwable);
+ private Flux<ChatCompletionChunk> handleDirectFallbackStream(final
Throwable throwable,
+ final Optional<FallbackContext> fallbackCtxOpt, final String
requestBody, final boolean stream) {
+ LOG.warn("Main direct stream failed, attempting fallback...",
throwable);
- if (fallbackClientOpt.isEmpty()) {
- return Mono.error(throwable);
+ if (fallbackCtxOpt.isEmpty()) {
+ return Flux.error(throwable);
}
- return
SimpleModelFallbackStrategy.INSTANCE.fallback(fallbackClientOpt.get(),
requestBody, throwable);
+ final FallbackContext ctx = fallbackCtxOpt.get();
+ LOG.info("Using fallback OpenAiApi for direct stream");
+ final ChatCompletionRequest fallbackRequest =
OpenAiProtocolAdapter.toChatCompletionRequest(
+ requestBody, stream, ctx.config());
+ return ctx.api().chatCompletionStream(fallbackRequest);
}
- /**
- * Execute the AI call with retry and fallback.
- *
- * @param mainClient the main chat client
- * @param fallbackClientOpt the optional fallback chat client
- * @param requestBody the request body
- * @return a Flux containing the ChatResponse
- */
- public Flux<ChatResponse> executeStream(final ChatClient mainClient, final
Optional<ChatClient> fallbackClientOpt, final String requestBody) {
- final Flux<ChatResponse> mainStream = doChatStream(mainClient,
requestBody);
+ private Mono<ResponseEntity<ChatCompletion>>
handleDirectFallbackCall(final Throwable throwable,
+ final Optional<FallbackContext> fallbackCtxOpt, final String
requestBody) {
+ LOG.warn("Main direct call failed, attempting fallback...", throwable);
- return mainStream
- .retryWhen(Retry.max(1)
- .onRetryExhaustedThrow((retryBackoffSpec, retrySignal)
-> {
- LOG.warn("Retrying stream once failed. Attempts:
{}. Triggering fallback.",
- retrySignal.totalRetries(),
retrySignal.failure());
- return new NonTransientAiException("Stream failed
after 1 retry. Triggering fallback.", retrySignal.failure());
- }))
- .onErrorResume(NonTransientAiException.class,
- throwable -> handleFallbackStream(throwable,
fallbackClientOpt, requestBody));
- }
+ if (fallbackCtxOpt.isEmpty()) {
+ return Mono.error(throwable);
+ }
- protected Flux<ChatResponse> doChatStream(final ChatClient client, final
String requestBody) {
- return Flux.defer(() ->
client.prompt().user(requestBody).stream().chatResponse())
+ final FallbackContext ctx = fallbackCtxOpt.get();
+ LOG.info("Using fallback OpenAiApi for direct call");
+ final ChatCompletionRequest fallbackRequest =
OpenAiProtocolAdapter.toChatCompletionRequest(
+ requestBody, false, ctx.config());
+ return Mono.fromCallable(() ->
ctx.api().chatCompletionEntity(fallbackRequest))
.subscribeOn(Schedulers.boundedElastic());
}
- private Flux<ChatResponse> handleFallbackStream(final Throwable throwable,
final Optional<ChatClient> fallbackClientOpt, final String requestBody) {
- LOG.warn("AI main stream failed or retries exhausted, attempting to
fallback...", throwable);
-
- if (fallbackClientOpt.isEmpty()) {
- return Flux.error(throwable);
+ /**
+ * Determine if the error is retryable.
+ * Retries transient network errors and retryable HTTP statuses (429, 5xx).
+ * Non-retryable: NonTransientAiException, client errors (400/401/403/404).
+ */
+ private static boolean isRetryable(final Throwable throwable) {
+ if (throwable instanceof NonTransientAiException) {
+ return false;
}
+ final WebClientResponseException webClientEx =
UpstreamErrorLogger.findWebClientResponseException(throwable);
+ if (Objects.nonNull(webClientEx)) {
+ final int status = webClientEx.getStatusCode().value();
+ return status == 429 || status >= 500;
Review Comment:
not use magic value
##########
shenyu-plugin/shenyu-plugin-ai/shenyu-plugin-ai-proxy/src/main/java/org/apache/shenyu/plugin/ai/proxy/enhanced/service/UpstreamErrorLogger.java:
##########
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shenyu.plugin.ai.proxy.enhanced.service;
+
+import org.slf4j.Logger;
+import
org.springframework.web.reactive.function.client.WebClientResponseException;
+
+import java.util.Objects;
+
+/**
+ * Shared utility for logging upstream AI service errors with
WebClientResponseException details.
+ */
+public final class UpstreamErrorLogger {
+
+ private static final int MAX_BODY_LOG_LENGTH = 512;
+
+ private UpstreamErrorLogger() {
+ }
+
+ public static void logUpstreamError(final Logger log, final Throwable e,
final String mode) {
+ if (Objects.isNull(e)) {
+ return;
+ }
+ final WebClientResponseException webClientEx =
findWebClientResponseException(e);
+ if (Objects.nonNull(webClientEx)) {
+ log.error("[AiProxy] {} failed, status={}, upstreamBody={}",
+ mode, webClientEx.getStatusCode(),
truncateBody(webClientEx.getResponseBodyAsString()), e);
+ } else {
+ log.error("[AiProxy] {} failed", mode, e);
+ }
+ }
+
+ private static String truncateBody(final String body) {
+ if (Objects.isNull(body)) {
+ return "null";
Review Comment:
why return null string?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
