davsclaus commented on code in PR #26608: URL: https://github.com/apache/camel/pull/26608#discussion_r4056474425
########## components/camel-ai/camel-openai/src/main/java/org/apache/camel/component/openai/OpenAIChatCompletionMultimodalSupport.java: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.openai; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; + +import com.openai.models.chat.completions.ChatCompletionContentPart; +import com.openai.models.chat.completions.ChatCompletionContentPartImage; +import com.openai.models.chat.completions.ChatCompletionContentPartInputAudio; +import com.openai.models.chat.completions.ChatCompletionContentPartText; +import com.openai.models.chat.completions.ChatCompletionMessageParam; +import com.openai.models.chat.completions.ChatCompletionUserMessageParam; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.WrappedFile; +import org.apache.camel.util.ObjectHelper; + +/** + * Builds chat-completion user messages from text, image, PDF and audio bodies. + */ +final class OpenAIChatCompletionMultimodalSupport { + + private OpenAIChatCompletionMultimodalSupport() { + } + + static ChatCompletionMessageParam buildUserMessage(Message in, String userPrompt) throws Exception { + Object body = in.getBody(); + + if (body instanceof WrappedFile || body instanceof File || body instanceof Path) { + return buildFileMessage(in, userPrompt); + } + if (body instanceof byte[] || body instanceof InputStream) { + return buildBinaryMessage(in, userPrompt); + } + return buildTextMessage(in, userPrompt); + } + + private static ChatCompletionMessageParam buildTextMessage(Message in, String userPrompt) { + String prompt = userPrompt != null ? userPrompt : in.getBody(String.class); + if (prompt == null || prompt.trim().isEmpty()) { + return null; + } + return createTextMessage(prompt); + } + + private static ChatCompletionMessageParam buildFileMessage(Message in, String userPrompt) throws Exception { + Object body = in.getBody(); + File inputFile = null; + if (body instanceof WrappedFile<?> wrappedFile && wrappedFile.getFile() instanceof File file) { + inputFile = file; + } else if (body instanceof File file) { + inputFile = file; + } else if (body instanceof Path path) { + inputFile = path.toFile(); + } + + String mime = inputFile != null + ? MimeTypeHelper.resolveForFile(in, inputFile) : MimeTypeHelper.resolveForBinary(in); + + if (MimeTypeHelper.isText(mime)) { + String prompt = userPrompt; + if (prompt == null || prompt.isEmpty()) { + prompt = in.getBody(String.class); + } + if (prompt == null || prompt.isEmpty()) { + throw new IllegalArgumentException( + "File content or user message configuration must contain the prompt text"); + } + return createTextMessage(prompt); + } + if (MimeTypeHelper.isImage(mime)) { + byte[] image = inputFile != null ? Files.readAllBytes(inputFile.toPath()) : readBodyBytes(in); + return createImageMessage(image, mime, userPrompt); + } + if (MimeTypeHelper.isPdf(mime)) { + byte[] document = inputFile != null ? Files.readAllBytes(inputFile.toPath()) : readBodyBytes(in); + return createFileMessage(document, mime, fileName(in, inputFile), userPrompt); + } + if (MimeTypeHelper.isAudio(mime)) { + byte[] audio = inputFile != null ? Files.readAllBytes(inputFile.toPath()) : readBodyBytes(in); + return createAudioMessage(audio, mime, userPrompt); + } + throw unsupportedMimeType(mime, + inputFile != null ? inputFile.getName() : in.getHeader(Exchange.FILE_NAME, String.class)); + } + + private static ChatCompletionMessageParam buildBinaryMessage(Message in, String userPrompt) throws Exception { + String mime = MimeTypeHelper.resolveForBinary(in); + if (MimeTypeHelper.isText(mime)) { + return buildTextMessage(in, userPrompt); + } + if (MimeTypeHelper.isImage(mime)) { + return createImageMessage(readBodyBytes(in), mime, userPrompt); + } + if (MimeTypeHelper.isPdf(mime)) { + return createFileMessage(readBodyBytes(in), mime, fileName(in, null), userPrompt); + } + if (MimeTypeHelper.isAudio(mime)) { + return createAudioMessage(readBodyBytes(in), mime, userPrompt); + } + throw unsupportedMimeType(mime, in.getHeader(Exchange.FILE_NAME, String.class)); + } Review Comment: Confirming from the CI run on the `Regen` head ([35495100065](https://github.com/apache/camel/actions/runs/35495100065)): `OpenAIVisionBodyTypesMockTest.byteArrayBodyWithoutMimeInfoIsTreatedAsText` fails on all three reruns (`expected: <text response> but was: <hello bytes>` — the exchange failed, so the original body came back). That test predates this PR (CAMEL-23739), so the `throw unsupportedMimeType(...)` in `buildBinaryMessage` changes an existing contract. The earlier review asked to stop the silent text fallback for an *unrecognised* MIME type; a `byte[]` with *no* MIME information at all (no `CamelOpenAIMediaType` / `Content-Type` / `CamelFileContentType` / `CamelFileName`) is a different case and should keep falling back to text. Suggest: `if (mime == null || MimeTypeHelper.isText(mime)) return buildTextMessage(in, userPrompt);` before the image/pdf/audio guards, and keep the throw for a MIME type that is present but unsupported. That keeps the existing test green and the new `OpenAIChatCompletionMultimodalMockTest` case meaningful. The PR is otherwise approved; it can be merged once this run is green. _Claude Code on behalf of davsclaus_ -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
