This is an automated email from the ASF dual-hosted git repository.
wenjin272 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 1618728b [integration][java][python] Send multimodal user messages
through OpenAI Chat Completions (#1164)
1618728b is described below
commit 1618728b8d2b0ad9f06704dadf0637552efa431e
Author: Edson <[email protected]>
AuthorDate: Tue Sep 29 04:14:52 2026 -0400
[integration][java][python] Send multimodal user messages through OpenAI
Chat Completions (#1164)
Generated-by: Claude Code 2.1.259 (Claude Opus 5.5)
---
.../messages/UnsupportedContentBlockException.java | 53 ++++++
docs/content/docs/development/chat_models.md | 26 ++-
.../openai/OpenAIChatCompletionsUtils.java | 146 ++++++++++++++-
.../OpenAIChatCompletionsMultimodalLiveTest.java | 197 +++++++++++++++++++++
.../OpenAIChatCompletionsMultimodalTest.java | 161 +++++++++++++++++
python/flink_agents/api/chat_message.py | 25 +++
.../chat_models/openai/openai_utils.py | 96 +++++++++-
.../openai/tests/test_openai_multimodal.py | 143 +++++++++++++++
.../openai/tests/test_openai_multimodal_live.py | 161 +++++++++++++++++
9 files changed, 1000 insertions(+), 8 deletions(-)
diff --git
a/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
b/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
new file mode 100644
index 00000000..da58ccc4
--- /dev/null
+++
b/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.api.chat.messages;
+
+/**
+ * Thrown when a chat model integration cannot send a {@link ContentBlock} to
its provider, such as
+ * a media type the provider does not accept or a media block in a message
role that only takes
+ * text. Integrations throw it rather than silently dropping or converting the
block.
+ *
+ * <p>Messages name the block type, media type and source type only, never the
media payload or URL.
+ */
+public class UnsupportedContentBlockException extends IllegalArgumentException
{
+
+ private static final long serialVersionUID = 1L;
+
+ public UnsupportedContentBlockException(String message) {
+ super(message);
+ }
+
+ /**
+ * Creates the exception for a block {@code provider} cannot send, with a
message of the form
+ * "{provider} cannot send a(n) {type} block ({media type}, {source type}
source): {reason}."
+ */
+ public static UnsupportedContentBlockException forBlock(
+ String provider, ContentBlock block, String reason) {
+ String type = block.getType();
+ String description =
+ ("aeiou".indexOf(type.charAt(0)) >= 0 ? "an " : "a ") + type +
" block";
+ if (block instanceof MediaBlock) {
+ MediaBlock media = (MediaBlock) block;
+ description +=
+ " (" + media.getMediaType() + ", " +
media.getSource().getType() + " source)";
+ }
+ return new UnsupportedContentBlockException(
+ provider + " cannot send " + description + ": " + reason +
".");
+ }
+}
diff --git a/docs/content/docs/development/chat_models.md
b/docs/content/docs/development/chat_models.md
index b04fea20..11a1b8a8 100644
--- a/docs/content/docs/development/chat_models.md
+++ b/docs/content/docs/development/chat_models.md
@@ -60,9 +60,11 @@ order you want them presented. Supported block types are
`TextBlock`, `ImageBloc
`AudioBlock`, `VideoBlock`, and `DocumentBlock`. For example, a message can
contain
a question followed by an image:
-**Provider support:** Built-in provider integrations currently send only the
text
-portion of a message. These examples construct multimodal messages; sending
their
-media to a model requires a provider integration that supports those block
types.
+**Provider support:** The OpenAI Chat Completions integration, and the Azure
OpenAI
+and vLLM integrations built on it, send media blocks to the model; see
+[Multimodal Input](#multimodal-input) for what each block becomes. The other
+built-in integrations, including the OpenAI Responses integration, currently
send
+only the text portion of a message.
{{< tabs "Message content blocks" >}}
@@ -594,6 +596,8 @@ public class MyAgent extends Agent {
{{< /tabs >}}
+Azure OpenAI sends media blocks the same way as the OpenAI Chat Completions
API; see [Multimodal Input](#multimodal-input).
+
#### Available Models
Azure OpenAI supports OpenAI models deployed through your Azure subscription.
Visit the [Azure OpenAI Models
documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models)
for the complete and up-to-date list of available models.
@@ -1003,6 +1007,20 @@ public class MyAgent extends Agent {
{{< /tabs >}}
+#### Multimodal Input
+
+The Chat Completions connections (OpenAI, Azure OpenAI and vLLM) send the
media blocks of a user message as content parts, in block order:
+
+| Block | Sent as |
+|---|---|
+| `TextBlock` | a `text` part |
+| `ImageBlock` | an `image_url` part: the URL, or a `data:` URI for Base64
data |
+| `AudioBlock` | an `input_audio` part, for Base64 data in `audio/wav` or
`audio/mpeg` (WAV or MP3) |
+| `DocumentBlock` | a `file` part: the Base64 data as a `data:` URI, with the
block's `name` as the file name (`document` when unset). OpenAI accepts PDF
documents only |
+| `VideoBlock` | not supported |
+
+A user message with only text blocks is still sent as a plain string. Media in
system, assistant or tool messages, audio or documents given by URL, other
audio types, and video raise `UnsupportedContentBlockException` (Java) or
`UnsupportedContentBlockError` (Python) before the request is sent. Whether a
model accepts a given part depends on the model and the server.
+
#### Responses API
{{< hint info >}}
@@ -1281,6 +1299,8 @@ public class MyAgent extends Agent {
{{< /tabs >}}
+vLLM receives media blocks the same way as the OpenAI Chat Completions API;
see [Multimodal Input](#multimodal-input). The model must be multimodal; see
the [vLLM multimodal inputs
docs](https://docs.vllm.ai/en/stable/features/multimodal_inputs/).
+
#### Available Models
A vLLM server serves the model(s) it was started with. Query `GET /v1/models`
on the server to list them; the `model` value in the setup must match one of
the returned names.
diff --git
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtils.java
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtils.java
index 60b1f2f0..dd53038c 100644
---
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtils.java
+++
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsUtils.java
@@ -23,6 +23,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.openai.core.JsonValue;
import com.openai.core.Timeout;
import com.openai.models.chat.completions.ChatCompletionAssistantMessageParam;
+import com.openai.models.chat.completions.ChatCompletionContentPart;
+import com.openai.models.chat.completions.ChatCompletionContentPartImage;
+import com.openai.models.chat.completions.ChatCompletionContentPartInputAudio;
+import com.openai.models.chat.completions.ChatCompletionContentPartText;
import com.openai.models.chat.completions.ChatCompletionMessage;
import
com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall;
import com.openai.models.chat.completions.ChatCompletionMessageParam;
@@ -30,8 +34,17 @@ import
com.openai.models.chat.completions.ChatCompletionMessageToolCall;
import com.openai.models.chat.completions.ChatCompletionSystemMessageParam;
import com.openai.models.chat.completions.ChatCompletionToolMessageParam;
import com.openai.models.chat.completions.ChatCompletionUserMessageParam;
+import org.apache.flink.agents.api.chat.messages.AudioBlock;
+import org.apache.flink.agents.api.chat.messages.Base64Source;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ContentBlock;
+import org.apache.flink.agents.api.chat.messages.DocumentBlock;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.MediaBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
+import org.apache.flink.agents.api.chat.messages.UrlSource;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
import java.math.BigDecimal;
@@ -41,6 +54,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -167,18 +181,25 @@ final class OpenAIChatCompletionsUtils {
.collect(Collectors.toList());
}
- /** Convert a single Flink Agents ChatMessage to an OpenAI
ChatCompletionMessageParam. */
+ /**
+ * Convert a single Flink Agents ChatMessage to an OpenAI
ChatCompletionMessageParam.
+ *
+ * <p>Only user messages can carry media; a media block in any other role,
or one the Chat
+ * Completions API has no content part for, throws {@link
UnsupportedContentBlockException}.
+ */
public static ChatCompletionMessageParam
convertToOpenAIMessage(ChatMessage message) {
MessageRole role = message.getRole();
String content = Optional.ofNullable(message.getText()).orElse("");
+ if (role != MessageRole.USER) {
+ requireTextOnly(message);
+ }
switch (role) {
case SYSTEM:
return ChatCompletionMessageParam.ofSystem(
ChatCompletionSystemMessageParam.builder().content(content).build());
case USER:
- return ChatCompletionMessageParam.ofUser(
-
ChatCompletionUserMessageParam.builder().content(content).build());
+ return
ChatCompletionMessageParam.ofUser(convertUserMessage(message, content));
case ASSISTANT:
ChatCompletionAssistantMessageParam.Builder assistantBuilder =
ChatCompletionAssistantMessageParam.builder();
@@ -209,6 +230,125 @@ final class OpenAIChatCompletionsUtils {
}
}
+ /**
+ * A text-only user message keeps the plain string content, as before; a
message with media is
+ * sent as content parts in block order.
+ */
+ private static ChatCompletionUserMessageParam convertUserMessage(
+ ChatMessage message, String text) {
+ List<ContentBlock> blocks = message.getBlocks();
+ if (blocks.stream().noneMatch(block -> block instanceof MediaBlock)) {
+ return
ChatCompletionUserMessageParam.builder().content(text).build();
+ }
+ List<ChatCompletionContentPart> parts = new ArrayList<>(blocks.size());
+ for (ContentBlock block : blocks) {
+ parts.add(toContentPart(block));
+ }
+ return
ChatCompletionUserMessageParam.builder().contentOfArrayOfContentParts(parts).build();
+ }
+
+ private static ChatCompletionContentPart toContentPart(ContentBlock block)
{
+ if (block instanceof TextBlock) {
+ return ChatCompletionContentPart.ofText(
+ ChatCompletionContentPartText.builder()
+ .text(((TextBlock) block).getText())
+ .build());
+ }
+ if (block instanceof ImageBlock) {
+ ImageBlock image = (ImageBlock) block;
+ String url =
+ image.getSource() instanceof UrlSource
+ ? ((UrlSource) image.getSource()).getUrl()
+ : dataUri(image);
+ return ChatCompletionContentPart.ofImageUrl(
+ ChatCompletionContentPartImage.builder()
+ .imageUrl(
+
ChatCompletionContentPartImage.ImageUrl.builder()
+ .url(url)
+ .build())
+ .build());
+ }
+ if (block instanceof AudioBlock) {
+ AudioBlock audio = (AudioBlock) block;
+ if (!(audio.getSource() instanceof Base64Source)) {
+ throw unsupported(block, "audio input takes base64 data, not a
URL");
+ }
+ ChatCompletionContentPartInputAudio.InputAudio.Format format =
+ audioFormat(audio.getMediaType());
+ if (format == null) {
+ throw unsupported(block, "audio input takes WAV or MP3 only");
+ }
+ return ChatCompletionContentPart.ofInputAudio(
+ ChatCompletionContentPartInputAudio.builder()
+ .inputAudio(
+
ChatCompletionContentPartInputAudio.InputAudio.builder()
+ .data(((Base64Source)
audio.getSource()).getData())
+ .format(format)
+ .build())
+ .build());
+ }
+ if (block instanceof DocumentBlock) {
+ DocumentBlock document = (DocumentBlock) block;
+ if (!(document.getSource() instanceof Base64Source)) {
+ throw unsupported(block, "file input takes base64 data, not a
URL");
+ }
+ // OpenAI accepts PDF documents only; other types are left for the
server to reject.
+ return ChatCompletionContentPart.ofFile(
+ ChatCompletionContentPart.File.builder()
+ .file(
+
ChatCompletionContentPart.File.FileObject.builder()
+ .fileData(dataUri(document))
+ .filename(
+ document.getName() != null
+ ?
document.getName()
+ : "document")
+ .build())
+ .build());
+ }
+ throw unsupported(block, "there is no content part for it");
+ }
+
+ private static void requireTextOnly(ChatMessage message) {
+ for (ContentBlock block : message.getBlocks()) {
+ if (block instanceof MediaBlock) {
+ throw unsupported(
+ block,
+ "only user messages can carry media, not "
+ + message.getRole().getValue()
+ + " messages");
+ }
+ }
+ }
+
+ /** Maps an audio media type onto the formats Chat Completions accepts, or
null. */
+ private static ChatCompletionContentPartInputAudio.InputAudio.Format
audioFormat(
+ String mediaType) {
+ String essence = mediaType.split(";",
2)[0].trim().toLowerCase(Locale.ROOT);
+ switch (essence) {
+ case "audio/wav":
+ case "audio/wave":
+ case "audio/x-wav":
+ case "audio/vnd.wave":
+ return
ChatCompletionContentPartInputAudio.InputAudio.Format.WAV;
+ case "audio/mpeg":
+ case "audio/mp3":
+ return
ChatCompletionContentPartInputAudio.InputAudio.Format.MP3;
+ default:
+ return null;
+ }
+ }
+
+ private static String dataUri(MediaBlock block) {
+ return "data:"
+ + block.getMediaType()
+ + ";base64,"
+ + ((Base64Source) block.getSource()).getData();
+ }
+
+ private static UnsupportedContentBlockException unsupported(ContentBlock
block, String reason) {
+ return UnsupportedContentBlockException.forBlock("OpenAI Chat
Completions", block, reason);
+ }
+
/**
* Convert an OpenAI {@link ChatCompletionMessage} to a Flink Agents
{@link ChatMessage}. {@code
* message.refusal()} is written as {@code extraArgs["refusal"]} on the
returned ChatMessage
diff --git
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalLiveTest.java
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalLiveTest.java
new file mode 100644
index 00000000..f40c0d35
--- /dev/null
+++
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalLiveTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import org.apache.flink.agents.api.chat.messages.AudioBlock;
+import org.apache.flink.agents.api.chat.messages.Base64Source;
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ContentBlock;
+import org.apache.flink.agents.api.chat.messages.DocumentBlock;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import javax.imageio.ImageIO;
+
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Sends real multimodal requests through the OpenAI Chat Completions
connection, to check that the
+ * provider accepts the content parts {@link
OpenAIChatCompletionsMultimodalTest} pins.
+ *
+ * <p>Skipped unless {@code OPENAI_API_KEY} is set. {@code
OPENAI_API_BASE_URL}, {@code
+ * OPENAI_MULTIMODAL_MODEL} (default {@code gpt-4o-mini}) and {@code
OPENAI_AUDIO_MODEL} (default
+ * {@code gpt-4o-audio-preview}) override the endpoint and models.
+ */
+class OpenAIChatCompletionsMultimodalLiveTest {
+
+ private static final String API_KEY = System.getenv("OPENAI_API_KEY");
+
+ @BeforeAll
+ static void requireCredentials() {
+ assumeTrue(API_KEY != null && !API_KEY.isBlank(), "OPENAI_API_KEY is
not set");
+ }
+
+ @Test
+ @DisplayName("A base64 image is accepted")
+ void testImage() throws Exception {
+ assertThat(
+ chat(
+ multimodalModel(),
+ TextBlock.of("What color is this image? Answer
in one word."),
+ ImageBlock.fromBase64("image/png",
redSquarePng())))
+ .isNotBlank();
+ }
+
+ @Test
+ @DisplayName("WAV audio is accepted")
+ void testAudio() {
+ assertThat(
+ chat(
+ env("OPENAI_AUDIO_MODEL",
"gpt-4o-audio-preview"),
+ TextBlock.of("Describe this audio clip in one
sentence."),
+ AudioBlock.fromBase64("audio/wav",
silentWav())))
+ .isNotBlank();
+ }
+
+ @Test
+ @DisplayName("A named PDF is accepted")
+ void testNamedPdf() {
+ assertThat(
+ chat(
+ multimodalModel(),
+ TextBlock.of("What text does this PDF
contain?"),
+ new DocumentBlock(
+ "application/pdf",
+ new Base64Source(helloPdf()),
+ "hello.pdf",
+ null,
+ null)))
+ .isNotBlank();
+ }
+
+ @Test
+ @DisplayName("A PDF without a name is accepted under the default file
name")
+ void testUnnamedPdf() {
+ assertThat(
+ chat(
+ multimodalModel(),
+ TextBlock.of("What text does this PDF
contain?"),
+ DocumentBlock.fromBase64("application/pdf",
helloPdf())))
+ .isNotBlank();
+ }
+
+ private static String chat(String model, ContentBlock... blocks) {
+ ResourceDescriptor.Builder descriptor =
+
ResourceDescriptor.Builder.newBuilder(OpenAICompletionsConnection.class.getName())
+ .addInitialArgument("api_key", API_KEY);
+ String apiBaseUrl = System.getenv("OPENAI_API_BASE_URL");
+ if (apiBaseUrl != null && !apiBaseUrl.isBlank()) {
+ descriptor.addInitialArgument("api_base_url", apiBaseUrl);
+ }
+ OpenAICompletionsConnection connection =
+ new OpenAICompletionsConnection(
+ descriptor.build(),
ResourceContext.fromGetResource((name, type) -> null));
+ Map<String, Object> params = new HashMap<>();
+ params.put("model", model);
+ return connection
+ .chat(List.of(ChatMessage.user(List.of(blocks))), List.of(),
params)
+ .getText();
+ }
+
+ private static String multimodalModel() {
+ return env("OPENAI_MULTIMODAL_MODEL", "gpt-4o-mini");
+ }
+
+ private static String env(String name, String defaultValue) {
+ String value = System.getenv(name);
+ return value == null || value.isBlank() ? defaultValue : value;
+ }
+
+ private static String redSquarePng() throws Exception {
+ BufferedImage image = new BufferedImage(16, 16,
BufferedImage.TYPE_INT_RGB);
+ for (int x = 0; x < 16; x++) {
+ for (int y = 0; y < 16; y++) {
+ image.setRGB(x, y, Color.RED.getRGB());
+ }
+ }
+ ByteArrayOutputStream png = new ByteArrayOutputStream();
+ ImageIO.write(image, "png", png);
+ return Base64.getEncoder().encodeToString(png.toByteArray());
+ }
+
+ /** Half a second of 16 kHz mono 16-bit PCM silence. */
+ static String silentWav() {
+ int sampleRate = 16_000;
+ int dataSize = sampleRate / 2 * 2;
+ ByteBuffer wav = ByteBuffer.allocate(44 +
dataSize).order(ByteOrder.LITTLE_ENDIAN);
+ wav.put("RIFF".getBytes(US_ASCII)).putInt(36 +
dataSize).put("WAVE".getBytes(US_ASCII));
+ wav.put("fmt ".getBytes(US_ASCII)).putInt(16).putShort((short)
1).putShort((short) 1);
+ wav.putInt(sampleRate).putInt(sampleRate * 2).putShort((short)
2).putShort((short) 16);
+ wav.put("data".getBytes(US_ASCII)).putInt(dataSize);
+ return Base64.getEncoder().encodeToString(wav.array());
+ }
+
+ /** A one-page PDF reading "Hello PDF", with a valid cross-reference
table. */
+ static String helloPdf() {
+ String stream = "BT /F1 24 Tf 20 40 Td (Hello PDF) Tj ET";
+ String[] objects = {
+ "<< /Type /Catalog /Pages 2 0 R >>",
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 100] /Contents 4
0 R"
+ + " /Resources << /Font << /F1 5 0 R >> >> >>",
+ "<< /Length " + stream.length() + " >>\nstream\n" + stream +
"\nendstream",
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"
+ };
+ StringBuilder pdf = new StringBuilder("%PDF-1.4\n");
+ List<Integer> offsets = new ArrayList<>();
+ for (int i = 0; i < objects.length; i++) {
+ offsets.add(pdf.length());
+ pdf.append(i + 1).append(" 0
obj\n").append(objects[i]).append("\nendobj\n");
+ }
+ int xref = pdf.length();
+ pdf.append("xref\n0 ").append(objects.length + 1).append("\n0000000000
65535 f \n");
+ for (int offset : offsets) {
+ pdf.append(String.format("%010d 00000 n \n", offset));
+ }
+ pdf.append("trailer\n<< /Size ")
+ .append(objects.length + 1)
+ .append(" /Root 1 0 R >>\nstartxref\n")
+ .append(xref)
+ .append("\n%%EOF\n");
+ return
Base64.getEncoder().encodeToString(pdf.toString().getBytes(US_ASCII));
+ }
+}
diff --git
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalTest.java
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalTest.java
new file mode 100644
index 00000000..be6266a0
--- /dev/null
+++
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIChatCompletionsMultimodalTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.openai.core.ObjectMappers;
+import org.apache.flink.agents.api.chat.messages.AudioBlock;
+import org.apache.flink.agents.api.chat.messages.Base64Source;
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ContentBlock;
+import org.apache.flink.agents.api.chat.messages.DocumentBlock;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
+import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
+import org.apache.flink.agents.api.chat.messages.VideoBlock;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Tests how content blocks become Chat Completions content parts. Assertions
run on the serialized
+ * request message, i.e. the wire shape OpenAI, Azure OpenAI and vLLM receive.
+ */
+class OpenAIChatCompletionsMultimodalTest {
+
+ private static final String IMAGE_URL =
"https://example.com/cat.png?sig=secret";;
+ private static final String BASE64 = "aGVsbG8=";
+
+ private static JsonNode wire(ChatMessage message) {
+ return ObjectMappers.jsonMapper()
+
.valueToTree(OpenAIChatCompletionsUtils.convertToOpenAIMessage(message));
+ }
+
+ @Test
+ @DisplayName("A text-only user message keeps plain string content")
+ void testTextOnlyUserMessageKeepsStringContent() {
+ JsonNode message =
+ wire(ChatMessage.user(List.of(TextBlock.of("Describe "),
TextBlock.of("this"))));
+
+ assertThat(message.get("content").isTextual()).isTrue();
+ assertThat(message.get("content").asText()).isEqualTo("Describe this");
+ }
+
+ @Test
+ @DisplayName("User media becomes content parts in block order")
+ void testUserMediaBecomesOrderedContentParts() {
+ JsonNode parts =
+ wire(ChatMessage.user(
+ List.of(
+ TextBlock.of("Compare"),
+ ImageBlock.fromUrl("image/png",
IMAGE_URL),
+ ImageBlock.fromBase64("image/jpeg",
BASE64))))
+ .get("content");
+
+ assertThat(parts).hasSize(3);
+ assertThat(parts.get(0).get("type").asText()).isEqualTo("text");
+ assertThat(parts.get(0).get("text").asText()).isEqualTo("Compare");
+ assertThat(parts.get(1).get("type").asText()).isEqualTo("image_url");
+
assertThat(parts.get(1).at("/image_url/url").asText()).isEqualTo(IMAGE_URL);
+ assertThat(parts.get(2).at("/image_url/url").asText())
+ .isEqualTo("data:image/jpeg;base64," + BASE64);
+ }
+
+ @Test
+ @DisplayName("WAV and MP3 audio becomes input_audio with the matching
format")
+ void testAudioBecomesInputAudio() {
+ JsonNode parts =
+ wire(ChatMessage.user(
+ List.of(
+ AudioBlock.fromBase64("audio/wav",
BASE64),
+ AudioBlock.fromBase64("audio/mpeg",
BASE64))))
+ .get("content");
+
+ assertThat(parts.get(0).get("type").asText()).isEqualTo("input_audio");
+
assertThat(parts.get(0).at("/input_audio/data").asText()).isEqualTo(BASE64);
+
assertThat(parts.get(0).at("/input_audio/format").asText()).isEqualTo("wav");
+
assertThat(parts.get(1).at("/input_audio/format").asText()).isEqualTo("mp3");
+ }
+
+ @Test
+ @DisplayName("A base64 document becomes a file part with a data URI and a
file name")
+ void testDocumentBecomesFilePart() {
+ JsonNode parts =
+ wire(ChatMessage.user(
+ List.of(
+ new DocumentBlock(
+ "application/pdf",
+ new Base64Source(BASE64),
+ "report.pdf",
+ null,
+ null),
+
DocumentBlock.fromBase64("application/pdf", BASE64))))
+ .get("content");
+
+ assertThat(parts.get(0).get("type").asText()).isEqualTo("file");
+ assertThat(parts.get(0).at("/file/file_data").asText())
+ .isEqualTo("data:application/pdf;base64," + BASE64);
+
assertThat(parts.get(0).at("/file/filename").asText()).isEqualTo("report.pdf");
+
assertThat(parts.get(1).at("/file/filename").asText()).isEqualTo("document");
+ }
+
+ @Test
+ @DisplayName("Blocks without a Chat Completions content part fail without
leaking the source")
+ void testUnsupportedUserBlocksFailExplicitly() {
+ List<ContentBlock> unsupported =
+ List.of(
+ VideoBlock.fromUrl("video/mp4", IMAGE_URL),
+ AudioBlock.fromUrl("audio/wav", IMAGE_URL),
+ AudioBlock.fromBase64("audio/ogg", BASE64),
+ DocumentBlock.fromUrl("application/pdf", IMAGE_URL));
+ for (ContentBlock block : unsupported) {
+ assertThatThrownBy(
+ () ->
+
OpenAIChatCompletionsUtils.convertToOpenAIMessage(
+
ChatMessage.user(List.of(TextBlock.of("hi"), block))))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessageContaining(block.getType() + " block")
+ .hasMessageNotContaining("secret")
+ .hasMessageNotContaining(BASE64);
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = MessageRole.class,
+ names = {"SYSTEM", "ASSISTANT", "TOOL"})
+ @DisplayName("Media outside user messages fails explicitly")
+ void testMediaOutsideUserMessagesFails(MessageRole role) {
+ ChatMessage message =
+ new ChatMessage(
+ role,
+ List.of(TextBlock.of("see"),
ImageBlock.fromUrl("image/png", IMAGE_URL)));
+
+ assertThatThrownBy(() ->
OpenAIChatCompletionsUtils.convertToOpenAIMessage(message))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessageContaining("only user messages can carry media");
+ }
+}
diff --git a/python/flink_agents/api/chat_message.py
b/python/flink_agents/api/chat_message.py
index e6f7c7fc..fbc95c1a 100644
--- a/python/flink_agents/api/chat_message.py
+++ b/python/flink_agents/api/chat_message.py
@@ -177,6 +177,31 @@ ContentBlock = Annotated[
]
+class UnsupportedContentBlockError(ValueError):
+ """A chat model integration cannot send a content block to its provider.
+
+ Raised for a media type the provider does not accept, or a media block in
+ a message role that only takes text, rather than silently dropping or
+ converting the block. Messages name the block type, media type and source
+ type only, never the media payload or URL.
+ """
+
+ @classmethod
+ def for_block(
+ cls, provider: str, block: ContentBlock, reason: str
+ ) -> "UnsupportedContentBlockError":
+ """Create the error for a block the provider cannot send.
+
+ The message reads "{provider} cannot send a(n) {type} block ({media
type},
+ {source type} source): {reason}."
+ """
+ article = "an" if block.type[0] in "aeiou" else "a"
+ description = f"{article} {block.type} block"
+ if isinstance(block, MediaBlock):
+ description += f" ({block.media_type}, {block.source.type} source)"
+ return cls(f"{provider} cannot send {description}: {reason}.")
+
+
def _blocks_of(text: str) -> List[ContentBlock]:
"""An empty text becomes an empty block list rather than an empty text
block."""
return [TextBlock(text=text)] if text else []
diff --git
a/python/flink_agents/integrations/chat_models/openai/openai_utils.py
b/python/flink_agents/integrations/chat_models/openai/openai_utils.py
index 8a08934f..c02e3f35 100644
--- a/python/flink_agents/integrations/chat_models/openai/openai_utils.py
+++ b/python/flink_agents/integrations/chat_models/openai/openai_utils.py
@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Sequence,
Tuple
import openai
from openai.types.chat import (
+ ChatCompletionContentPartParam,
ChatCompletionMessage,
ChatCompletionMessageParam,
ChatCompletionMessageToolCallParam,
@@ -36,7 +37,19 @@ if TYPE_CHECKING:
)
from openai.types.chat.chat_completion_message_tool_call_param import
Function
-from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.api.chat_message import (
+ AudioBlock,
+ Base64Source,
+ ChatMessage,
+ ContentBlock,
+ DocumentBlock,
+ ImageBlock,
+ MediaBlock,
+ MessageRole,
+ TextBlock,
+ UnsupportedContentBlockError,
+ UrlSource,
+)
DEFAULT_OPENAI_API_BASE_URL = "https://api.openai.com/v1";
@@ -121,6 +134,79 @@ def _convert_to_openai_tool_call(tool_call: dict) ->
ChatCompletionMessageToolCa
return openai_tool_call
+_AUDIO_FORMATS = {
+ "audio/wav": "wav",
+ "audio/wave": "wav",
+ "audio/x-wav": "wav",
+ "audio/vnd.wave": "wav",
+ "audio/mpeg": "mp3",
+ "audio/mp3": "mp3",
+}
+
+
+def _unsupported(block: ContentBlock, reason: str) ->
UnsupportedContentBlockError:
+ return UnsupportedContentBlockError.for_block(
+ "OpenAI Chat Completions", block, reason
+ )
+
+
+def _data_uri(block: MediaBlock) -> str:
+ return f"data:{block.media_type};base64,{block.source.data}"
+
+
+def _to_content_part(block: ContentBlock) -> ChatCompletionContentPartParam:
+ if isinstance(block, TextBlock):
+ return {"type": "text", "text": block.text}
+ if isinstance(block, ImageBlock):
+ url = (
+ block.source.url
+ if isinstance(block.source, UrlSource)
+ else _data_uri(block)
+ )
+ return {"type": "image_url", "image_url": {"url": url}}
+ if isinstance(block, AudioBlock):
+ if not isinstance(block.source, Base64Source):
+ raise _unsupported(block, "audio input takes base64 data, not a
URL")
+ audio_format = _AUDIO_FORMATS.get(
+ block.media_type.split(";", 1)[0].strip().lower()
+ )
+ if audio_format is None:
+ raise _unsupported(block, "audio input takes WAV or MP3 only")
+ return {
+ "type": "input_audio",
+ "input_audio": {"data": block.source.data, "format": audio_format},
+ }
+ if isinstance(block, DocumentBlock):
+ if not isinstance(block.source, Base64Source):
+ raise _unsupported(block, "file input takes base64 data, not a
URL")
+ # OpenAI accepts PDF documents only; other types are left for the
+ # server to reject.
+ return {
+ "type": "file",
+ "file": {
+ "file_data": _data_uri(block),
+ "filename": block.name if block.name is not None else
"document",
+ },
+ }
+ raise _unsupported(block, "there is no content part for it")
+
+
+def _user_content(message: ChatMessage) -> str |
List[ChatCompletionContentPartParam]:
+ """Keep plain string content for text-only messages; media switches to
parts."""
+ if not any(isinstance(block, MediaBlock) for block in message.blocks):
+ return message.text
+ return [_to_content_part(block) for block in message.blocks]
+
+
+def _require_text_only(message: ChatMessage) -> None:
+ for block in message.blocks:
+ if isinstance(block, MediaBlock):
+ reason = (
+ f"only user messages can carry media, not {message.role.value}
messages"
+ )
+ raise _unsupported(block, reason)
+
+
def convert_to_openai_messages(
messages: Sequence[ChatMessage],
) -> List[ChatCompletionMessageParam]:
@@ -141,8 +227,14 @@ def convert_to_openai_message(message: ChatMessage) ->
ChatCompletionMessagePara
extra_args are not forwarded as message fields, except that a tool
message takes its tool_call_id from extra_args["external_id"], and an
assistant message carries extra_args["refusal"] when that value is a str.
+
+ Only user messages can carry media. A media block in any other role, or
+ one the Chat Completions API has no content part for, raises
+ UnsupportedContentBlockError.
"""
role = message.role
+ if role != MessageRole.USER:
+ _require_text_only(message)
# Handle SYSTEM role messages
if role == MessageRole.SYSTEM:
@@ -156,7 +248,7 @@ def convert_to_openai_message(message: ChatMessage) ->
ChatCompletionMessagePara
elif role == MessageRole.USER:
user_message: ChatCompletionUserMessageParam = {
"role": "user",
- "content": message.text,
+ "content": _user_content(message),
}
return user_message
# Handle ASSISTANT role messages
diff --git
a/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal.py
b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal.py
new file mode 100644
index 00000000..a2895cec
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal.py
@@ -0,0 +1,143 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""How content blocks become Chat Completions content parts.
+
+The converter is shared by the OpenAI, Azure OpenAI and vLLM connections, so
+these cases pin the wire shape all three send. They mirror the Java
+OpenAIChatCompletionsMultimodalTest.
+"""
+
+import pytest
+
+from flink_agents.api.chat_message import (
+ AudioBlock,
+ Base64Source,
+ ChatMessage,
+ ContentBlock,
+ DocumentBlock,
+ ImageBlock,
+ MessageRole,
+ TextBlock,
+ UnsupportedContentBlockError,
+ VideoBlock,
+)
+from flink_agents.integrations.chat_models.openai.openai_utils import (
+ convert_to_openai_message,
+)
+
+IMAGE_URL = "https://example.com/cat.png?sig=secret";
+BASE64 = "aGVsbG8="
+
+
+def test_text_only_user_message_keeps_string_content() -> None:
+ """A user message without media keeps the plain string content."""
+ message = ChatMessage.user([TextBlock(text="Describe "),
TextBlock(text="this")])
+
+ assert convert_to_openai_message(message)["content"] == "Describe this"
+
+
+def test_user_media_becomes_ordered_content_parts() -> None:
+ """Media switches the user content to parts, in block order."""
+ message = ChatMessage.user(
+ [
+ TextBlock(text="Compare"),
+ ImageBlock.from_url("image/png", IMAGE_URL),
+ ImageBlock.from_base64("image/jpeg", BASE64),
+ ]
+ )
+
+ assert convert_to_openai_message(message)["content"] == [
+ {"type": "text", "text": "Compare"},
+ {"type": "image_url", "image_url": {"url": IMAGE_URL}},
+ {"type": "image_url", "image_url": {"url":
f"data:image/jpeg;base64,{BASE64}"}},
+ ]
+
+
+def test_audio_becomes_input_audio() -> None:
+ """WAV and MP3 audio map onto the two formats input_audio accepts."""
+ message = ChatMessage.user(
+ [
+ AudioBlock.from_base64("audio/wav", BASE64),
+ AudioBlock.from_base64("audio/mpeg", BASE64),
+ ]
+ )
+
+ assert convert_to_openai_message(message)["content"] == [
+ {"type": "input_audio", "input_audio": {"data": BASE64, "format":
"wav"}},
+ {"type": "input_audio", "input_audio": {"data": BASE64, "format":
"mp3"}},
+ ]
+
+
+def test_document_becomes_file_part() -> None:
+ """A base64 document is sent as file data with a file name."""
+ message = ChatMessage.user(
+ [
+ DocumentBlock(
+ media_type="application/pdf",
+ source=Base64Source(data=BASE64),
+ name="report.pdf",
+ ),
+ DocumentBlock.from_base64("application/pdf", BASE64),
+ ]
+ )
+
+ parts = convert_to_openai_message(message)["content"]
+ assert parts[0] == {
+ "type": "file",
+ "file": {
+ "file_data": f"data:application/pdf;base64,{BASE64}",
+ "filename": "report.pdf",
+ },
+ }
+ assert parts[1]["file"]["filename"] == "document"
+
+
[email protected](
+ "block",
+ [
+ VideoBlock.from_url("video/mp4", IMAGE_URL),
+ AudioBlock.from_url("audio/wav", IMAGE_URL),
+ AudioBlock.from_base64("audio/ogg", BASE64),
+ DocumentBlock.from_url("application/pdf", IMAGE_URL),
+ ],
+ ids=["video", "audio-url", "audio-ogg", "document-url"],
+)
+def test_unsupported_user_blocks_fail_explicitly(block: ContentBlock) -> None:
+ """Blocks without a content part fail, naming the block but not its
source."""
+ message = ChatMessage.user([TextBlock(text="hi"), block])
+
+ with pytest.raises(UnsupportedContentBlockError) as error:
+ convert_to_openai_message(message)
+
+ assert f"{block.type} block" in str(error.value)
+ assert "secret" not in str(error.value)
+ assert BASE64 not in str(error.value)
+
+
[email protected](
+ "role", [MessageRole.SYSTEM, MessageRole.ASSISTANT, MessageRole.TOOL]
+)
+def test_media_outside_user_messages_fails(role: MessageRole) -> None:
+ """Only user messages can carry media."""
+ message = ChatMessage(
+ role=role,
+ blocks=[TextBlock(text="see"), ImageBlock.from_url("image/png",
IMAGE_URL)],
+ )
+
+ with pytest.raises(UnsupportedContentBlockError, match="only user
messages"):
+ convert_to_openai_message(message)
diff --git
a/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal_live.py
b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal_live.py
new file mode 100644
index 00000000..73ed2f6a
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/openai/tests/test_openai_multimodal_live.py
@@ -0,0 +1,161 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""Real multimodal requests through the OpenAI Chat Completions connection.
+
+Checks that the provider accepts the content parts test_openai_multimodal.py
+pins. Skipped unless TEST_API_KEY is set; TEST_API_BASE_URL,
+TEST_MULTIMODAL_MODEL (default gpt-4o-mini) and TEST_AUDIO_MODEL (default
+gpt-4o-audio-preview) override the endpoint and models. Mirrors the Java
+OpenAIChatCompletionsMultimodalLiveTest.
+"""
+
+import base64
+import io
+import os
+import struct
+import wave
+import zlib
+
+import pytest
+
+from flink_agents.api.chat_message import (
+ AudioBlock,
+ Base64Source,
+ ChatMessage,
+ ContentBlock,
+ DocumentBlock,
+ ImageBlock,
+ TextBlock,
+)
+from flink_agents.integrations.chat_models.openai.openai_chat_model import (
+ OpenAIChatModelConnection,
+)
+
+pytestmark = [
+ pytest.mark.integration,
+ pytest.mark.skipif(
+ not os.environ.get("TEST_API_KEY"), reason="TEST_API_KEY is not set"
+ ),
+]
+
+MULTIMODAL_MODEL = os.environ.get("TEST_MULTIMODAL_MODEL") or "gpt-4o-mini"
+AUDIO_MODEL = os.environ.get("TEST_AUDIO_MODEL") or "gpt-4o-audio-preview"
+
+
+def _chat(model: str, *blocks: ContentBlock) -> str:
+ connection = OpenAIChatModelConnection(
+ api_key=os.environ["TEST_API_KEY"],
+ api_base_url=os.environ.get("TEST_API_BASE_URL"),
+ )
+ return connection.chat([ChatMessage.user(list(blocks))], model=model).text
+
+
+def _red_square_png() -> str:
+ """A 16x16 red PNG."""
+ rows = b"".join(b"\x00" + b"\xff\x00\x00" * 16 for _ in range(16))
+
+ def chunk(kind: bytes, data: bytes) -> bytes:
+ body = kind + data
+ return struct.pack(">I", len(data)) + body + struct.pack(">I",
zlib.crc32(body))
+
+ png = (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", struct.pack(">IIBBBBB", 16, 16, 8, 2, 0, 0, 0))
+ + chunk(b"IDAT", zlib.compress(rows))
+ + chunk(b"IEND", b"")
+ )
+ return base64.b64encode(png).decode()
+
+
+def _silent_wav() -> str:
+ """Half a second of 16 kHz mono 16-bit PCM silence."""
+ buffer = io.BytesIO()
+ with wave.open(buffer, "wb") as wav:
+ wav.setnchannels(1)
+ wav.setsampwidth(2)
+ wav.setframerate(16_000)
+ wav.writeframes(b"\x00\x00" * 8_000)
+ return base64.b64encode(buffer.getvalue()).decode()
+
+
+def _hello_pdf() -> str:
+ """A one-page PDF reading "Hello PDF", with a valid cross-reference
table."""
+ stream = "BT /F1 24 Tf 20 40 Td (Hello PDF) Tj ET"
+ objects = [
+ "<< /Type /Catalog /Pages 2 0 R >>",
+ "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 100] /Contents 4 0 R"
+ " /Resources << /Font << /F1 5 0 R >> >> >>",
+ f"<< /Length {len(stream)} >>\nstream\n{stream}\nendstream",
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ ]
+ pdf = "%PDF-1.4\n"
+ offsets = []
+ for number, body in enumerate(objects, start=1):
+ offsets.append(len(pdf))
+ pdf += f"{number} 0 obj\n{body}\nendobj\n"
+ xref = len(pdf)
+ pdf += f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n"
+ pdf += "".join(f"{offset:010d} 00000 n \n" for offset in offsets)
+ pdf += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
+ pdf += f"startxref\n{xref}\n%%EOF\n"
+ return base64.b64encode(pdf.encode("ascii")).decode()
+
+
+def test_image() -> None:
+ """A base64 image is accepted."""
+ answer = _chat(
+ MULTIMODAL_MODEL,
+ TextBlock(text="What color is this image? Answer in one word."),
+ ImageBlock.from_base64("image/png", _red_square_png()),
+ )
+ assert answer.strip()
+
+
+def test_audio() -> None:
+ """WAV audio is accepted."""
+ answer = _chat(
+ AUDIO_MODEL,
+ TextBlock(text="Describe this audio clip in one sentence."),
+ AudioBlock.from_base64("audio/wav", _silent_wav()),
+ )
+ assert answer.strip()
+
+
+def test_named_pdf() -> None:
+ """A named PDF is accepted."""
+ answer = _chat(
+ MULTIMODAL_MODEL,
+ TextBlock(text="What text does this PDF contain?"),
+ DocumentBlock(
+ media_type="application/pdf",
+ source=Base64Source(data=_hello_pdf()),
+ name="hello.pdf",
+ ),
+ )
+ assert answer.strip()
+
+
+def test_unnamed_pdf() -> None:
+ """A PDF without a name is accepted under the default file name."""
+ answer = _chat(
+ MULTIMODAL_MODEL,
+ TextBlock(text="What text does this PDF contain?"),
+ DocumentBlock.from_base64("application/pdf", _hello_pdf()),
+ )
+ assert answer.strip()
![https://example.com/cat.png?sig=secret"](https://example.com/cat.png?sig=secret"){kind=link}