This is an automated email from the ASF dual-hosted git repository.
wenjin272 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new e057ca661 [integration][java][python] Reject media blocks in providers
that send text only (#1186)
e057ca661 is described below
commit e057ca661d78bd77492655dfd075c959873dd196
Author: Edson <[email protected]>
AuthorDate: Wed Oct 7 09:32:34 2026 -0400
[integration][java][python] Reject media blocks in providers that send text
only (#1186)
Generated-by: Claude Code 2.1.259 (Claude Opus 5.5)
---
.../messages/UnsupportedContentBlockException.java | 16 +++++
docs/content/docs/development/chat_models.md | 10 ++-
.../anthropic/AnthropicChatModelConnection.java | 3 +
.../AnthropicChatModelConnectionTest.java | 20 ++++++
.../bedrock/BedrockChatModelConnection.java | 3 +
.../bedrock/BedrockChatModelConnectionTest.java | 20 ++++++
.../gemini/GeminiChatModelConnection.java | 3 +
.../gemini/GeminiChatModelConnectionTest.java | 20 ++++++
.../openai/OpenAIResponsesModelConnection.java | 3 +
.../openai/OpenAIResponsesModelConnectionTest.java | 20 ++++++
.../watsonx/WatsonxChatModelConnection.java | 5 +-
.../watsonx/WatsonxChatModelConnectionTest.java | 26 ++++++++
python/flink_agents/api/chat_message.py | 14 +++++
.../chat_models/anthropic/anthropic_chat_model.py | 8 ++-
.../chat_models/dashscope_chat_model.py | 8 ++-
.../tests/test_text_only_providers_reject_media.py | 72 ++++++++++++++++++++++
.../chat_models/watsonx/watsonx_chat_model.py | 31 ++++++----
17 files changed, 265 insertions(+), 17 deletions(-)
diff --git
a/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
b/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
index da58ccc47..857943541 100644
---
a/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
+++
b/api/src/main/java/org/apache/flink/agents/api/chat/messages/UnsupportedContentBlockException.java
@@ -18,6 +18,8 @@
package org.apache.flink.agents.api.chat.messages;
+import java.util.List;
+
/**
* Thrown when a chat model integration cannot send a {@link ContentBlock} to
its provider, such as
* a media type the provider does not accept or a media block in a message
role that only takes
@@ -50,4 +52,18 @@ public class UnsupportedContentBlockException extends
IllegalArgumentException {
return new UnsupportedContentBlockException(
provider + " cannot send " + description + ": " + reason +
".");
}
+
+ /**
+ * Throws for the first media block in {@code messages}. For integrations
that send text only,
+ * so that media fails explicitly rather than being dropped from the
request.
+ */
+ public static void rejectMedia(String provider, List<ChatMessage>
messages) {
+ for (ChatMessage message : messages) {
+ for (ContentBlock block : message.getBlocks()) {
+ if (block instanceof MediaBlock) {
+ throw forBlock(provider, block, "this integration sends
text only");
+ }
+ }
+ }
+ }
}
diff --git a/docs/content/docs/development/chat_models.md
b/docs/content/docs/development/chat_models.md
index 6b144d7b8..ab540e41c 100644
--- a/docs/content/docs/development/chat_models.md
+++ b/docs/content/docs/development/chat_models.md
@@ -63,8 +63,10 @@ a question followed by an image:
**Provider support:** The OpenAI Chat Completions integration, and the Azure
OpenAI
and vLLM integrations built on it, send media blocks to the model; see
[Multimodal Input](#multimodal-input) for what each block becomes. Ollama sends
-Base64 images; see its section. The other built-in integrations, including the
-OpenAI Responses integration, currently send only the text portion of a
message.
+Base64 images; see its section. The Anthropic, Gemini, Amazon Bedrock, IBM
+watsonx.ai, DashScope and OpenAI Responses integrations do not send media yet:
a
+message with a media block raises `UnsupportedContentBlockException` (Java) or
+`UnsupportedContentBlockError` (Python).
{{< tabs "Message content blocks" >}}
@@ -288,7 +290,7 @@ Model availability varies by AWS region and requires
explicit model access enabl
{{< /hint >}}
{{< hint warning >}}
-**Current limitations:** The integration uses text content blocks only.
Extended thinking / reasoning content blocks (e.g. Claude extended thinking),
citation blocks, and image / document content blocks are not yet supported.
+**Current limitations:** The integration uses text content blocks only.
Extended thinking / reasoning content blocks (e.g. Claude extended thinking),
citation blocks, and image / document content blocks are not yet supported; a
message with a media block raises `UnsupportedContentBlockException`.
{{< /hint >}}
### Anthropic
@@ -1023,6 +1025,8 @@ The Chat Completions connections (OpenAI, Azure OpenAI
and vLLM) send the media
A user message with only text blocks is still sent as a plain string. Media in
system, assistant or tool messages, audio or documents given by URL, other
audio types, and video raise `UnsupportedContentBlockException` (Java) or
`UnsupportedContentBlockError` (Python) before the request is sent. Whether a
model accepts a given part depends on the model and the server.
+The other built-in providers (Anthropic, Gemini, Amazon Bedrock, IBM
watsonx.ai, DashScope and the OpenAI Responses API) do not send media yet: a
message with a media block raises the same error instead of being sent as its
text.
+
#### Responses API
{{< hint info >}}
diff --git
a/integrations/chat-models/anthropic/src/main/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnection.java
b/integrations/chat-models/anthropic/src/main/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnection.java
index ff8022d86..f4551659e 100644
---
a/integrations/chat-models/anthropic/src/main/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnection.java
+++
b/integrations/chat-models/anthropic/src/main/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnection.java
@@ -39,6 +39,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -422,6 +423,8 @@ public class AnthropicChatModelConnection extends
BaseChatModelConnection {
List<org.apache.flink.agents.api.tools.Tool> tools,
Map<String, Object> modelParams,
Object outputSchema) {
+ // Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockException.rejectMedia("Anthropic", messages);
try {
BuiltRequest built = buildRequest(messages, tools, modelParams,
outputSchema);
Message response = client.messages().create(built.params);
diff --git
a/integrations/chat-models/anthropic/src/test/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnectionTest.java
b/integrations/chat-models/anthropic/src/test/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnectionTest.java
index 4b22b40f3..98fb93dd0 100644
---
a/integrations/chat-models/anthropic/src/test/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnectionTest.java
+++
b/integrations/chat-models/anthropic/src/test/java/org/apache/flink/agents/integrations/chatmodels/anthropic/AnthropicChatModelConnectionTest.java
@@ -31,7 +31,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
import org.apache.flink.agents.api.tools.Tool;
@@ -1059,4 +1061,22 @@ class AnthropicChatModelConnectionTest {
return ToolResponse.success(null);
}
}
+
+ @Test
+ @DisplayName("Media blocks fail explicitly instead of being dropped")
+ void testMediaBlocksFailExplicitly() {
+ List<ChatMessage> messages =
+ List.of(
+ ChatMessage.user(
+ List.of(
+
org.apache.flink.agents.api.chat.messages.TextBlock.of(
+ "Describe this"),
+ ImageBlock.fromBase64("image/png",
"aGVsbG8="))));
+
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>(), null))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(
+ "Anthropic cannot send an image block (image/png,
base64 source): this"
+ + " integration sends text only.");
+ }
}
diff --git
a/integrations/chat-models/bedrock/src/main/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnection.java
b/integrations/chat-models/bedrock/src/main/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnection.java
index 386d211a0..a6a295a9e 100644
---
a/integrations/chat-models/bedrock/src/main/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnection.java
+++
b/integrations/chat-models/bedrock/src/main/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnection.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.agents.api.RetryExecutor;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -271,6 +272,8 @@ public class BedrockChatModelConnection extends
BaseChatModelConnection {
List<Tool> tools,
Map<String, Object> modelParams,
Object outputSchema) {
+ // Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockException.rejectMedia("Amazon Bedrock",
messages);
ConverseRequest request = buildRequest(messages, tools, modelParams,
outputSchema);
String modelId = request.modelId();
diff --git
a/integrations/chat-models/bedrock/src/test/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnectionTest.java
b/integrations/chat-models/bedrock/src/test/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnectionTest.java
index 3cd315c9b..dc63ab320 100644
---
a/integrations/chat-models/bedrock/src/test/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnectionTest.java
+++
b/integrations/chat-models/bedrock/src/test/java/org/apache/flink/agents/integrations/chatmodels/bedrock/BedrockChatModelConnectionTest.java
@@ -23,7 +23,10 @@ import com.fasterxml.jackson.annotation.JsonValue;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -671,4 +674,21 @@ class BedrockChatModelConnectionTest {
.outputConfig())
.isNull();
}
+
+ @Test
+ @DisplayName("Media blocks fail explicitly instead of being dropped")
+ void testMediaBlocksFailExplicitly() {
+ List<ChatMessage> messages =
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of("Describe this"),
+ ImageBlock.fromBase64("image/png",
"aGVsbG8="))));
+
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>(), null))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(
+ "Amazon Bedrock cannot send an image block (image/png,
base64 source): this"
+ + " integration sends text only.");
+ }
}
diff --git
a/integrations/chat-models/gemini/src/main/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnection.java
b/integrations/chat-models/gemini/src/main/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnection.java
index 0a7c5a732..038559783 100644
---
a/integrations/chat-models/gemini/src/main/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnection.java
+++
b/integrations/chat-models/gemini/src/main/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnection.java
@@ -36,6 +36,7 @@ import com.google.genai.types.Part;
import com.google.genai.types.Tool;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -278,6 +279,8 @@ public class GeminiChatModelConnection extends
BaseChatModelConnection {
List<org.apache.flink.agents.api.tools.Tool> tools,
Map<String, Object> arguments,
Object outputSchema) {
+ // Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockException.rejectMedia("Gemini", messages);
Map<String, Object> args = arguments != null ? new
HashMap<>(arguments) : new HashMap<>();
Object modelObj = args.remove("model");
diff --git
a/integrations/chat-models/gemini/src/test/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnectionTest.java
b/integrations/chat-models/gemini/src/test/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnectionTest.java
index ea27d4ca1..bae581dc4 100644
---
a/integrations/chat-models/gemini/src/test/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnectionTest.java
+++
b/integrations/chat-models/gemini/src/test/java/org/apache/flink/agents/integrations/chatmodels/gemini/GeminiChatModelConnectionTest.java
@@ -31,7 +31,10 @@ import com.google.genai.types.FunctionDeclaration;
import com.google.genai.types.GenerateContentConfig;
import com.google.genai.types.Part;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -990,4 +993,21 @@ class GeminiChatModelConnectionTest {
assertThat(judged.get()).isEqualTo(named);
}
}
+
+ @Test
+ @DisplayName("Media blocks fail explicitly instead of being dropped")
+ void testMediaBlocksFailExplicitly() {
+ List<ChatMessage> messages =
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of("Describe this"),
+ ImageBlock.fromBase64("image/png",
"aGVsbG8="))));
+
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>(), null))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(
+ "Gemini cannot send an image block (image/png, base64
source): this"
+ + " integration sends text only.");
+ }
}
diff --git
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnection.java
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnection.java
index 080d1e183..8abc8539c 100644
---
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnection.java
+++
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnection.java
@@ -30,6 +30,7 @@ import com.openai.models.ReasoningEffort;
import com.openai.models.responses.*;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -142,6 +143,8 @@ public class OpenAIResponsesModelConnection extends
BaseChatModelConnection {
List<ChatMessage> messages,
List<org.apache.flink.agents.api.tools.Tool> tools,
Map<String, Object> modelParams) {
+ // Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockException.rejectMedia("OpenAI Responses",
messages);
ResponseCreateParams params = buildRequest(messages, tools,
modelParams);
Response response = client.responses().create(params);
ChatMessage result = convertResponse(response);
diff --git
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnectionTest.java
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnectionTest.java
index ac018f2df..a86806981 100644
---
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnectionTest.java
+++
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/OpenAIResponsesModelConnectionTest.java
@@ -20,7 +20,10 @@ package
org.apache.flink.agents.integrations.chatmodels.openai;
import com.openai.errors.BadRequestException;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -284,4 +287,21 @@ class OpenAIResponsesModelConnectionTest {
.hasMessageContaining(FakeOpenAIErrorEndpoint.ERROR_MESSAGE);
}
}
+
+ @Test
+ @DisplayName("Media blocks fail explicitly instead of being dropped")
+ void testMediaBlocksFailExplicitly() {
+ List<ChatMessage> messages =
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of("Describe this"),
+ ImageBlock.fromBase64("image/png",
"aGVsbG8="))));
+
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>()))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(
+ "OpenAI Responses cannot send an image block
(image/png, base64 source): this"
+ + " integration sends text only.");
+ }
}
diff --git
a/integrations/chat-models/watsonx/src/main/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnection.java
b/integrations/chat-models/watsonx/src/main/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnection.java
index 17aed4fae..f59ef2c01 100644
---
a/integrations/chat-models/watsonx/src/main/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnection.java
+++
b/integrations/chat-models/watsonx/src/main/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnection.java
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.victools.jsonschema.generator.Option;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -269,7 +270,7 @@ public class WatsonxChatModelConnection extends
BaseChatModelConnection {
@Override
public ChatMessage chat(
List<ChatMessage> messages, List<Tool> tools, Map<String, Object>
modelParams) {
- return doChat(messages, tools, modelParams, null);
+ return chat(messages, tools, modelParams, null);
}
/**
@@ -284,6 +285,8 @@ public class WatsonxChatModelConnection extends
BaseChatModelConnection {
List<Tool> tools,
Map<String, Object> modelParams,
Object outputSchema) {
+ // Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockException.rejectMedia("IBM watsonx.ai",
messages);
return doChat(messages, tools, modelParams, outputSchema);
}
diff --git
a/integrations/chat-models/watsonx/src/test/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnectionTest.java
b/integrations/chat-models/watsonx/src/test/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnectionTest.java
index c136f0c67..1624aa8d1 100644
---
a/integrations/chat-models/watsonx/src/test/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnectionTest.java
+++
b/integrations/chat-models/watsonx/src/test/java/org/apache/flink/agents/integrations/chatmodels/watsonx/WatsonxChatModelConnectionTest.java
@@ -26,7 +26,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.chat.messages.ImageBlock;
import org.apache.flink.agents.api.chat.messages.MessageRole;
+import org.apache.flink.agents.api.chat.messages.TextBlock;
+import
org.apache.flink.agents.api.chat.messages.UnsupportedContentBlockException;
import org.apache.flink.agents.api.chat.model.BaseChatModelConnection;
import org.apache.flink.agents.api.resource.ResourceContext;
import org.apache.flink.agents.api.resource.ResourceDescriptor;
@@ -42,6 +45,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -953,4 +957,26 @@ class WatsonxChatModelConnectionTest {
server.stop(0);
}
}
+
+ @Test
+ @DisplayName("Media blocks fail explicitly instead of being dropped")
+ void testMediaBlocksFailExplicitly() {
+ List<ChatMessage> messages =
+ List.of(
+ ChatMessage.user(
+ List.of(
+ TextBlock.of("Describe this"),
+ ImageBlock.fromBase64("image/png",
"aGVsbG8="))));
+
+ String expected =
+ "IBM watsonx.ai cannot send an image block (image/png, base64
source): this"
+ + " integration sends text only.";
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>(), null))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(expected);
+ // The three-argument overload is what BaseChatModelSetup calls.
+ assertThatThrownBy(() -> connection().chat(messages, List.of(), new
HashMap<>()))
+ .isInstanceOf(UnsupportedContentBlockException.class)
+ .hasMessage(expected);
+ }
}
diff --git a/python/flink_agents/api/chat_message.py
b/python/flink_agents/api/chat_message.py
index fbc95c1a0..7563b2add 100644
--- a/python/flink_agents/api/chat_message.py
+++ b/python/flink_agents/api/chat_message.py
@@ -201,6 +201,20 @@ class UnsupportedContentBlockError(ValueError):
description += f" ({block.media_type}, {block.source.type} source)"
return cls(f"{provider} cannot send {description}: {reason}.")
+ @classmethod
+ def reject_media(cls, provider: str, messages: Sequence["ChatMessage"]) ->
None:
+ """Raise for the first media block in the messages.
+
+ For integrations that send text only, so that media fails explicitly
+ rather than being dropped from the request.
+ """
+ for message in messages:
+ for block in message.blocks:
+ if isinstance(block, MediaBlock):
+ raise cls.for_block(
+ provider, block, "this integration sends text only"
+ )
+
def _blocks_of(text: str) -> List[ContentBlock]:
"""An empty text becomes an empty block list rather than an empty text
block."""
diff --git
a/python/flink_agents/integrations/chat_models/anthropic/anthropic_chat_model.py
b/python/flink_agents/integrations/chat_models/anthropic/anthropic_chat_model.py
index b648c75d6..4ecee69b8 100644
---
a/python/flink_agents/integrations/chat_models/anthropic/anthropic_chat_model.py
+++
b/python/flink_agents/integrations/chat_models/anthropic/anthropic_chat_model.py
@@ -26,7 +26,11 @@ from pydantic import BaseModel, Field, PrivateAttr
from typing_extensions import override
from flink_agents.api.agents.types import OutputSchema, render_output_schema
-from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.api.chat_message import (
+ ChatMessage,
+ MessageRole,
+ UnsupportedContentBlockError,
+)
from flink_agents.api.chat_models.chat_model import (
BaseChatModelConnection,
BaseChatModelSetup,
@@ -395,6 +399,8 @@ class AnthropicChatModelConnection(BaseChatModelConnection):
ChatMessage
Model response message
"""
+ # Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockError.reject_media("Anthropic", messages)
anthropic_tools = None
if tools is not None:
anthropic_tools = [
diff --git
a/python/flink_agents/integrations/chat_models/dashscope_chat_model.py
b/python/flink_agents/integrations/chat_models/dashscope_chat_model.py
index 9eb993994..70efbbb65 100644
--- a/python/flink_agents/integrations/chat_models/dashscope_chat_model.py
+++ b/python/flink_agents/integrations/chat_models/dashscope_chat_model.py
@@ -26,7 +26,11 @@ from pydantic import BaseModel, Field
from typing_extensions import override
from flink_agents.api.agents.types import OutputSchema, render_output_schema
-from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.api.chat_message import (
+ ChatMessage,
+ MessageRole,
+ UnsupportedContentBlockError,
+)
from flink_agents.api.chat_models.chat_model import (
BaseChatModelConnection,
BaseChatModelSetup,
@@ -231,6 +235,8 @@ class DashScopeChatModelConnection(BaseChatModelConnection):
ChatMessage
Model response message.
"""
+ # Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockError.reject_media("DashScope", messages)
dashscope_messages = self.__convert_to_dashscope_messages(messages)
dashscope_tools: List[Dict[str, Any]] | None = (
diff --git
a/python/flink_agents/integrations/chat_models/tests/test_text_only_providers_reject_media.py
b/python/flink_agents/integrations/chat_models/tests/test_text_only_providers_reject_media.py
new file mode 100644
index 000000000..e6c8ceb58
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/tests/test_text_only_providers_reject_media.py
@@ -0,0 +1,72 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""Providers that send text only fail on media instead of dropping it.
+
+Covers the connections not yet migrated to multimodal requests (#1059). Each
+check runs before any client call, so no service is contacted.
+"""
+
+from typing import Callable, Dict
+
+import pytest
+
+from flink_agents.api.chat_message import (
+ ChatMessage,
+ ImageBlock,
+ TextBlock,
+ UnsupportedContentBlockError,
+)
+from flink_agents.api.chat_models.chat_model import BaseChatModelConnection
+from flink_agents.integrations.chat_models.anthropic.anthropic_chat_model
import (
+ AnthropicChatModelConnection,
+)
+from flink_agents.integrations.chat_models.dashscope_chat_model import (
+ DashScopeChatModelConnection,
+)
+from flink_agents.integrations.chat_models.watsonx.watsonx_chat_model import (
+ WatsonxChatModelConnection,
+)
+
+CONNECTIONS: Dict[str, Callable[[], BaseChatModelConnection]] = {
+ "Anthropic": lambda: AnthropicChatModelConnection(api_key="fake-key"),
+ "DashScope": lambda: DashScopeChatModelConnection(api_key="fake-key"),
+ "IBM watsonx.ai": lambda: WatsonxChatModelConnection(
+ url="https://us-south.ml.cloud.ibm.com";,
+ api_key="fake-key",
+ project_id="fake-project",
+ ),
+}
+
+
[email protected]("provider", list(CONNECTIONS))
+def test_media_blocks_fail_explicitly(provider: str) -> None:
+ """A media block raises instead of being dropped from the request."""
+ message = ChatMessage.user(
+ [
+ TextBlock(text="Describe this"),
+ ImageBlock.from_base64("image/png", "aGVsbG8="),
+ ]
+ )
+
+ with pytest.raises(UnsupportedContentBlockError) as error:
+ CONNECTIONS[provider]().chat([message], model="m")
+
+ assert str(error.value) == (
+ f"{provider} cannot send an image block (image/png, base64 source): "
+ "this integration sends text only."
+ )
diff --git
a/python/flink_agents/integrations/chat_models/watsonx/watsonx_chat_model.py
b/python/flink_agents/integrations/chat_models/watsonx/watsonx_chat_model.py
index d76590598..07face20d 100644
--- a/python/flink_agents/integrations/chat_models/watsonx/watsonx_chat_model.py
+++ b/python/flink_agents/integrations/chat_models/watsonx/watsonx_chat_model.py
@@ -32,7 +32,11 @@ from pydantic import BaseModel, Field, PrivateAttr
from typing_extensions import override
from flink_agents.api.agents.types import OutputSchema, render_output_schema
-from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.api.chat_message import (
+ ChatMessage,
+ MessageRole,
+ UnsupportedContentBlockError,
+)
from flink_agents.api.chat_models.chat_model import (
BaseChatModelConnection,
BaseChatModelSetup,
@@ -49,16 +53,19 @@ RETRYABLE_STATUS_CODES = frozenset({408, 429, 500, 502,
503, 504})
REQUEST_OWNED_PARAMS = frozenset(
{"model_id", "messages", "tools", "project_id", "space_id"}
)
-RESERVED_ADDITIONAL_KWARGS = frozenset(
- {
- "model",
- "temperature",
- "max_tokens",
- "extract_reasoning",
- "tool_choice",
- "tool_choice_option",
- }
-) | REQUEST_OWNED_PARAMS
+RESERVED_ADDITIONAL_KWARGS = (
+ frozenset(
+ {
+ "model",
+ "temperature",
+ "max_tokens",
+ "extract_reasoning",
+ "tool_choice",
+ "tool_choice_option",
+ }
+ )
+ | REQUEST_OWNED_PARAMS
+)
def _normalize(value: str | None) -> str | None:
@@ -441,6 +448,8 @@ class WatsonxChatModelConnection(BaseChatModelConnection):
``extra_args["finish_reason"]``; the key is absent when the provider
reports none.
"""
+ # Media blocks are not sent yet; fail rather than drop them (#1059).
+ UnsupportedContentBlockError.reject_media("IBM watsonx.ai", messages)
model_name = kwargs.pop("model", DEFAULT_MODEL)
extract_reasoning = bool(kwargs.pop("extract_reasoning", False))
tool_choice = kwargs.pop("tool_choice", None)
