This is an automated email from the ASF dual-hosted git repository.
wenjin272 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 6d7afb0f [integration] Add out of the box vLLM chat model support for
Java and Python (#945)
6d7afb0f is described below
commit 6d7afb0ffcb43c18285d91d349b3796f49595966
Author: Edson <[email protected]>
AuthorDate: Tue Aug 11 23:37:11 2026 -0400
[integration] Add out of the box vLLM chat model support for Java and
Python (#945)
---
.../flink/agents/api/resource/ResourceName.java | 12 ++
.../org/apache/flink/agents/api/yaml/Aliases.java | 4 +
.../apache/flink/agents/api/yaml/AliasesTest.java | 14 ++
docs/content/docs/development/chat_models.md | 113 ++++++++++++++++
docs/content/docs/development/yaml.md | 1 +
docs/content/docs/faq/faq.md | 1 +
.../chatmodels/openai/VLLMChatModelConnection.java | 99 ++++++++++++++
.../chatmodels/openai/VLLMChatModelSetup.java | 58 +++++++++
.../openai/VLLMChatModelConnectionTest.java | 144 +++++++++++++++++++++
.../chatmodels/openai/VLLMChatModelSetupTest.java | 80 ++++++++++++
python/flink_agents/api/resource.py | 8 ++
python/flink_agents/api/yaml/aliases.py | 4 +
python/flink_agents/api/yaml/tests/test_aliases.py | 20 ++-
.../integrations/chat_models/vllm/__init__.py | 17 +++
.../chat_models/vllm/tests/__init__.py | 17 +++
.../chat_models/vllm/tests/test_vllm_chat_model.py | 131 +++++++++++++++++++
.../chat_models/vllm/vllm_chat_model.py | 106 +++++++++++++++
17 files changed, 828 insertions(+), 1 deletion(-)
diff --git
a/api/src/main/java/org/apache/flink/agents/api/resource/ResourceName.java
b/api/src/main/java/org/apache/flink/agents/api/resource/ResourceName.java
index b1fe9df0..ccacfef3 100644
--- a/api/src/main/java/org/apache/flink/agents/api/resource/ResourceName.java
+++ b/api/src/main/java/org/apache/flink/agents/api/resource/ResourceName.java
@@ -95,6 +95,12 @@ public final class ResourceName {
public static final String AZURE_OPENAI_SETUP =
"org.apache.flink.agents.integrations.chatmodels.openai.AzureOpenAIChatModelSetup";
+ // vLLM (OpenAI-compatible)
+ public static final String VLLM_CONNECTION =
+
"org.apache.flink.agents.integrations.chatmodels.openai.VLLMChatModelConnection";
+ public static final String VLLM_SETUP =
+
"org.apache.flink.agents.integrations.chatmodels.openai.VLLMChatModelSetup";
+
// Python Wrapper
public static final String PYTHON_WRAPPER_CONNECTION =
"org.apache.flink.agents.api.chat.model.python.PythonChatModelConnection";
@@ -134,6 +140,12 @@ public final class ResourceName {
public static final String TONGYI_SETUP =
"flink_agents.integrations.chat_models.tongyi_chat_model.TongyiChatModelSetup";
+ // vLLM (OpenAI-compatible)
+ public static final String VLLM_CONNECTION =
+
"flink_agents.integrations.chat_models.vllm.vllm_chat_model.VLLMChatModelConnection";
+ public static final String VLLM_SETUP =
+
"flink_agents.integrations.chat_models.vllm.vllm_chat_model.VLLMChatModelSetup";
+
private Python() {}
}
diff --git a/api/src/main/java/org/apache/flink/agents/api/yaml/Aliases.java
b/api/src/main/java/org/apache/flink/agents/api/yaml/Aliases.java
index 8043964c..6798ef87 100644
--- a/api/src/main/java/org/apache/flink/agents/api/yaml/Aliases.java
+++ b/api/src/main/java/org/apache/flink/agents/api/yaml/Aliases.java
@@ -90,12 +90,14 @@ public final class Aliases {
chatConnJava.put("azure_openai",
ResourceName.ChatModel.AZURE_OPENAI_CONNECTION);
chatConnJava.put("azure", ResourceName.ChatModel.AZURE_CONNECTION);
chatConnJava.put("bedrock", ResourceName.ChatModel.BEDROCK_CONNECTION);
+ chatConnJava.put("vllm", ResourceName.ChatModel.VLLM_CONNECTION);
Map<String, String> chatConnPython = new HashMap<>();
chatConnPython.put("ollama",
ResourceName.ChatModel.Python.OLLAMA_CONNECTION);
chatConnPython.put("openai",
ResourceName.ChatModel.Python.OPENAI_COMPLETIONS_CONNECTION);
chatConnPython.put("anthropic",
ResourceName.ChatModel.Python.ANTHROPIC_CONNECTION);
chatConnPython.put("tongyi",
ResourceName.ChatModel.Python.TONGYI_CONNECTION);
chatConnPython.put("azure_openai",
ResourceName.ChatModel.Python.AZURE_OPENAI_CONNECTION);
+ chatConnPython.put("vllm",
ResourceName.ChatModel.Python.VLLM_CONNECTION);
ca.put(ResourceType.CHAT_MODEL_CONNECTION,
buildLangBuckets(chatConnJava, chatConnPython));
// CHAT_MODEL
@@ -108,12 +110,14 @@ public final class Aliases {
chatJava.put("azure_openai",
ResourceName.ChatModel.AZURE_OPENAI_SETUP);
chatJava.put("azure", ResourceName.ChatModel.AZURE_SETUP);
chatJava.put("bedrock", ResourceName.ChatModel.BEDROCK_SETUP);
+ chatJava.put("vllm", ResourceName.ChatModel.VLLM_SETUP);
Map<String, String> chatPython = new HashMap<>();
chatPython.put("ollama", ResourceName.ChatModel.Python.OLLAMA_SETUP);
chatPython.put("openai",
ResourceName.ChatModel.Python.OPENAI_COMPLETIONS_SETUP);
chatPython.put("anthropic",
ResourceName.ChatModel.Python.ANTHROPIC_SETUP);
chatPython.put("tongyi", ResourceName.ChatModel.Python.TONGYI_SETUP);
chatPython.put("azure_openai",
ResourceName.ChatModel.Python.AZURE_OPENAI_SETUP);
+ chatPython.put("vllm", ResourceName.ChatModel.Python.VLLM_SETUP);
ca.put(ResourceType.CHAT_MODEL, buildLangBuckets(chatJava,
chatPython));
// EMBEDDING_MODEL_CONNECTION
diff --git
a/api/src/test/java/org/apache/flink/agents/api/yaml/AliasesTest.java
b/api/src/test/java/org/apache/flink/agents/api/yaml/AliasesTest.java
index 3f125ffb..ab8d723e 100644
--- a/api/src/test/java/org/apache/flink/agents/api/yaml/AliasesTest.java
+++ b/api/src/test/java/org/apache/flink/agents/api/yaml/AliasesTest.java
@@ -89,6 +89,20 @@ class AliasesTest {
assertThat(mem0).isEqualTo(ResourceName.VectorStore.Python.MEM0_VECTOR_STORE);
}
+ @Test
+ void clazzAliasCoversChatModelVLLMInBothLanguages() {
+ String javaConn =
+ Aliases.resolveClazz("vllm",
ResourceType.CHAT_MODEL_CONNECTION, Language.JAVA);
+ assertThat(javaConn).isEqualTo(ResourceName.ChatModel.VLLM_CONNECTION);
+ String javaSetup = Aliases.resolveClazz("vllm",
ResourceType.CHAT_MODEL, Language.JAVA);
+ assertThat(javaSetup).isEqualTo(ResourceName.ChatModel.VLLM_SETUP);
+ String pythonConn =
+ Aliases.resolveClazz("vllm",
ResourceType.CHAT_MODEL_CONNECTION, Language.PYTHON);
+
assertThat(pythonConn).isEqualTo(ResourceName.ChatModel.Python.VLLM_CONNECTION);
+ String pythonSetup = Aliases.resolveClazz("vllm",
ResourceType.CHAT_MODEL, Language.PYTHON);
+
assertThat(pythonSetup).isEqualTo(ResourceName.ChatModel.Python.VLLM_SETUP);
+ }
+
@Test
void clazzAliasMissPassesThrough() {
String fqn =
diff --git a/docs/content/docs/development/chat_models.md
b/docs/content/docs/development/chat_models.md
index f9b53ac9..2fb2d19b 100644
--- a/docs/content/docs/development/chat_models.md
+++ b/docs/content/docs/development/chat_models.md
@@ -1207,6 +1207,119 @@ Some popular options include:
Model availability and specifications may change. Always check the official
DashScope documentation for the latest information before implementing in
production.
{{< /hint >}}
+### vLLM
+
+[vLLM](https://docs.vllm.ai) serves open-weight models behind an
OpenAI-compatible API and is a popular choice for self-hosted production
deployments. Flink Agents provides a dedicated connection that reuses the
OpenAI integration with vLLM-friendly defaults, in both Java and Python.
+
+#### Prerequisites
+
+1. Install vLLM and start a server. For agent use, enable automatic tool
calling — Flink Agents sends tools without a named `tool_choice`, so the server
needs `--enable-auto-tool-choice` plus a model-specific `--tool-call-parser`
(for Qwen2.5, vLLM recommends `hermes`):
+
+ ```bash
+ vllm serve Qwen/Qwen2.5-7B-Instruct --enable-auto-tool-choice
--tool-call-parser hermes
+ ```
+
+ Without these flags the server can chat but tool calls are not parsed into
the OpenAI `tool_calls` field, so the model cannot drive an agent's tools. The
parser is model-specific; see the [vLLM tool calling
docs](https://docs.vllm.ai/en/stable/features/tool_calling/).
+2. By default the server listens on `http://localhost:8000` and requires no
API key. If the server is started with `--api-key`, pass the same key in the
connection.
+
+#### VLLMChatModelConnection Parameters
+
+{{< tabs "VLLMChatModelConnection Parameters" >}}
+
+{{< tab "Python" >}}
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_base_url` | str | `"http://localhost:8000/v1"` | vLLM server URL |
+| `api_key` | str | `"EMPTY"` | Only needed when the server is started with
`--api-key`; the placeholder default works for unauthenticated servers. Unlike
the OpenAI connection, the `OPENAI_API_KEY` / `OPENAI_API_BASE_URL` environment
variables are not consulted |
+| `timeout` | float | `60.0` | HTTP request timeout in seconds |
+| `max_retries` | int | `3` | Maximum number of API retries |
+
+{{< /tab >}}
+
+{{< tab "Java" >}}
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_base_url` | String | `"http://localhost:8000/v1"` | vLLM server URL |
+| `api_key` | String | `"EMPTY"` | Only needed when the server is started with
`--api-key`; the placeholder default works for unauthenticated servers |
+| `timeout` | int | SDK default | Seconds before an API call times out |
+| `max_retries` | int | SDK default | Retry attempts on failure |
+
+{{< /tab >}}
+
+{{< /tabs >}}
+
+#### VLLMChatModelSetup Parameters
+
+Same as the [OpenAI Completions setup](#openaicompletionssetup-parameters),
with one difference: `model` is **required** and has no default — it must match
the model name served by the vLLM server (see `vllm serve <model>`, or query
`GET /v1/models`).
+
+#### Usage Example
+
+{{< tabs "vLLM Usage Example" >}}
+
+{{< tab "Python" >}}
+
+```python
+class MyAgent(Agent):
+
+ @chat_model_connection
+ @staticmethod
+ def vllm_connection() -> ResourceDescriptor:
+ return ResourceDescriptor(
+ clazz=ResourceName.ChatModel.VLLM_CONNECTION,
+ api_base_url="http://localhost:8000/v1";,
+ )
+
+ @chat_model_setup
+ @staticmethod
+ def vllm_chat_model() -> ResourceDescriptor:
+ return ResourceDescriptor(
+ clazz=ResourceName.ChatModel.VLLM_SETUP,
+ connection="vllm_connection",
+ model="Qwen/Qwen2.5-7B-Instruct",
+ temperature=0.3,
+ )
+
+ ...
+```
+
+{{< /tab >}}
+
+{{< tab "Java" >}}
+
+```java
+public class MyAgent extends Agent {
+
+ @ChatModelConnection
+ public static ResourceDescriptor vllmConnection() {
+ return ResourceDescriptor.Builder.newBuilder(
+ ResourceName.ChatModel.VLLM_CONNECTION)
+ .addInitialArgument("api_base_url", "http://localhost:8000/v1";)
+ .build();
+ }
+
+ @ChatModelSetup
+ public static ResourceDescriptor vllmChatModel() {
+ return
ResourceDescriptor.Builder.newBuilder(ResourceName.ChatModel.VLLM_SETUP)
+ .addInitialArgument("connection", "vllmConnection")
+ .addInitialArgument("model", "Qwen/Qwen2.5-7B-Instruct")
+ .addInitialArgument("temperature", 0.3d)
+ .build();
+ }
+
+ // ...
+}
+```
+
+{{< /tab >}}
+
+{{< /tabs >}}
+
+#### Available Models
+
+A vLLM server serves the model(s) it was started with. Query `GET /v1/models`
on the server to list them; the `model` value in the setup must match one of
the returned names.
+
## Using Cross-Language Providers
Flink Agents supports cross-language chat model integration, allowing you to
use chat models implemented in one language (Java or Python) from agents
written in the other language. This is particularly useful when a chat model
provider is only available in one language (e.g., Tongyi is currently
Python-only).
diff --git a/docs/content/docs/development/yaml.md
b/docs/content/docs/development/yaml.md
index 7e8f69e2..56684d63 100644
--- a/docs/content/docs/development/yaml.md
+++ b/docs/content/docs/development/yaml.md
@@ -533,6 +533,7 @@ Common chat-model aliases:
| `azure` | — | Azure AI (Java)
|
| `bedrock` | — | Bedrock (Java)
|
| `tongyi` | Tongyi (Python) | —
|
+| `vllm` | vLLM (Python) | vLLM (Java)
|
Embedding-model aliases (apply to both `embedding_model_connections` and
`embedding_model_setups`):
diff --git a/docs/content/docs/faq/faq.md b/docs/content/docs/faq/faq.md
index 4b9fd7e7..51f20283 100644
--- a/docs/content/docs/faq/faq.md
+++ b/docs/content/docs/faq/faq.md
@@ -105,6 +105,7 @@ Flink Agents provides built-in integrations for many
ecosystem providers. Some i
| [Ollama]({{< ref "docs/development/chat_models#ollama" >}}) | ✅ | ✅ |
| [OpenAI]({{< ref "docs/development/chat_models#openai" >}}) | ✅ | ✅ |
| [Tongyi (DashScope)]({{< ref "docs/development/chat_models#tongyi-dashscope"
>}}) | ✅ | ❌ |
+| [vLLM]({{< ref "docs/development/chat_models#vllm" >}}) | ✅ | ✅ |
**Embedding Models**
diff --git
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnection.java
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnection.java
new file mode 100644
index 00000000..69b84730
--- /dev/null
+++
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnection.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Chat model connection for a <a href="https://docs.vllm.ai";>vLLM</a> server.
+ *
+ * <p>vLLM exposes an OpenAI-compatible API, so this connection reuses {@link
+ * OpenAICompletionsConnection} with vLLM-friendly defaults:
+ *
+ * <ul>
+ * <li><b>api_base_url</b> (optional): defaults to {@code
http://localhost:8000/v1}, the default
+ * address of {@code vllm serve}
+ * <li><b>api_key</b> (optional): defaults to a placeholder value, since
vLLM servers started
+ * without {@code --api-key} do not require a credential (the underlying
OpenAI SDK requires a
+ * non-empty key, but the server ignores it). Set it explicitly when the
server is started
+ * with {@code --api-key}.
+ * </ul>
+ *
+ * <p>All other connection arguments ({@code timeout}, {@code max_retries},
{@code default_headers},
+ * {@code model}) behave exactly as in {@link OpenAICompletionsConnection}.
+ *
+ * <p>Example usage:
+ *
+ * <pre>{@code
+ * public class MyAgent extends Agent {
+ * @ChatModelConnection
+ * public static ResourceDescriptor vllm() {
+ * return
ResourceDescriptor.Builder.newBuilder(VLLMChatModelConnection.class.getName())
+ * .addInitialArgument("api_base_url",
"http://my-vllm-host:8000/v1";)
+ * .build();
+ * }
+ * }
+ * }</pre>
+ */
+public class VLLMChatModelConnection extends OpenAICompletionsConnection {
+
+ /** Default base URL of a local {@code vllm serve} instance. */
+ public static final String DEFAULT_VLLM_API_BASE_URL =
"http://localhost:8000/v1";;
+
+ /**
+ * Placeholder credential used when the vLLM server is started without
{@code --api-key}. The
+ * OpenAI SDK requires a non-empty key, but the server ignores its value.
+ */
+ public static final String DEFAULT_VLLM_API_KEY = "EMPTY";
+
+ public VLLMChatModelConnection(ResourceDescriptor descriptor,
ResourceContext resourceContext) {
+ super(withVLLMDefaults(descriptor), resourceContext);
+ }
+
+ /**
+ * vLLM implements the OpenAI {@code json_schema} response format for
whatever model it serves
+ * (via guided decoding), so structured-output capability does not depend
on OpenAI model names
+ * — the inherited allowlist would wrongly reject served models such as
{@code
+ * Qwen/Qwen2.5-7B-Instruct}. See <a
+ *
href="https://docs.vllm.ai/en/stable/features/structured_outputs.html";>vLLM
structured
+ * outputs</a>.
+ */
+ @Override
+ protected boolean supportsNativeStructuredOutput(String effectiveModel) {
+ return effectiveModel != null && !effectiveModel.isBlank();
+ }
+
+ // Package-visible so tests can assert on the descriptor the defaults
produce.
+ static ResourceDescriptor withVLLMDefaults(ResourceDescriptor descriptor) {
+ Map<String, Object> arguments = new
HashMap<>(descriptor.getInitialArguments());
+ Object apiBaseUrl = arguments.get("api_base_url");
+ if (apiBaseUrl == null
+ || (apiBaseUrl instanceof String && ((String)
apiBaseUrl).isBlank())) {
+ arguments.put("api_base_url", DEFAULT_VLLM_API_BASE_URL);
+ }
+ Object apiKey = arguments.get("api_key");
+ if (apiKey == null || (apiKey instanceof String && ((String)
apiKey).isBlank())) {
+ arguments.put("api_key", DEFAULT_VLLM_API_KEY);
+ }
+ return new ResourceDescriptor(descriptor.getClazz(), arguments);
+ }
+}
diff --git
a/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetup.java
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetup.java
new file mode 100644
index 00000000..1cc5cb90
--- /dev/null
+++
b/integrations/chat-models/openai/src/main/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetup.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+
+/**
+ * Chat model setup for a <a href="https://docs.vllm.ai";>vLLM</a> server,
delegating execution to
+ * {@link VLLMChatModelConnection}.
+ *
+ * <p>Behaves like {@link OpenAICompletionsSetup} with one difference:
<b>model</b> is required and
+ * has no default, because a vLLM server only serves the model(s) it was
started with — there is no
+ * meaningful universal default. The value must match the model name announced
by the server (see
+ * {@code vllm serve <model>}, or query {@code GET /v1/models}).
+ *
+ * <p>Example usage:
+ *
+ * <pre>{@code
+ * public class MyAgent extends Agent {
+ * @ChatModelSetup
+ * public static ResourceDescriptor vllmModel() {
+ * return
ResourceDescriptor.Builder.newBuilder(VLLMChatModelSetup.class.getName())
+ * .addInitialArgument("connection", "vllm")
+ * .addInitialArgument("model", "Qwen/Qwen2.5-7B-Instruct")
+ * .addInitialArgument("temperature", 0.3d)
+ * .build();
+ * }
+ * }
+ * }</pre>
+ */
+public class VLLMChatModelSetup extends OpenAICompletionsSetup {
+
+ public VLLMChatModelSetup(ResourceDescriptor descriptor, ResourceContext
resourceContext) {
+ super(descriptor, resourceContext);
+ String model = descriptor.getArgument("model");
+ if (model == null || model.isBlank()) {
+ throw new IllegalArgumentException(
+ "model is required for vLLM: it must match the model name
served by the vLLM"
+ + " server (see `vllm serve <model>` or GET
/v1/models).");
+ }
+ }
+}
diff --git
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnectionTest.java
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnectionTest.java
new file mode 100644
index 00000000..44e94095
--- /dev/null
+++
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelConnectionTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import com.openai.models.chat.completions.ChatCompletionCreateParams;
+import org.apache.flink.agents.api.chat.messages.ChatMessage;
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+/**
+ * Unit tests for {@link VLLMChatModelConnection} — constructor/default
handling only, no network
+ * access.
+ */
+class VLLMChatModelConnectionTest {
+
+ private static final ResourceContext NOOP =
ResourceContext.fromGetResource((a, b) -> null);
+
+ private static ResourceDescriptor.Builder connectionDescriptor() {
+ return
ResourceDescriptor.Builder.newBuilder(VLLMChatModelConnection.class.getName());
+ }
+
+ @Test
+ @DisplayName("Constructor succeeds with no arguments: api_key and
api_base_url are defaulted")
+ void testConstructorNoArguments() {
+ // The parent OpenAI connection throws when api_key is missing, so a
successful
+ // construction here proves the vLLM defaults were injected.
+ ResourceDescriptor desc = connectionDescriptor().build();
+ VLLMChatModelConnection conn = new VLLMChatModelConnection(desc, NOOP);
+ assertThat(conn).isInstanceOf(OpenAICompletionsConnection.class);
+ }
+
+ @Test
+ @DisplayName("Constructor defaults blank api_key and api_base_url")
+ void testConstructorBlankArgumentsAreDefaulted() {
+ ResourceDescriptor desc =
+ connectionDescriptor()
+ .addInitialArgument("api_key", "")
+ .addInitialArgument("api_base_url", " ")
+ .build();
+ // Pin the substituted values, not just the absence of an exception: a
blank
+ // api_base_url would make the parent silently build against the SDK's
default
+ // endpoint rather than throw, so doesNotThrowAnyException() alone
cannot catch a
+ // lost isBlank() branch.
+
assertThat(VLLMChatModelConnection.withVLLMDefaults(desc).getInitialArguments())
+ .containsEntry("api_key",
VLLMChatModelConnection.DEFAULT_VLLM_API_KEY)
+ .containsEntry("api_base_url",
VLLMChatModelConnection.DEFAULT_VLLM_API_BASE_URL);
+ assertThatCode(() -> new VLLMChatModelConnection(desc,
NOOP)).doesNotThrowAnyException();
+ }
+
+ @Test
+ @DisplayName("Constructor honors explicit api_key and api_base_url")
+ void testConstructorExplicitArguments() {
+ ResourceDescriptor desc =
+ connectionDescriptor()
+ .addInitialArgument("api_key", "secret-key")
+ .addInitialArgument("api_base_url",
"http://vllm-host:8000/v1";)
+ .addInitialArgument("timeout", 30)
+ .addInitialArgument("max_retries", 1)
+ .build();
+ assertThatCode(() -> new VLLMChatModelConnection(desc,
NOOP)).doesNotThrowAnyException();
+ }
+
+ @Test
+ @DisplayName("withVLLMDefaults preserves explicit values and injects
defaults only when absent")
+ void testWithVLLMDefaultsPreservesExplicitValues() {
+ ResourceDescriptor explicit =
+ connectionDescriptor()
+ .addInitialArgument("api_key", "secret-key")
+ .addInitialArgument("api_base_url",
"http://vllm-host:8000/v1";)
+ .build();
+
assertThat(VLLMChatModelConnection.withVLLMDefaults(explicit).getInitialArguments())
+ .containsEntry("api_key", "secret-key")
+ .containsEntry("api_base_url", "http://vllm-host:8000/v1";);
+
+ ResourceDescriptor empty = connectionDescriptor().build();
+
assertThat(VLLMChatModelConnection.withVLLMDefaults(empty).getInitialArguments())
+ .containsEntry("api_key",
VLLMChatModelConnection.DEFAULT_VLLM_API_KEY)
+ .containsEntry("api_base_url",
VLLMChatModelConnection.DEFAULT_VLLM_API_BASE_URL);
+ }
+
+ /** A representative POJO output schema. */
+ public static class Person {
+ public String name;
+ public int age;
+ }
+
+ @Test
+ @DisplayName("Structured-output capability follows the served model, not
OpenAI model names")
+ void testSupportsNativeStructuredOutputForServedModels() {
+ VLLMChatModelConnection conn =
+ new VLLMChatModelConnection(connectionDescriptor().build(),
NOOP);
+
assertThat(conn.supportsNativeStructuredOutput("Qwen/Qwen2.5-7B-Instruct")).isTrue();
+
assertThat(conn.supportsNativeStructuredOutput("meta-llama/Llama-3.1-8B-Instruct"))
+ .isTrue();
+ assertThat(conn.supportsNativeStructuredOutput(null)).isFalse();
+ assertThat(conn.supportsNativeStructuredOutput(" ")).isFalse();
+ }
+
+ @Test
+ @DisplayName("Native response_format json_schema applied when serving a
Qwen model")
+ void testNativeResponseFormatAppliedForQwenModel() {
+ VLLMChatModelConnection conn =
+ new VLLMChatModelConnection(connectionDescriptor().build(),
NOOP);
+ java.util.Map<String, Object> modelParams = new HashMap<>();
+ modelParams.put("model", "Qwen/Qwen2.5-7B-Instruct");
+
+ ChatCompletionCreateParams params =
+ conn.buildRequest(
+ List.of(ChatMessage.user("hi")), List.of(),
modelParams, Person.class);
+
+ assertThat(params.responseFormat()).isPresent();
+ }
+
+ @Test
+ @DisplayName("Defaults do not leak into the caller's descriptor")
+ void testCallerDescriptorNotMutated() {
+ ResourceDescriptor desc = connectionDescriptor().build();
+ new VLLMChatModelConnection(desc, NOOP);
+ assertThat(desc.getInitialArguments()).doesNotContainKeys("api_key",
"api_base_url");
+ }
+}
diff --git
a/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetupTest.java
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetupTest.java
new file mode 100644
index 00000000..13956a6e
--- /dev/null
+++
b/integrations/chat-models/openai/src/test/java/org/apache/flink/agents/integrations/chatmodels/openai/VLLMChatModelSetupTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.agents.integrations.chatmodels.openai;
+
+import org.apache.flink.agents.api.resource.ResourceContext;
+import org.apache.flink.agents.api.resource.ResourceDescriptor;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link VLLMChatModelSetup}. */
+class VLLMChatModelSetupTest {
+
+ private static final ResourceContext NOOP =
ResourceContext.fromGetResource((a, b) -> null);
+
+ private static ResourceDescriptor.Builder descriptorBuilder() {
+ return
ResourceDescriptor.Builder.newBuilder(VLLMChatModelSetup.class.getName());
+ }
+
+ @Test
+ @DisplayName("Constructor throws when model is missing: vLLM has no
default model")
+ void testConstructorMissingModel() {
+ ResourceDescriptor desc =
+ descriptorBuilder().addInitialArgument("connection",
"vllm").build();
+ assertThatThrownBy(() -> new VLLMChatModelSetup(desc, NOOP))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("model is required for vLLM");
+ }
+
+ @Test
+ @DisplayName("Constructor throws when model is blank")
+ void testConstructorBlankModel() {
+ ResourceDescriptor desc =
+ descriptorBuilder()
+ .addInitialArgument("connection", "vllm")
+ .addInitialArgument("model", " ")
+ .build();
+ assertThatThrownBy(() -> new VLLMChatModelSetup(desc, NOOP))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("model is required for vLLM");
+ }
+
+ @Test
+ @DisplayName("getParameters carries the served model name and inherited
OpenAI defaults")
+ void testGetParameters() {
+ ResourceDescriptor desc =
+ descriptorBuilder()
+ .addInitialArgument("connection", "vllm")
+ .addInitialArgument("model",
"Qwen/Qwen2.5-7B-Instruct")
+ .addInitialArgument("temperature", 0.3d)
+ .addInitialArgument("max_tokens", 512)
+ .build();
+ VLLMChatModelSetup setup = new VLLMChatModelSetup(desc, NOOP);
+
+ Map<String, Object> params = setup.getParameters();
+ assertThat(params)
+ .containsEntry("model", "Qwen/Qwen2.5-7B-Instruct")
+ .containsEntry("temperature", 0.3d)
+ .containsEntry("max_tokens", 512);
+ }
+}
diff --git a/python/flink_agents/api/resource.py
b/python/flink_agents/api/resource.py
index 21d03342..b945c9ad 100644
--- a/python/flink_agents/api/resource.py
+++ b/python/flink_agents/api/resource.py
@@ -257,6 +257,10 @@ class ResourceName:
TONGYI_CONNECTION =
"flink_agents.integrations.chat_models.tongyi_chat_model.TongyiChatModelConnection"
TONGYI_SETUP =
"flink_agents.integrations.chat_models.tongyi_chat_model.TongyiChatModelSetup"
+ # vLLM (OpenAI-compatible)
+ VLLM_CONNECTION =
"flink_agents.integrations.chat_models.vllm.vllm_chat_model.VLLMChatModelConnection"
+ VLLM_SETUP =
"flink_agents.integrations.chat_models.vllm.vllm_chat_model.VLLMChatModelSetup"
+
# Java Wrapper
JAVA_WRAPPER_CONNECTION = (
"flink_agents.api.chat_models.java_chat_model.JavaChatModelConnection"
@@ -299,6 +303,10 @@ class ResourceName:
AZURE_OPENAI_CONNECTION =
"org.apache.flink.agents.integrations.chatmodels.openai.AzureOpenAIChatModelConnection"
AZURE_OPENAI_SETUP =
"org.apache.flink.agents.integrations.chatmodels.openai.AzureOpenAIChatModelSetup"
+ # vLLM (OpenAI-compatible)
+ VLLM_CONNECTION =
"org.apache.flink.agents.integrations.chatmodels.openai.VLLMChatModelConnection"
+ VLLM_SETUP =
"org.apache.flink.agents.integrations.chatmodels.openai.VLLMChatModelSetup"
+
class EmbeddingModel:
"""EmbeddingModel resource names."""
diff --git a/python/flink_agents/api/yaml/aliases.py
b/python/flink_agents/api/yaml/aliases.py
index c5dda4bd..2f8da7d1 100644
--- a/python/flink_agents/api/yaml/aliases.py
+++ b/python/flink_agents/api/yaml/aliases.py
@@ -66,6 +66,7 @@ CLAZZ_ALIASES: Dict[ResourceType, Dict[str, Dict[str, str]]]
= {
"anthropic": ResourceName.ChatModel.ANTHROPIC_CONNECTION,
"tongyi": ResourceName.ChatModel.TONGYI_CONNECTION,
"azure_openai": ResourceName.ChatModel.AZURE_OPENAI_CONNECTION,
+ "vllm": ResourceName.ChatModel.VLLM_CONNECTION,
},
"java": {
"ollama": ResourceName.ChatModel.Java.OLLAMA_CONNECTION,
@@ -76,6 +77,7 @@ CLAZZ_ALIASES: Dict[ResourceType, Dict[str, Dict[str, str]]]
= {
"azure_openai":
ResourceName.ChatModel.Java.AZURE_OPENAI_CONNECTION,
"azure": ResourceName.ChatModel.Java.AZURE_CONNECTION,
"bedrock": ResourceName.ChatModel.Java.BEDROCK_CONNECTION,
+ "vllm": ResourceName.ChatModel.Java.VLLM_CONNECTION,
},
},
ResourceType.CHAT_MODEL: {
@@ -85,6 +87,7 @@ CLAZZ_ALIASES: Dict[ResourceType, Dict[str, Dict[str, str]]]
= {
"anthropic": ResourceName.ChatModel.ANTHROPIC_SETUP,
"tongyi": ResourceName.ChatModel.TONGYI_SETUP,
"azure_openai": ResourceName.ChatModel.AZURE_OPENAI_SETUP,
+ "vllm": ResourceName.ChatModel.VLLM_SETUP,
},
"java": {
"ollama": ResourceName.ChatModel.Java.OLLAMA_SETUP,
@@ -95,6 +98,7 @@ CLAZZ_ALIASES: Dict[ResourceType, Dict[str, Dict[str, str]]]
= {
"azure_openai": ResourceName.ChatModel.Java.AZURE_OPENAI_SETUP,
"azure": ResourceName.ChatModel.Java.AZURE_SETUP,
"bedrock": ResourceName.ChatModel.Java.BEDROCK_SETUP,
+ "vllm": ResourceName.ChatModel.Java.VLLM_SETUP,
},
},
ResourceType.EMBEDDING_MODEL_CONNECTION: {
diff --git a/python/flink_agents/api/yaml/tests/test_aliases.py
b/python/flink_agents/api/yaml/tests/test_aliases.py
index a6d728c6..dfb3b296 100644
--- a/python/flink_agents/api/yaml/tests/test_aliases.py
+++ b/python/flink_agents/api/yaml/tests/test_aliases.py
@@ -127,6 +127,24 @@ def test_resolve_clazz_default_language_is_python() ->
None:
assert default == explicit
+def test_resolve_clazz_covers_chat_model_vllm_in_both_languages() -> None:
+ assert resolve_clazz("vllm", ResourceType.CHAT_MODEL_CONNECTION).endswith(
+ "vllm_chat_model.VLLMChatModelConnection"
+ )
+ assert resolve_clazz("vllm", ResourceType.CHAT_MODEL).endswith(
+ "vllm_chat_model.VLLMChatModelSetup"
+ )
+ # Like `azure_openai`, the Java and Python simple names collide
+ # (VLLMChatModelConnection); pin both the package and the class so a Java
entry
+ # that lost its `.Java` suffix and resolved to the Python class fails here.
+ java_conn = resolve_clazz("vllm", ResourceType.CHAT_MODEL_CONNECTION,
"java")
+ assert java_conn.startswith("org.apache.flink.agents")
+ assert java_conn.endswith("VLLMChatModelConnection")
+ java_setup = resolve_clazz("vllm", ResourceType.CHAT_MODEL, "java")
+ assert java_setup.startswith("org.apache.flink.agents")
+ assert java_setup.endswith("VLLMChatModelSetup")
+
+
def test_resolve_clazz_covers_chat_model_java_gemini_and_azure_openai() ->
None:
assert resolve_clazz("gemini", ResourceType.CHAT_MODEL_CONNECTION,
"java").endswith(
"GeminiChatModelConnection"
@@ -134,7 +152,7 @@ def
test_resolve_clazz_covers_chat_model_java_gemini_and_azure_openai() -> None:
assert resolve_clazz("gemini", ResourceType.CHAT_MODEL, "java").endswith(
"GeminiChatModelSetup"
)
- # `azure_openai` is the only alias whose Java/Python simple names collide
+ # `azure_openai` (like `vllm`) has colliding Java/Python simple names
# (AzureOpenAIChatModelConnection); pin the package so a Java entry that
lost
# its `.Java` suffix and resolved to the Python class would fail here.
azure_conn = resolve_clazz(
diff --git a/python/flink_agents/integrations/chat_models/vllm/__init__.py
b/python/flink_agents/integrations/chat_models/vllm/__init__.py
new file mode 100644
index 00000000..e154fadd
--- /dev/null
+++ b/python/flink_agents/integrations/chat_models/vllm/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
diff --git
a/python/flink_agents/integrations/chat_models/vllm/tests/__init__.py
b/python/flink_agents/integrations/chat_models/vllm/tests/__init__.py
new file mode 100644
index 00000000..e154fadd
--- /dev/null
+++ b/python/flink_agents/integrations/chat_models/vllm/tests/__init__.py
@@ -0,0 +1,17 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
diff --git
a/python/flink_agents/integrations/chat_models/vllm/tests/test_vllm_chat_model.py
b/python/flink_agents/integrations/chat_models/vllm/tests/test_vllm_chat_model.py
new file mode 100644
index 00000000..36b9b8e2
--- /dev/null
+++
b/python/flink_agents/integrations/chat_models/vllm/tests/test_vllm_chat_model.py
@@ -0,0 +1,131 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import BaseModel
+
+from flink_agents.api.agents.types import OutputSchema
+from flink_agents.api.chat_message import ChatMessage, MessageRole
+from flink_agents.integrations.chat_models.openai.openai_chat_model import (
+ OpenAIChatModelConnection,
+)
+from flink_agents.integrations.chat_models.vllm.vllm_chat_model import (
+ DEFAULT_VLLM_API_BASE_URL,
+ DEFAULT_VLLM_API_KEY,
+ VLLMChatModelConnection,
+ VLLMChatModelSetup,
+)
+
+
+def test_connection_defaults_to_local_vllm_server() -> None:
+ connection = VLLMChatModelConnection(name="vllm")
+ assert isinstance(connection, OpenAIChatModelConnection)
+ assert connection.api_base_url == DEFAULT_VLLM_API_BASE_URL
+ assert connection.api_key == DEFAULT_VLLM_API_KEY
+
+
+def test_connection_honors_explicit_arguments() -> None:
+ connection = VLLMChatModelConnection(
+ name="vllm",
+ api_key="secret-key",
+ api_base_url="http://vllm-host:8000/v1";,
+ timeout=30.0,
+ max_retries=1,
+ )
+ assert connection.api_key == "secret-key"
+ assert connection.api_base_url == "http://vllm-host:8000/v1";
+ assert connection.timeout == 30.0
+ assert connection.max_retries == 1
+
+
+def test_setup_requires_model() -> None:
+ with pytest.raises(ValueError, match="model is required for vLLM"):
+ VLLMChatModelSetup(name="vllm_model", connection="vllm")
+
+
+def test_connection_defaults_whitespace_only_arguments() -> None:
+ # Semantic parity with the Java connection, which treats blank values as
absent.
+ connection = VLLMChatModelConnection(name="vllm", api_key=" ",
api_base_url=" ")
+ assert connection.api_key == DEFAULT_VLLM_API_KEY
+ assert connection.api_base_url == DEFAULT_VLLM_API_BASE_URL
+
+
+def test_setup_rejects_whitespace_only_model() -> None:
+ with pytest.raises(ValueError, match="model is required for vLLM"):
+ VLLMChatModelSetup(name="vllm_model", connection="vllm", model=" ")
+
+
+def test_setup_rejects_empty_model() -> None:
+ with pytest.raises(ValueError, match="model is required for vLLM"):
+ VLLMChatModelSetup(name="vllm_model", connection="vllm", model="")
+
+
+class _Person(BaseModel):
+ """A representative BaseModel output schema."""
+
+ name: str
+ age: int
+
+
+def test_supports_native_structured_output_follows_served_model() -> None:
+ connection = VLLMChatModelConnection(name="vllm")
+ assert
connection.supports_native_structured_output("Qwen/Qwen2.5-7B-Instruct")
+ assert connection.supports_native_structured_output(
+ "meta-llama/Llama-3.1-8B-Instruct"
+ )
+ assert not connection.supports_native_structured_output(None)
+ assert not connection.supports_native_structured_output(" ")
+
+
+def test_native_response_format_applied_for_qwen_model() -> None:
+ connection = VLLMChatModelConnection(name="vllm")
+ mock_client = MagicMock()
+ mock_message = MagicMock()
+ mock_message.role = "assistant"
+ mock_message.content = "ok"
+ mock_message.tool_calls = None
+ mock_client.chat.completions.create.return_value.choices = [
+ MagicMock(message=mock_message)
+ ]
+ mock_client.chat.completions.create.return_value.usage = None
+ connection._client = mock_client
+
+ connection.chat(
+ [ChatMessage(role=MessageRole.USER, content="hi")],
+ model="Qwen/Qwen2.5-7B-Instruct",
+ output_schema=OutputSchema(output_schema=_Person),
+ )
+
+ kwargs = mock_client.chat.completions.create.call_args.kwargs
+ assert "response_format" in kwargs
+ assert kwargs["response_format"]["type"] == "json_schema"
+
+
+def test_setup_carries_served_model_name() -> None:
+ setup = VLLMChatModelSetup(
+ name="vllm_model",
+ connection="vllm",
+ model="Qwen/Qwen2.5-7B-Instruct",
+ temperature=0.3,
+ max_tokens=512,
+ )
+ kwargs = setup.model_kwargs
+ assert kwargs["model"] == "Qwen/Qwen2.5-7B-Instruct"
+ assert kwargs["temperature"] == 0.3
+ assert kwargs["max_tokens"] == 512
diff --git
a/python/flink_agents/integrations/chat_models/vllm/vllm_chat_model.py
b/python/flink_agents/integrations/chat_models/vllm/vllm_chat_model.py
new file mode 100644
index 00000000..59a8eccc
--- /dev/null
+++ b/python/flink_agents/integrations/chat_models/vllm/vllm_chat_model.py
@@ -0,0 +1,106 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+from typing import Any
+
+from typing_extensions import override
+
+from flink_agents.integrations.chat_models.openai.openai_chat_model import (
+ OpenAIChatModelConnection,
+ OpenAIChatModelSetup,
+)
+
+DEFAULT_VLLM_API_BASE_URL = "http://localhost:8000/v1";
+"""Default base URL of a local ``vllm serve`` instance."""
+
+DEFAULT_VLLM_API_KEY = "EMPTY"
+"""Placeholder credential used when the vLLM server is started without
+``--api-key``. The OpenAI SDK requires a non-empty key, but the server ignores
+its value."""
+
+
+class VLLMChatModelConnection(OpenAIChatModelConnection):
+ """Connection to a `vLLM <https://docs.vllm.ai>`_ server.
+
+ vLLM exposes an OpenAI-compatible API, so this connection reuses
+ :class:`OpenAIChatModelConnection` with vLLM-friendly defaults:
+
+ * ``api_base_url`` defaults to ``http://localhost:8000/v1``, the default
+ address of ``vllm serve``.
+ * ``api_key`` defaults to a placeholder, since vLLM servers started without
+ ``--api-key`` do not require a credential. Set it explicitly when the
+ server is started with ``--api-key``.
+
+ Defaults are applied for ``None``, empty, and whitespace-only values,
+ matching the Java connection. Unlike :class:`OpenAIChatModelConnection`,
+ the ``OPENAI_API_KEY`` / ``OPENAI_API_BASE_URL`` environment variables are
+ **not** consulted: the vLLM defaults deliberately do not depend on the
+ developer's environment, and the Java connection has no environment
+ fallback either.
+
+ All other attributes (``timeout``, ``max_retries``, ``default_headers``,
+ ``reuse_client``) behave exactly as in :class:`OpenAIChatModelConnection`.
+ """
+
+ def __init__(
+ self,
+ *,
+ api_key: str | None = None,
+ api_base_url: str | None = None,
+ **kwargs: Any,
+ ) -> None:
+ """Init method."""
+ super().__init__(
+ api_key=api_key if api_key and api_key.strip() else
DEFAULT_VLLM_API_KEY,
+ api_base_url=(
+ api_base_url
+ if api_base_url and api_base_url.strip()
+ else DEFAULT_VLLM_API_BASE_URL
+ ),
+ **kwargs,
+ )
+
+ @override
+ def supports_native_structured_output(self, effective_model: str | None)
-> bool:
+ """VLLM implements the OpenAI ``json_schema`` response format for
whatever
+ model it serves (via guided decoding), so structured-output capability
does
+ not depend on OpenAI model names — the inherited allowlist would
wrongly
+ reject served models such as ``Qwen/Qwen2.5-7B-Instruct``.
+ See https://docs.vllm.ai/en/stable/features/structured_outputs.html.
+ """
+ return bool(effective_model and effective_model.strip())
+
+
+class VLLMChatModelSetup(OpenAIChatModelSetup):
+ """Settings for a chat model served by vLLM.
+
+ Behaves like :class:`OpenAIChatModelSetup` with one difference: ``model``
+ is required and has no default, because a vLLM server only serves the
+ model(s) it was started with — there is no meaningful universal default.
+ The value must match the model name announced by the server (see
+ ``vllm serve <model>``, or query ``GET /v1/models``).
+ """
+
+ def __init__(self, *, model: str | None = None, **kwargs: Any) -> None:
+ """Init method."""
+ if not model or not model.strip():
+ msg = (
+ "model is required for vLLM: it must match the model name
served "
+ "by the vLLM server (see `vllm serve <model>` or GET
/v1/models)."
+ )
+ raise ValueError(msg)
+ super().__init__(model=model, **kwargs)
