kaxil commented on code in PR #76974:
URL: https://github.com/apache/airflow/pull/76974#discussion_r4622882819


##########
providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py:
##########
@@ -0,0 +1,937 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods.
+
+One backend drives every provider pydantic-ai can batch: it encodes each 
request with
+``batch_encode``, submits them with ``batch_create``, polls with 
``batch_status``, collects with
+``batch_results`` and reads each result with ``batch_decode``. The provider 
subclasses set only
+what differs between providers (the settings they accept and how a finished 
job is read).
+
+pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's 
state record, so the
+trigger and a later worker rebuild a model from the connection and continue 
the same batch without
+the original process.
+
+Importing this module never imports a provider SDK: the model classes are 
built by
+:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and 
the SDK-specific
+settings types are imported inside the methods that need them.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import dataclasses
+import functools
+import queue
+import threading
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, 
Coroutine, Generator, Iterator, Mapping
+from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, 
suppress
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast
+
+from pydantic import TypeAdapter
+from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError
+from pydantic_ai.messages import (
+    ModelRequest,
+    ModelRequestPart,
+    ModelResponse,
+    SystemPromptPart,
+    UserPromptPart,
+)
+from pydantic_ai.models import ModelRequestParameters
+
+# pydantic-ai raises this for a job that ended with no results file. Nothing 
public tells it from an API failure
+# (the handle carries no result destination), and a test pins that it stays a 
``ModelHTTPError``.
+from pydantic_ai.models._batch_codec import BatchJobError
+from pydantic_ai.models.batch import (
+    Batch,
+    BatchRequest as PydanticAIBatchRequest,
+    BatchResult,
+    EncodedBatchRequest,
+)
+from pydantic_ai.output import OutputObjectDefinition
+from pydantic_ai.settings import ModelSettings
+from pydantic_ai.tools import ToolDefinition
+
+from airflow.providers.common.ai.batch.base import (
+    BatchAdapter,
+    BatchState,
+    ExtractedOutput,
+    RawResultItem,
+    SubmitResult,
+)
+from airflow.providers.common.ai.batch.output_schema import build_output_spec
+from airflow.providers.common.ai.exceptions import (
+    LLMBatchInputError,
+    LLMBatchJobError,
+    LLMBatchLimitExceededError,
+    LLMBatchModelMismatchError,
+    LLMBatchStateReadError,
+)
+from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook
+from airflow.providers.common.ai.utils.task_logger import get_task_logger
+
+if TYPE_CHECKING:
+    from pydantic_ai.models import Model
+
+    from airflow.providers.common.ai.batch.base import BatchRequest
+    from airflow.providers.common.ai.batch.output_schema import OutputSpec
+    from airflow.sdk import Connection
+
+log = get_task_logger()
+
+T = TypeVar("T")
+
+_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch)
+
+#: Sampling settings with the same meaning on every provider. A key in this 
set (or one of a
+#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other 
key is a raw
+#: provider request field.
+_GENERIC_SETTINGS = frozenset(
+    {
+        "temperature",
+        "top_p",
+        "top_k",
+        "seed",
+        "presence_penalty",
+        "frequency_penalty",
+        "logit_bias",
+        "stop_sequences",
+    }
+)
+
+#: Request fields this adapter sets itself. Taking them from 
``request_params`` would replace the
+#: prompt, the model or the structured-output constraint, so they are dropped 
with a warning.
+_MANAGED_KEYS = frozenset(
+    {
+        "model",
+        "messages",
+        "input",
+        "system",
+        "system_instruction",
+        "contents",
+        "tools",
+        "tool_choice",
+        "stream",
+        "max_tokens",
+        "max_output_tokens",
+        "max_completion_tokens",
+        "output_config",
+        "text",
+        "response_format",
+    }
+)
+
+#: pydantic-ai's status for a request that never ran, as this provider names 
it.
+_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": 
"cancelled", "expired": "expired"}
+
+#: Gemini documents no request limit, and the ABC wants an int.
+_NO_REQUEST_LIMIT = 2**63 - 1
+
+#: Decoded rows a batch's results may run ahead of the task that reads them.
+_RESULT_BUFFER = 64
+
+#: Seconds a closed results stream waits for its helper thread to finish after 
cancelling it.
+_STREAM_JOIN_TIMEOUT = 5.0
+
+
[email protected](frozen=True)
+class _ResultsContext:
+    """What a reader needs, besides the raw results, to decode them."""
+
+    model: Model
+    parameters: ModelRequestParameters
+
+
+def _run(coro: Coroutine[Any, Any, T]) -> T:

Review Comment:
   It does exactly what `_run` did, plus copying the caller's context, so 
`_run` is gone.



##########
providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py:
##########
@@ -0,0 +1,937 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods.
+
+One backend drives every provider pydantic-ai can batch: it encodes each 
request with
+``batch_encode``, submits them with ``batch_create``, polls with 
``batch_status``, collects with
+``batch_results`` and reads each result with ``batch_decode``. The provider 
subclasses set only
+what differs between providers (the settings they accept and how a finished 
job is read).
+
+pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's 
state record, so the
+trigger and a later worker rebuild a model from the connection and continue 
the same batch without
+the original process.
+
+Importing this module never imports a provider SDK: the model classes are 
built by
+:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and 
the SDK-specific
+settings types are imported inside the methods that need them.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import dataclasses
+import functools
+import queue
+import threading
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, 
Coroutine, Generator, Iterator, Mapping
+from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, 
suppress
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast
+
+from pydantic import TypeAdapter
+from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError
+from pydantic_ai.messages import (
+    ModelRequest,
+    ModelRequestPart,
+    ModelResponse,
+    SystemPromptPart,
+    UserPromptPart,
+)
+from pydantic_ai.models import ModelRequestParameters
+
+# pydantic-ai raises this for a job that ended with no results file. Nothing 
public tells it from an API failure
+# (the handle carries no result destination), and a test pins that it stays a 
``ModelHTTPError``.
+from pydantic_ai.models._batch_codec import BatchJobError
+from pydantic_ai.models.batch import (
+    Batch,
+    BatchRequest as PydanticAIBatchRequest,
+    BatchResult,
+    EncodedBatchRequest,
+)
+from pydantic_ai.output import OutputObjectDefinition
+from pydantic_ai.settings import ModelSettings
+from pydantic_ai.tools import ToolDefinition
+
+from airflow.providers.common.ai.batch.base import (
+    BatchAdapter,
+    BatchState,
+    ExtractedOutput,
+    RawResultItem,
+    SubmitResult,
+)
+from airflow.providers.common.ai.batch.output_schema import build_output_spec
+from airflow.providers.common.ai.exceptions import (
+    LLMBatchInputError,
+    LLMBatchJobError,
+    LLMBatchLimitExceededError,
+    LLMBatchModelMismatchError,
+    LLMBatchStateReadError,
+)
+from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook
+from airflow.providers.common.ai.utils.task_logger import get_task_logger
+
+if TYPE_CHECKING:
+    from pydantic_ai.models import Model
+
+    from airflow.providers.common.ai.batch.base import BatchRequest
+    from airflow.providers.common.ai.batch.output_schema import OutputSpec
+    from airflow.sdk import Connection
+
+log = get_task_logger()
+
+T = TypeVar("T")
+
+_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch)
+
+#: Sampling settings with the same meaning on every provider. A key in this 
set (or one of a
+#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other 
key is a raw
+#: provider request field.
+_GENERIC_SETTINGS = frozenset(
+    {
+        "temperature",
+        "top_p",
+        "top_k",
+        "seed",
+        "presence_penalty",
+        "frequency_penalty",
+        "logit_bias",
+        "stop_sequences",
+    }
+)
+
+#: Request fields this adapter sets itself. Taking them from 
``request_params`` would replace the
+#: prompt, the model or the structured-output constraint, so they are dropped 
with a warning.
+_MANAGED_KEYS = frozenset(
+    {
+        "model",
+        "messages",
+        "input",
+        "system",
+        "system_instruction",
+        "contents",
+        "tools",
+        "tool_choice",
+        "stream",
+        "max_tokens",
+        "max_output_tokens",
+        "max_completion_tokens",
+        "output_config",
+        "text",
+        "response_format",
+    }
+)
+
+#: pydantic-ai's status for a request that never ran, as this provider names 
it.
+_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": 
"cancelled", "expired": "expired"}
+
+#: Gemini documents no request limit, and the ABC wants an int.
+_NO_REQUEST_LIMIT = 2**63 - 1
+
+#: Decoded rows a batch's results may run ahead of the task that reads them.
+_RESULT_BUFFER = 64
+
+#: Seconds a closed results stream waits for its helper thread to finish after 
cancelling it.
+_STREAM_JOIN_TIMEOUT = 5.0
+
+
[email protected](frozen=True)
+class _ResultsContext:
+    """What a reader needs, besides the raw results, to decode them."""
+
+    model: Model
+    parameters: ModelRequestParameters
+
+
+def _run(coro: Coroutine[Any, Any, T]) -> T:
+    """
+    Run ``coro`` to completion from synchronous code.
+
+    ``asyncio.run`` refuses to start inside a thread that already runs a loop, 
which is where a
+    kill signal's ``on_kill`` can land while a poll is in flight, so that case 
uses a helper thread.
+    """
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        # No loop in this thread. Run outside the ``except`` so an error from 
``coro`` isn't chained
+        # onto this probe in the task log.
+        pass
+    else:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            return executor.submit(asyncio.run, coro).result()
+    return asyncio.run(coro)
+
+
+def _stream(source: AsyncGenerator[T, None]) -> Generator[T, None, None]:
+    """
+    Read an async generator from synchronous code without collecting it first.
+
+    The generator runs as a task on its own event loop in a helper thread and 
hands items over through a
+    queue of :data:`_RESULT_BUFFER` items, so a slow reader holds the download 
back instead of buffering it.
+    An error from the generator is raised at the reader, whatever its type.
+
+    When the reader stops (closes the iterator, is dropped, or fails), the 
task is cancelled from the reader's
+    thread. A read the generator is awaiting is interrupted rather than waited 
for, and the generator's own
+    cleanup still runs, so whatever it holds open is closed. The wait for the 
thread is bounded.
+    """
+    items: queue.Queue[
+        tuple[Literal["item"], T] | tuple[Literal["error"], BaseException] | 
tuple[Literal["end"], None]
+    ]
+    items = queue.Queue(maxsize=_RESULT_BUFFER)
+    closed = threading.Event()
+    started = threading.Event()
+    running: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[None]]] = []
+
+    def put(message: tuple[Any, Any]) -> bool:
+        # A bounded put that gives up once the reader is gone, so a full queue 
never strands the thread.
+        while not closed.is_set():
+            try:
+                items.put(message, timeout=0.1)
+                return True
+            except queue.Full:
+                continue
+        return False
+
+    async def produce() -> None:
+        try:
+            async for item in source:
+                if not put(("item", item)):
+                    return
+            put(("end", None))
+        except BaseException as e:
+            # Also a cancellation the reader did not ask for: forwarding it 
keeps the reader from waiting forever.
+            put(("error", e))
+        finally:
+            await source.aclose()
+
+    async def main() -> None:
+        task = asyncio.ensure_future(produce())
+        running.append((asyncio.get_running_loop(), task))
+        started.set()
+        await asyncio.wait([task])
+
+    thread = threading.Thread(target=asyncio.run, args=(main(),), 
name="llm-batch-results", daemon=True)
+    thread.start()
+    try:
+        while True:
+            try:
+                message = items.get(timeout=1)
+            except queue.Empty:
+                if not thread.is_alive() and items.empty():
+                    raise RuntimeError("The batch results reader stopped 
without finishing.") from None
+                continue
+            if message[0] == "item":
+                yield message[1]
+            elif message[0] == "error":
+                raise message[1]
+            else:
+                return
+    finally:
+        closed.set()
+        if started.wait(timeout=1):
+            loop, task = running[0]
+            # A RuntimeError means the loop already finished, so there is 
nothing left to cancel.
+            with suppress(RuntimeError):
+                loop.call_soon_threadsafe(task.cancel)
+        thread.join(timeout=_STREAM_JOIN_TIMEOUT)
+        if thread.is_alive():
+            log.debug(
+                "Batch results thread did not stop within the timeout; it is a 
daemon and will be left behind"
+            )
+
+
[email protected]
+def _typed_settings(settings_type: type) -> frozenset[str]:
+    """Return the keys a request may set as typed settings: the generic ones 
and the provider's own."""
+    return _GENERIC_SETTINGS | (set(settings_type.__annotations__) - 
set(ModelSettings.__annotations__))
+
+
+def _error_body(error: ModelHTTPError) -> Mapping[str, Any]:
+    """Return the ``error`` object of an HTTP error's body, which carries the 
provider's own type and message."""
+    body = error.body
+    inner = body.get("error") if isinstance(body, Mapping) else None
+    return inner if isinstance(inner, Mapping) else {}
+
+
+def _reason(error: ModelHTTPError) -> str:
+    """Return the provider's own explanation of an HTTP error; pydantic-ai's 
message wraps it in status and model."""
+    return str(_error_body(error).get("message") or error.message)
+
+
+def _error_info(status: str, message: str, provider_code: str | int | None = 
None) -> dict[str, Any]:
+    return {"type": status, "message": message, "provider_code": 
provider_code, "stage": "provider"}
+
+
+def _failed(
+    custom_id: str,
+    index: int,
+    status: Literal["errored", "expired", "cancelled"],
+    error: Mapping[str, Any],
+) -> RawResultItem:
+    return RawResultItem(
+        custom_id=custom_id,
+        index=index,
+        provider_status=status,
+        model=None,
+        usage=None,
+        finish_reason=None,
+        error=error,
+        raw=None,
+    )
+
+
+class PydanticAIBatchAdapter(BatchAdapter):
+    """
+    Batch adapter for any provider whose pydantic-ai ``Model`` implements the 
``batch_*`` methods.
+
+    .. note::
+
+        Experimental: this can change or be removed in a minor release of this 
provider.
+        See :ref:`howto/stability`.
+
+    The adapter is bound to one connection and one ``model_id``. It builds a 
fresh ``Model`` for
+    every operation and runs it in its own event loop, so no client crosses a 
loop, and it never
+    keeps a connection open between operations.
+
+    :param hook: The hook the models are built from (credentials, 
``provider_kwargs``, the
+        connection's extra).
+    :param model_id: The batch's ``"<provider>:<model>"`` id.
+    :param batch_handle: The serialized ``Batch`` handle recorded when the 
batch was submitted.
+    :param output_spec: The task's output spec. Needed only to read results.
+    :param model_factory: Builds the ``Model`` for a model id instead of the 
hook. For tests.
+    """
+
+    conn_types = frozenset({"pydanticai"})
+    #: ``True`` when a key that is not a typed setting is passed to the 
provider as a raw request field.
+    accepts_raw_params: ClassVar[bool] = True
+
+    def __init__(
+        self,
+        *,
+        hook: PydanticAIHook | None = None,
+        model_id: str | None = None,
+        batch_handle: str | None = None,
+        output_spec: OutputSpec | None = None,
+        model_factory: Callable[[str], Model] | None = None,
+    ) -> None:
+        super().__init__()
+        self._hook = hook
+        self._model_id = model_id
+        self._handle = batch_handle
+        self._output_spec = output_spec or build_output_spec(str)
+        self._model_factory = model_factory
+
+    @classmethod
+    def from_connection(
+        cls,
+        conn: Connection,
+        *,
+        model_id: str | None = None,
+        batch_handle: str | None = None,
+        output_spec: OutputSpec | None = None,
+    ) -> PydanticAIBatchAdapter:
+        hook = PydanticAIHook.get_hook(conn.conn_id, {"model_id": model_id})
+        if hook._get_fallback_conn_ids():
+            log.warning(
+                "Connection has fallback connections configured; batch ignores 
them and uses the primary",
+                llm_conn_id=conn.conn_id,
+            )
+        return cls(hook=hook, model_id=model_id, batch_handle=batch_handle, 
output_spec=output_spec)
+
+    @classmethod
+    def _settings_type(cls) -> type:
+        """Return the provider's ``ModelSettings`` class, whose own keys a 
request may set directly."""
+        raise NotImplementedError(f"{cls.__name__} does not name its 
ModelSettings type.")
+
+    def _require_model_id(self) -> str:
+        if not self._model_id:
+            raise LLMBatchInputError(
+                f"The {self.name!r} batch adapter needs the batch's model_id 
('<provider>:<model>') to "
+                "rebuild its model."
+            )
+        return self._model_id
+
+    async def _amodel(self, model_id: str) -> Model:
+        if self._model_factory is not None:
+            return self._model_factory(model_id)
+        if self._hook is None:
+            raise LLMBatchInputError(f"The {self.name!r} batch adapter was 
built without a connection.")
+        return await self._hook.aget_primary_model(model_id)
+
+    @asynccontextmanager
+    async def _existing(self, batch_id: str) -> AsyncIterator[tuple[Model, 
Batch]]:
+        """
+        Yield the model an already-submitted batch belongs to, and its handle.
+
+        The model comes from the batch, not from the task's ``model_id``: 
pydantic-ai checks the
+        handle's group on every call, and OpenAI's and Gemini's groups name 
the model the batch was
+        submitted on. That differs from the task's model when every request 
overrode it, or when the
+        task's model changed since the batch was submitted. Only the provider 
prefix is taken from
+        ``model_id``, because the connection that submitted the batch is the 
one that resolves it.
+        """
+        model_id = self._model_id
+        if not model_id:
+            raise LLMBatchStateReadError(
+                f"Cannot reach batch {batch_id!r}: it was deferred by an 
earlier version of this provider, "
+                "which did not record the model it needs. Retry the task so it 
re-attaches."
+            )
+        if not self._handle:
+            async with await self._amodel(model_id) as model:
+                yield model, await self._unrecorded_batch(model, batch_id)
+            return
+        batch = _BATCH_ADAPTER.validate_json(self._handle)
+        provider = model_id.partition(":")[0]
+        async with await self._amodel(f"{provider}:{batch.model_name}") as 
model:
+            yield model, batch
+
+    async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch:
+        """Return a handle for a batch recorded without one, by an earlier 
version of this provider."""
+        raise LLMBatchStateReadError(
+            f"No pydantic-ai batch handle is recorded for batch {batch_id!r}, 
so it cannot be polled. "
+            "It was submitted before this provider recorded handles; retry the 
task so it re-attaches."
+        )
+
+    def _build_settings(
+        self,
+        model: Model,
+        *layers: Mapping[str, Any] | None,
+        max_tokens: int,
+    ) -> tuple[ModelSettings, set[str]]:
+        """
+        Turn ``request_params`` and a request's ``params`` into 
``ModelSettings``.
+
+        Typed keys become settings. Any other key is a raw provider request 
field and goes through
+        ``extra_body``, which is how these parameters worked before the 
adapter used pydantic-ai.

Review Comment:
   Dropped.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to