kaxil commented on code in PR #76975: URL: https://github.com/apache/airflow/pull/76975#discussion_r4623387705
########## providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py: ########## @@ -0,0 +1,1128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods. + +One backend drives every provider pydantic-ai can batch: it encodes each request with +``batch_encode``, submits them with ``batch_create``, polls with ``batch_status``, collects with +``batch_results`` and reads each result with ``batch_decode``. The provider subclasses set only +what differs between providers (the settings they accept and how a finished job is read). + +pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's state record, so the +trigger and a later worker rebuild a model from the connection and continue the same batch without +the original process. + +Importing this module never imports a provider SDK: the model classes are built by +:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and the SDK-specific +settings types are imported inside the methods that need them. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import functools +import queue +import threading +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterator, Mapping +from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, suppress +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast + +from pydantic import BaseModel, JsonValue, TypeAdapter +from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError +from pydantic_ai.messages import ( + ModelRequest, + ModelRequestPart, + ModelResponse, + SystemPromptPart, + UserPromptPart, +) +from pydantic_ai.models import ModelRequestParameters + +# pydantic-ai raises this for a job that ended with no results file. Nothing public tells it from an API failure +# (the handle carries no result destination), and a test pins that it stays a ``ModelHTTPError``. +from pydantic_ai.models._batch_codec import BatchJobError +from pydantic_ai.models.batch import ( + Batch, + BatchRequest as PydanticAIBatchRequest, + BatchResult, + EncodedBatchRequest, +) +from pydantic_ai.output import OutputObjectDefinition +from pydantic_ai.settings import ModelSettings +from pydantic_ai.tools import ToolDefinition +from pydantic_core import to_json + +from airflow.providers.common.ai.batch.base import ( + BatchAdapter, + BatchState, + ExtractedOutput, + RawResultItem, + SubmitResult, +) +from airflow.providers.common.ai.batch.output_schema import build_output_spec +from airflow.providers.common.ai.exceptions import ( + LLMBatchInputError, + LLMBatchJobError, + LLMBatchLimitExceededError, + LLMBatchModelMismatchError, + LLMBatchStateReadError, +) +from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook +from airflow.providers.common.ai.utils.task_logger import get_task_logger + +if TYPE_CHECKING: + from pydantic_ai.models import Model + from pydantic_ai.models.openai import OpenAIResponsesModel + + from airflow.providers.common.ai.batch.base import BatchRequest, BatchStatus + from airflow.providers.common.ai.batch.output_schema import OutputSpec + from airflow.sdk import Connection + +log = get_task_logger() + +T = TypeVar("T") + +_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch) + +#: Sampling settings with the same meaning on every provider. A key in this set (or one of a +#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other key is a raw +#: provider request field. +_GENERIC_SETTINGS = frozenset( + { + "temperature", + "top_p", + "top_k", + "seed", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "stop_sequences", + } +) + +#: Request fields this adapter sets itself. Taking them from ``request_params`` would replace the +#: prompt, the model or the structured-output constraint, so they are dropped with a warning. +_MANAGED_KEYS = frozenset( + { + "model", + "messages", + "input", + "system", + "system_instruction", + "contents", + "tools", + "tool_choice", + "stream", + "max_tokens", + "max_output_tokens", + "max_completion_tokens", + "output_config", + "text", + "response_format", + } +) + +#: pydantic-ai's status for a request that never ran, as this provider names it. +_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": "cancelled", "expired": "expired"} + +#: The error codes of a request an OpenAI batch ended before running, and the status pydantic-ai gives each. +#: Mirrors ``_UNRUN_STATUSES`` in pydantic-ai's OpenAI batch module. +_OPENAI_UNRUN_STATUSES: dict[str, Literal["expired", "canceled"]] = { + "batch_expired": "expired", + "batch_cancelled": "canceled", +} + +#: Gemini documents no request limit, and the ABC wants an int. +_NO_REQUEST_LIMIT = 2**63 - 1 + +#: Decoded rows a batch's results may run ahead of the task that reads them. +_RESULT_BUFFER = 64 + +#: Seconds a closed results stream waits for its helper thread to finish after cancelling it. +_STREAM_JOIN_TIMEOUT = 5.0 + + [email protected](frozen=True) +class _ResultsContext: + """What a reader needs, besides the raw results, to decode them.""" + + model: Model + parameters: ModelRequestParameters + + +def _run(coro: Coroutine[Any, Any, T]) -> T: + """ + Run ``coro`` to completion from synchronous code. + + ``asyncio.run`` refuses to start inside a thread that already runs a loop, which is where a + kill signal's ``on_kill`` can land while a poll is in flight, so that case uses a helper thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + # No loop in this thread. Run outside the ``except`` so an error from ``coro`` isn't chained + # onto this probe in the task log. + pass + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + return asyncio.run(coro) + + +class _OpenAILineResponse(BaseModel): + status_code: int + body: JsonValue + + +class _OpenAILineError(BaseModel): + code: str | None = None + message: str | None = None + + +class _OpenAIOutputLine(BaseModel): + """A line of an OpenAI batch's output or error file.""" + + custom_id: str + response: _OpenAILineResponse | None = None + error: _OpenAILineError | None = None + + +class _OpenAIInputLine(BaseModel): + custom_id: str + + +def _openai_result(line: _OpenAIOutputLine) -> BatchResult: + """Mirror of ``_to_result`` in pydantic-ai's OpenAI batch module.""" + if (response := line.response) is not None: + if 200 <= response.status_code < 300: + return BatchResult( + custom_id=line.custom_id, status="succeeded", body=to_json(response.body).decode() + ) + return BatchResult(custom_id=line.custom_id, status="errored", body=response.model_dump_json()) + error = line.error or _OpenAILineError() + if status := _OPENAI_UNRUN_STATUSES.get(error.code or ""): + return BatchResult(custom_id=line.custom_id, status=status) + body = _OpenAILineResponse(status_code=500, body={"error": error.model_dump(mode="json")}) + return BatchResult(custom_id=line.custom_id, status="errored", body=body.model_dump_json()) + + +def _openai_unrun_result(details: Mapping[str, Any], custom_id: str, line_number: int) -> BatchResult: + """Mirror of ``_unrun_result`` in pydantic-ai's OpenAI batch module.""" + if details["status"] == "failed": + # The input file failed validation, so the batch ran none of its requests. A request gets the errors of its + # own line and those of no line; when they're all about other lines, it gets them all, which name them. + errors = details.get("errors", []) + own = [ + error + for error in errors + if not isinstance(error, Mapping) or error.get("line") in (line_number, None) + ] + body = _OpenAILineResponse(status_code=400, body={"errors": own or errors}) + return BatchResult(custom_id=custom_id, status="errored", body=body.model_dump_json()) + return BatchResult( + custom_id=custom_id, status="expired" if details["status"] == "expired" else "canceled" + ) + + +def _stream(source: AsyncGenerator[T, None]) -> Generator[T, None, None]: + """ + Read an async generator from synchronous code without collecting it first. + + The generator runs as a task on its own event loop in a helper thread and hands items over through a + queue of :data:`_RESULT_BUFFER` items, so a slow reader holds the download back instead of buffering it. + An error from the generator is raised at the reader, whatever its type. + + When the reader stops (closes the iterator, is dropped, or fails), the task is cancelled from the reader's + thread. A read the generator is awaiting is interrupted rather than waited for, and the generator's own + cleanup still runs, so whatever it holds open is closed. The wait for the thread is bounded. + """ + items: queue.Queue[ + tuple[Literal["item"], T] | tuple[Literal["error"], BaseException] | tuple[Literal["end"], None] + ] + items = queue.Queue(maxsize=_RESULT_BUFFER) + closed = threading.Event() + started = threading.Event() + running: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[None]]] = [] + + def put(message: tuple[Any, Any]) -> bool: + # A bounded put that gives up once the reader is gone, so a full queue never strands the thread. + while not closed.is_set(): + try: + items.put(message, timeout=0.1) + return True + except queue.Full: + continue + return False + + async def produce() -> None: + try: + async for item in source: + if not put(("item", item)): + return + put(("end", None)) + except BaseException as e: + # Also a cancellation the reader did not ask for: forwarding it keeps the reader from waiting forever. + put(("error", e)) + finally: + await source.aclose() + + async def main() -> None: + task = asyncio.ensure_future(produce()) + running.append((asyncio.get_running_loop(), task)) + started.set() + await asyncio.wait([task]) + + thread = threading.Thread(target=asyncio.run, args=(main(),), name="llm-batch-results", daemon=True) + thread.start() + try: + while True: + try: + message = items.get(timeout=1) + except queue.Empty: + if not thread.is_alive() and items.empty(): + raise RuntimeError("The batch results reader stopped without finishing.") from None + continue + if message[0] == "item": + yield message[1] + elif message[0] == "error": + raise message[1] + else: + return + finally: + closed.set() + if started.wait(timeout=1): + loop, task = running[0] + # A RuntimeError means the loop already finished, so there is nothing left to cancel. + with suppress(RuntimeError): + loop.call_soon_threadsafe(task.cancel) + thread.join(timeout=_STREAM_JOIN_TIMEOUT) + if thread.is_alive(): + log.debug( + "Batch results thread did not stop within the timeout; it is a daemon and will be left behind" + ) + + [email protected] +def _typed_settings(settings_type: type) -> frozenset[str]: + """Return the keys a request may set as typed settings: the generic ones and the provider's own.""" + return _GENERIC_SETTINGS | (set(settings_type.__annotations__) - set(ModelSettings.__annotations__)) + + +def _error_body(error: ModelHTTPError) -> Mapping[str, Any]: Review Comment: Yep, pydantic-ai had already unwrapped `error`, so we were getting `{}` back. `_error_body` now falls back to the body itself, the code comes from `provider_error_code` first, and the message is the provider's own text. A 429 row lands as `rate_limit_exceeded` / "slow down" again, and Anthropic rows now get the provider message too instead of pydantic-ai's wrapped string. Test decodes a real 429 line through the file path (15cabc9db1c). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
