MaksYermak commented on code in PR #68479: URL: https://github.com/apache/airflow/pull/68479#discussion_r3419014610
########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + +if TYPE_CHECKING: + from vertexai._genai import types + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @staticmethod + def build_agent_engine_name(project_id: str, location: str, agent_engine_id: str) -> str: + """Build a fully qualified Agent Engine resource name.""" + return f"projects/{project_id}/locations/{location}/reasoningEngines/{agent_engine_id}" + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + config: Any | None = None, Review Comment: @AlejandroMorgante the type here should be consistent with a type in the `Client` as I see in the client they use `types.AgentEngineConfigOrDict` type for `config` parameter, could you please update it here? ########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + +if TYPE_CHECKING: + from vertexai._genai import types + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @staticmethod + def build_agent_engine_name(project_id: str, location: str, agent_engine_id: str) -> str: + """Build a fully qualified Agent Engine resource name.""" + return f"projects/{project_id}/locations/{location}/reasoningEngines/{agent_engine_id}" + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Create an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param config: Optional. Configuration for the Agent Engine. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.create(agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def get_agent_engine( + self, + location: str, + agent_engine_id: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Get an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.get(name=name) + + @GoogleBaseHook.fallback_to_default_project_id + def query_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any | None = None, + request_timeout: float | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Query an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Optional. Configuration for the query request (``class_method``, ``input``). + :param request_timeout: Optional. Timeout in seconds for the HTTP request. Defaults to no timeout. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + # Use the SDK's _api_client.request() directly rather than the SDK's run_query_job + # (requires GCS) or _query (private method; triggers a Pydantic parsing bug in + # google-genai 2.8.0 when the response output type is Any). Calling request() bypasses + # Pydantic parsing while still letting the SDK handle URL construction and auth. + # Replace with a public synchronous query API when available; tracked at + # https://github.com/apache/airflow/issues/68605 + cfg = config if isinstance(config, dict) else {} + body: dict[str, Any] = {"classMethod": cfg.get("class_method", "query")} + if "input" in cfg: + input_val = cfg["input"] + if isinstance(input_val, str): + try: + input_val = json.loads(input_val) + except json.JSONDecodeError as err: + raise ValueError("Agent Engine query input must be valid JSON.") from err + if not isinstance(input_val, dict): + raise ValueError("Agent Engine query input must be a JSON object.") + body["input"] = input_val + + sdk_client = self.get_agent_engine_client(project_id=project_id, location=location) + http_options = HttpOptions( + timeout=int(request_timeout * 1000) if request_timeout is not None else None + ) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + api_client = getattr(sdk_client, "_api_client", None) + request = getattr(api_client, "request", None) + if request is None: + raise RuntimeError( + "The Vertex AI Agent Engine SDK no longer exposes _api_client.request. " + "QueryAgentEngineOperator must be updated to use a supported synchronous query API." + ) + response = request("post", f"{name}:query", body, http_options) + data = {} if not response.body else json.loads(response.body) + output = data.get("output") + return output if output is not None else data + + @GoogleBaseHook.fallback_to_default_project_id + def update_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any, + agent: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Update an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Required. Configuration for the Agent Engine update. + :param agent: Optional. The updated agent object to deploy. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.update(name=name, agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def delete_agent_engine( + self, + location: str, + agent_engine_id: str, + force: bool | None = None, + config: Any | None = None, Review Comment: @AlejandroMorgante the type here should be consistent with a type in the `Client` as I see in the client they use `types.DeleteAgentEngineConfigOrDict` type for config parameter, could you please update it here? ########## providers/google/src/airflow/providers/google/cloud/operators/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,402 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains Google Vertex AI Agent Engine operators.""" + +from __future__ import annotations + +from collections.abc import Sequence +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.compat.sdk import conf +from airflow.providers.google.cloud.hooks.vertex_ai.agent_engine import AgentEngineHook +from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator +from airflow.providers.google.cloud.triggers.vertex_ai import AgentEngineDeleteTrigger + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +def _serialize_value(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, dict): + return {key: _serialize_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_serialize_value(item) for item in value] + if isinstance(value, tuple): + return tuple(_serialize_value(item) for item in value) + return value + + +def _serialize_agent_engine(agent_engine: Any) -> dict[str, Any]: + api_resource = getattr(agent_engine, "api_resource", None) + if api_resource is not None: + return _serialize_value(api_resource) + return _serialize_value(agent_engine) + + +class CreateAgentEngineOperator(GoogleCloudBaseOperator): + """ + Create a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param config: Optional. Configuration for the Agent Engine. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + """ + + template_fields = ( + "project_id", + "location", + "agent", + "config", + "gcp_conn_id", + "impersonation_chain", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + agent: Any | None = None, + config: Any | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.agent = agent + self.config = config + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> dict[str, Any]: + self.log.info("Creating Agent Engine.") + agent_engine = self.hook.create_agent_engine( + project_id=self.project_id, + location=self.location, + agent=self.agent, + config=self.config, + ) + result = _serialize_agent_engine(agent_engine) Review Comment: @AlejandroMorgante as I see this agent_engine variable has a `AgentEngine` type. Mostly for google types we have a default method for converting to dict from obj as [example](https://github.com/apache/airflow/blob/main/providers/google/src/airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py#L121). Could you please check for `AgentEngine` type and update this code if the method or his equivalent exists? ########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + +if TYPE_CHECKING: + from vertexai._genai import types + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @staticmethod + def build_agent_engine_name(project_id: str, location: str, agent_engine_id: str) -> str: + """Build a fully qualified Agent Engine resource name.""" + return f"projects/{project_id}/locations/{location}/reasoningEngines/{agent_engine_id}" + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Create an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param config: Optional. Configuration for the Agent Engine. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.create(agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def get_agent_engine( + self, + location: str, + agent_engine_id: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Get an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.get(name=name) + + @GoogleBaseHook.fallback_to_default_project_id + def query_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any | None = None, + request_timeout: float | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Query an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Optional. Configuration for the query request (``class_method``, ``input``). + :param request_timeout: Optional. Timeout in seconds for the HTTP request. Defaults to no timeout. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + # Use the SDK's _api_client.request() directly rather than the SDK's run_query_job + # (requires GCS) or _query (private method; triggers a Pydantic parsing bug in + # google-genai 2.8.0 when the response output type is Any). Calling request() bypasses + # Pydantic parsing while still letting the SDK handle URL construction and auth. + # Replace with a public synchronous query API when available; tracked at + # https://github.com/apache/airflow/issues/68605 + cfg = config if isinstance(config, dict) else {} + body: dict[str, Any] = {"classMethod": cfg.get("class_method", "query")} + if "input" in cfg: + input_val = cfg["input"] + if isinstance(input_val, str): + try: + input_val = json.loads(input_val) + except json.JSONDecodeError as err: + raise ValueError("Agent Engine query input must be valid JSON.") from err + if not isinstance(input_val, dict): + raise ValueError("Agent Engine query input must be a JSON object.") + body["input"] = input_val + + sdk_client = self.get_agent_engine_client(project_id=project_id, location=location) + http_options = HttpOptions( + timeout=int(request_timeout * 1000) if request_timeout is not None else None + ) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + api_client = getattr(sdk_client, "_api_client", None) + request = getattr(api_client, "request", None) + if request is None: + raise RuntimeError( + "The Vertex AI Agent Engine SDK no longer exposes _api_client.request. " + "QueryAgentEngineOperator must be updated to use a supported synchronous query API." + ) + response = request("post", f"{name}:query", body, http_options) + data = {} if not response.body else json.loads(response.body) + output = data.get("output") + return output if output is not None else data + + @GoogleBaseHook.fallback_to_default_project_id + def update_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any, + agent: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Update an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Required. Configuration for the Agent Engine update. + :param agent: Optional. The updated agent object to deploy. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.update(name=name, agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def delete_agent_engine( + self, + location: str, + agent_engine_id: str, + force: bool | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.DeleteAgentEngineOperation: + """ + Delete an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param force: Optional. Whether to forcefully delete child resources. Defaults to ``False`` + when not specified. + :param config: Optional. Additional deletion configuration. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.delete(name=name, force=force, config=config) + + def is_agent_engine_deleted(self, project_id: str, location: str, agent_engine_id: str) -> bool: + """Return whether an Agent Engine no longer exists.""" + try: + self.get_agent_engine( + project_id=project_id, + location=location, + agent_engine_id=agent_engine_id, + ) + except ClientError as err: + if getattr(err, "code", None) == 404: + return True + raise + return False + + def wait_for_agent_engine_deleted( + self, + project_id: str, + location: str, + agent_engine_id: str, + poll_interval: float = 30, + timeout: float | None = None, + ) -> None: + """ + Wait until an Agent Engine no longer exists. + + :param project_id: The ID of the Google Cloud project that the service belongs to. + :param location: The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: The Agent Engine ID. + :param poll_interval: Time, in seconds, to wait between checks. + :param timeout: Optional timeout, in seconds. + """ + start_time = time.monotonic() + while True: + if self.is_agent_engine_deleted( + project_id=project_id, + location=location, + agent_engine_id=agent_engine_id, + ): + return + if timeout is not None and time.monotonic() - start_time > timeout: + raise TimeoutError(f"Timed out waiting for Agent Engine {agent_engine_id} to be deleted") + self.log.info("Waiting for Agent Engine %s to be deleted.", agent_engine_id) + time.sleep(poll_interval) Review Comment: @AlejandroMorgante also I am not sure that we need it because we have `DeleteAgentEngineOperation` object for delete and we can monitor its states. ########## providers/google/src/airflow/providers/google/cloud/operators/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,391 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains Google Vertex AI Agent Engine operators.""" + +from __future__ import annotations + +from collections.abc import Sequence +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.compat.sdk import conf +from airflow.providers.google.cloud.hooks.vertex_ai.agent_engine import AgentEngineHook +from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator +from airflow.providers.google.cloud.triggers.vertex_ai import AgentEngineDeleteTrigger + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +def _serialize_value(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, dict): + return {key: _serialize_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_serialize_value(item) for item in value] + if isinstance(value, tuple): + return tuple(_serialize_value(item) for item in value) + return value + + +def _serialize_agent_engine(agent_engine: Any) -> dict[str, Any]: + api_resource = getattr(agent_engine, "api_resource", None) + if api_resource is not None: + return _serialize_value(api_resource) + return _serialize_value(agent_engine) + + +class CreateAgentEngineOperator(GoogleCloudBaseOperator): + """ + Create a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param agent_engine: Optional. Deprecated alias for ``agent``. + :param config: Optional. Configuration for the Agent Engine. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + """ + + template_fields = ( + "project_id", + "location", + "agent", + "agent_engine", + "config", + "gcp_conn_id", + "impersonation_chain", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + agent: Any | None = None, + agent_engine: Any | None = None, + config: Any | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.agent = agent + self.agent_engine = agent_engine + self.config = config + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> dict[str, Any]: + agent_engine = self.hook.create_agent_engine( + project_id=self.project_id, + location=self.location, + agent=self.agent, + agent_engine=self.agent_engine, + config=self.config, + ) + return _serialize_agent_engine(agent_engine) + + +class GetAgentEngineOperator(GoogleCloudBaseOperator): + """ + Get a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param name: Required. The Agent Engine resource name. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + """ + + template_fields = ("project_id", "location", "name", "gcp_conn_id", "impersonation_chain") + + def __init__( + self, + *, + project_id: str, + location: str, + name: str, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.name = name + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> dict[str, Any]: + agent_engine = self.hook.get_agent_engine( + project_id=self.project_id, + location=self.location, + name=self.name, + ) + return _serialize_agent_engine(agent_engine) + + +class QueryAgentEngineOperator(GoogleCloudBaseOperator): + """ + Query a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param name: Required. The Agent Engine resource name. + :param config: Optional. Configuration for the query request (``class_method``, ``input``). + :param request_timeout: Optional. Timeout in seconds for the HTTP request. Defaults to no timeout. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + """ + + template_fields = ("project_id", "location", "name", "config", "gcp_conn_id", "impersonation_chain") + + def __init__( + self, + *, + project_id: str, + location: str, + name: str, + config: Any | None = None, + request_timeout: float | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.name = name + self.config = config + self.request_timeout = request_timeout + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> Any: + return self.hook.query_agent_engine( + project_id=self.project_id, + location=self.location, + name=self.name, + config=self.config, + request_timeout=self.request_timeout, + ) + + +class UpdateAgentEngineOperator(GoogleCloudBaseOperator): + """ + Update a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param name: Required. The Agent Engine resource name. + :param agent: Optional. The updated agent object to deploy. + :param agent_engine: Optional. Deprecated alias for ``agent``. + :param config: Required. Configuration for the Agent Engine update. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + """ + + template_fields = ( + "project_id", + "location", + "name", + "agent", + "agent_engine", + "config", + "gcp_conn_id", + "impersonation_chain", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + name: str, + config: Any, + agent: Any | None = None, + agent_engine: Any | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.name = name + self.agent = agent + self.agent_engine = agent_engine + self.config = config + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> dict[str, Any]: + agent_engine = self.hook.update_agent_engine( + project_id=self.project_id, + location=self.location, + name=self.name, + agent=self.agent, + agent_engine=self.agent_engine, + config=self.config, + ) + return _serialize_agent_engine(agent_engine) + + +class DeleteAgentEngineOperator(GoogleCloudBaseOperator): + """ + Delete a Vertex AI Agent Engine. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param name: Required. The Agent Engine resource name. + :param force: Optional. Whether to delete child resources. + :param config: Optional. Additional deletion configuration. + :param wait_for_completion: Whether to wait until the Agent Engine no longer exists. + :param poll_interval: Time, in seconds, to wait between checks. + :param timeout: Optional timeout, in seconds. + :param gcp_conn_id: The connection ID to use connecting to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term credentials. + :param deferrable: Run operator in the deferrable mode. + """ + + template_fields = ( + "project_id", + "location", + "name", + "force", + "config", + "gcp_conn_id", + "impersonation_chain", + ) + + def __init__( + self, + *, + project_id: str, + location: str, + name: str, + force: bool | None = None, + config: Any | None = None, + wait_for_completion: bool = True, + poll_interval: float = 30, + timeout: float | None = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.location = location + self.name = name + self.force = force + self.config = config + self.wait_for_completion = wait_for_completion + self.poll_interval = poll_interval + self.timeout = timeout + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + self.deferrable = deferrable + + @cached_property + def hook(self) -> AgentEngineHook: + return AgentEngineHook( + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + ) + + def execute(self, context: Context) -> dict[str, Any]: + operation = self.hook.delete_agent_engine( + project_id=self.project_id, + location=self.location, + name=self.name, + force=self.force, + config=self.config, + ) + result = _serialize_value(operation) + if not self.wait_for_completion: + return result + + if self.deferrable: + self.defer( + trigger=AgentEngineDeleteTrigger( + project_id=self.project_id, + location=self.location, + name=self.name, + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + poll_interval=self.poll_interval, + timeout=self.timeout, + ), + method_name="execute_complete", + kwargs={"operation": result}, + ) + + self.hook.wait_for_agent_engine_deleted( + project_id=self.project_id, + location=self.location, + name=self.name, + poll_interval=self.poll_interval, + timeout=self.timeout, + ) + return result Review Comment: I see, than we should monitor `DeleteAgentEngineOperation` state using custom approach. As I see this object has `done` property. And this property can be in two states. We had this case before in several operators for example [here](https://github.com/apache/airflow/blob/main/providers/google/src/airflow/providers/google/cloud/triggers/cloud_composer.py#L80-L86) ``` class DeleteAgentEngineOperation(_common.BaseModel): """Operation for deleting agent engines.""" name: Optional[str] = Field( default=None, description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", ) metadata: Optional[dict[str, Any]] = Field( default=None, description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", ) done: Optional[bool] = Field( default=None, description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", ) error: Optional[dict[str, Any]] = Field( default=None, description="""The error result of the operation in case of failure or cancellation.""", ) ``` ########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + +if TYPE_CHECKING: + from vertexai._genai import types + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @staticmethod + def build_agent_engine_name(project_id: str, location: str, agent_engine_id: str) -> str: + """Build a fully qualified Agent Engine resource name.""" + return f"projects/{project_id}/locations/{location}/reasoningEngines/{agent_engine_id}" + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Create an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param config: Optional. Configuration for the Agent Engine. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.create(agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def get_agent_engine( + self, + location: str, + agent_engine_id: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Get an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.get(name=name) + + @GoogleBaseHook.fallback_to_default_project_id + def query_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any | None = None, + request_timeout: float | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Query an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Optional. Configuration for the query request (``class_method``, ``input``). + :param request_timeout: Optional. Timeout in seconds for the HTTP request. Defaults to no timeout. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + # Use the SDK's _api_client.request() directly rather than the SDK's run_query_job + # (requires GCS) or _query (private method; triggers a Pydantic parsing bug in + # google-genai 2.8.0 when the response output type is Any). Calling request() bypasses + # Pydantic parsing while still letting the SDK handle URL construction and auth. + # Replace with a public synchronous query API when available; tracked at + # https://github.com/apache/airflow/issues/68605 + cfg = config if isinstance(config, dict) else {} + body: dict[str, Any] = {"classMethod": cfg.get("class_method", "query")} + if "input" in cfg: + input_val = cfg["input"] + if isinstance(input_val, str): + try: + input_val = json.loads(input_val) + except json.JSONDecodeError as err: + raise ValueError("Agent Engine query input must be valid JSON.") from err + if not isinstance(input_val, dict): + raise ValueError("Agent Engine query input must be a JSON object.") + body["input"] = input_val + + sdk_client = self.get_agent_engine_client(project_id=project_id, location=location) + http_options = HttpOptions( + timeout=int(request_timeout * 1000) if request_timeout is not None else None + ) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + api_client = getattr(sdk_client, "_api_client", None) + request = getattr(api_client, "request", None) + if request is None: + raise RuntimeError( + "The Vertex AI Agent Engine SDK no longer exposes _api_client.request. " + "QueryAgentEngineOperator must be updated to use a supported synchronous query API." + ) + response = request("post", f"{name}:query", body, http_options) + data = {} if not response.body else json.loads(response.body) + output = data.get("output") + return output if output is not None else data + + @GoogleBaseHook.fallback_to_default_project_id + def update_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any, + agent: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Update an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Required. Configuration for the Agent Engine update. + :param agent: Optional. The updated agent object to deploy. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.update(name=name, agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def delete_agent_engine( + self, + location: str, + agent_engine_id: str, + force: bool | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.DeleteAgentEngineOperation: + """ + Delete an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param force: Optional. Whether to forcefully delete child resources. Defaults to ``False`` + when not specified. + :param config: Optional. Additional deletion configuration. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.delete(name=name, force=force, config=config) + + def is_agent_engine_deleted(self, project_id: str, location: str, agent_engine_id: str) -> bool: + """Return whether an Agent Engine no longer exists.""" + try: + self.get_agent_engine( + project_id=project_id, + location=location, + agent_engine_id=agent_engine_id, + ) + except ClientError as err: + if getattr(err, "code", None) == 404: + return True + raise + return False Review Comment: @AlejandroMorgante you do not need this code, because delete method returns `DeleteAgentEngineOperation` object with `done` property. ``` done: Optional[bool] = Field( default=None, description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", ) ``` You can monitor the state of this object for example as we do [here](https://github.com/apache/airflow/blob/main/providers/google/src/airflow/providers/google/cloud/triggers/cloud_composer.py#L80-L86) ########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,280 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + +if TYPE_CHECKING: + from vertexai._genai import types + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @staticmethod + def build_agent_engine_name(project_id: str, location: str, agent_engine_id: str) -> str: + """Build a fully qualified Agent Engine resource name.""" + return f"projects/{project_id}/locations/{location}/reasoningEngines/{agent_engine_id}" + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Create an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param config: Optional. Configuration for the Agent Engine. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.create(agent=agent, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def get_agent_engine( + self, + location: str, + agent_engine_id: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> types.AgentEngine: + """ + Get an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + return client.get(name=name) + + @GoogleBaseHook.fallback_to_default_project_id + def query_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any | None = None, + request_timeout: float | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Query an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent_engine_id: Required. The Agent Engine ID. + :param config: Optional. Configuration for the query request (``class_method``, ``input``). + :param request_timeout: Optional. Timeout in seconds for the HTTP request. Defaults to no timeout. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + # Use the SDK's _api_client.request() directly rather than the SDK's run_query_job + # (requires GCS) or _query (private method; triggers a Pydantic parsing bug in + # google-genai 2.8.0 when the response output type is Any). Calling request() bypasses + # Pydantic parsing while still letting the SDK handle URL construction and auth. + # Replace with a public synchronous query API when available; tracked at + # https://github.com/apache/airflow/issues/68605 + cfg = config if isinstance(config, dict) else {} + body: dict[str, Any] = {"classMethod": cfg.get("class_method", "query")} + if "input" in cfg: + input_val = cfg["input"] + if isinstance(input_val, str): + try: + input_val = json.loads(input_val) + except json.JSONDecodeError as err: + raise ValueError("Agent Engine query input must be valid JSON.") from err + if not isinstance(input_val, dict): + raise ValueError("Agent Engine query input must be a JSON object.") + body["input"] = input_val + + sdk_client = self.get_agent_engine_client(project_id=project_id, location=location) + http_options = HttpOptions( + timeout=int(request_timeout * 1000) if request_timeout is not None else None + ) + name = self.build_agent_engine_name(project_id, location, agent_engine_id) + api_client = getattr(sdk_client, "_api_client", None) + request = getattr(api_client, "request", None) + if request is None: + raise RuntimeError( + "The Vertex AI Agent Engine SDK no longer exposes _api_client.request. " + "QueryAgentEngineOperator must be updated to use a supported synchronous query API." + ) + response = request("post", f"{name}:query", body, http_options) + data = {} if not response.body else json.loads(response.body) + output = data.get("output") + return output if output is not None else data + + @GoogleBaseHook.fallback_to_default_project_id + def update_agent_engine( + self, + location: str, + agent_engine_id: str, + config: Any, Review Comment: @AlejandroMorgante the type here should be consistent with a type in the `Client` as I see in the client they use `types.AgentEngineConfigOrDict` type for config parameter, could you please update it here? ########## providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/agent_engine.py: ########## @@ -0,0 +1,254 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a Google Cloud Vertex AI Agent Engine hook.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from typing import Any + +from asgiref.sync import sync_to_async +from google.genai._api_client import HttpOptions +from google.genai.errors import ClientError +from vertexai import Client + +from airflow.providers.google.common.hooks.base_google import ( + PROVIDE_PROJECT_ID, + GoogleBaseAsyncHook, + GoogleBaseHook, +) + + +class AgentEngineHook(GoogleBaseHook): + """Hook for Google Cloud Vertex AI Agent Engine APIs.""" + + def __init__( + self, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + **kwargs, + ) + + def get_agent_engine_client(self, project_id: str, location: str): + """Return the Vertex AI Agent Engine client.""" + return Client( + project=project_id, + location=location, + credentials=self.get_credentials(), + ).agent_engines + + @GoogleBaseHook.fallback_to_default_project_id + def create_agent_engine( + self, + location: str, + agent: Any | None = None, + agent_engine: Any | None = None, + config: Any | None = None, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Create an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param agent: Optional. The agent object to deploy. + :param agent_engine: Optional. Deprecated alias for ``agent``. + :param config: Optional. Configuration for the Agent Engine. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.create(agent=agent, agent_engine=agent_engine, config=config) + + @GoogleBaseHook.fallback_to_default_project_id + def get_agent_engine( + self, + location: str, + name: str, + project_id: str = PROVIDE_PROJECT_ID, + ) -> Any: + """ + Get an Agent Engine. + + :param location: Required. The ID of the Google Cloud location that the service belongs to. + :param name: Required. The Agent Engine resource name. + :param project_id: Optional. The ID of the Google Cloud project. Defaults to the project + configured in the connection. + """ + client = self.get_agent_engine_client(project_id=project_id, location=location) + return client.get(name=name) + + @GoogleBaseHook.fallback_to_default_project_id + def query_agent_engine( Review Comment: @AlejandroMorgante could we discuss it from Airflow users perspective and could you explain please what users should expect when use this method and when use `run_query_job`? As I see `run_query_job` is a public synchronous method which can be used for create a `QueryJob` and return an `Operation` object. And it is okay for using `AsyncQuery` inside this method, because it creates `Operation` and the client should not wait for `Operation` results. With this `Operation` object we can work on Airflow side. We can wait for operation's result in `deferreable` or `non-deferreable` mode and, in the end, return results for the user. In the current implementation I do not understand what result user will get. It can be `Any` type. And from this code I do not understand what data will be in the response as I see it can be `None`. Could you also clarify this moment? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
