eladkal commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1129320448
########## airflow/providers/google/cloud/hooks/bigquery_biengine.py: ########## @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery BI Engine Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): + """ + Hook for Google Bigquery Bi Engine reservation API. + + :param gcp_conn_id: The Airflow connection used for GCP credentials. + :param location: The location of the BigQuery resource. + :param impersonation_chain: This is the optional service account to impersonate using short term + credentials. + """ + + hook_name = "Google Bigquery BI Engine" Review Comment: Don't we need to set also `conn_name_attr`, `default_conn_name`, `conn_type` For example: https://github.com/apache/airflow/blob/c3867781e09b7e0e0d19c0991865a2453194d9a8/airflow/providers/google/cloud/hooks/bigquery.py#L85-L88 do we want users to use BigQuery conn for this hook or have a separated one? ########## airflow/providers/google/cloud/hooks/bigquery_biengine.py: ########## @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery BI Engine Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): + """ + Hook for Google Bigquery Bi Engine reservation API. + + :param gcp_conn_id: The Airflow connection used for GCP credentials. + :param location: The location of the BigQuery resource. + :param impersonation_chain: This is the optional service account to impersonate using short term + credentials. + """ + + hook_name = "Google Bigquery BI Engine" + + def __init__( + self, + gcp_conn_id: str = GoogleBaseHook.default_conn_name, + location: str | None = None, + impersonation_chain: str | Sequence[str] | None = None, + ) -> None: + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=impersonation_chain, + ) + self.location = location + self._client: ReservationServiceClient | None = None + + @staticmethod + def _convert_gb_to_kb(value: int) -> int: + """ + Convert GB value to KB. + + :param value: Value to convert + """ + return value * 1073741824 + + def get_client(self) -> ReservationServiceClient: + """ + Get reservation service client. + + :return: Google Bigquery Reservation client + """ + if not self._client: + self._client = ReservationServiceClient( + credentials=self.get_credentials(), client_info=CLIENT_INFO + ) + return self._client + + @GoogleBaseHook.fallback_to_default_project_id + def create_bi_reservation(self, project_id: str, size: int) -> None: + """ + Create BI Engine reservation + + :param project_id: The name of the project where we want to create/update + the BI Engine reservation. + :param size: The BI Engine reservation size in gigabyte. + """ + parent = f"projects/{project_id}/locations/{self.location}/biReservation" + client = self.get_client() + size = self._convert_gb_to_kb(value=size) + + try: + bi_reservation = client.get_bi_reservation(name=parent) + bi_reservation.size = size + bi_reservation.size + + client.update_bi_reservation(bi_reservation=bi_reservation) + + self.log.info("BI Engine reservation {parent} have been updated to {bi_reservation.size}Kb.") + except Exception as e: + self.log.error(e) + raise AirflowException(f"Failed to create BI engine reservation of {size}.") + + @GoogleBaseHook.fallback_to_default_project_id + def delete_bi_reservation(self, project_id: str, size: int | None = None) -> None: + """ + Delete/Update BI Engine reservation with the specified memory size + + :param project_id: The name of the project where we want to delete/update + the BI Engine reservation. + :param size: The BI Engine reservation size in gigabyte. + """ + parent = f"projects/{project_id}/locations/{self.location}/biReservation" + client = self.get_client() + try: + bi_reservation = client.get_bi_reservation(name=parent) + if size is not None: + size = self._convert_gb_to_kb(size) + bi_reservation.size = max(bi_reservation.size - size, 0) + else: + bi_reservation.size = 0 + + client.update_bi_reservation(bi_reservation=bi_reservation) + self.log.info("BI Engine reservation {parent} have been updated to {bi_reservation.size}Kb.") + except Exception as e: Review Comment: This is very broad. We should catch only the relevant exception -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
