MaksYermak commented on code in PR #43446: URL: https://github.com/apache/airflow/pull/43446#discussion_r1820429560
########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: Review Comment: This code is not needed, because you created a new Hook. And this if-clause was added for deprecate `delegate_to` parameter in old hooks. ########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: + raise RuntimeError( + "The `delegate_to` parameter has been deprecated before and finally" + " removed in this version" + " of Google Provider. You MUST convert it to `impersonate_chain`" + ) + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=None, + ) + self.discovery_doc = discovery_doc + + def get_conn(self) -> Resource: + """ + Establish a connection to the Google Financial Services API. + + :return: A Google Cloud Financial Services API service resource. + """ + if not self.connection: + self.connection = build_from_document(self.discovery_doc) + + return self.connection + + def get_instance(self, instance_resource_uri: str) -> dict: + """ + Get a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to get (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) + + :returns: A dictionary containing the instance metadata + """ + conn = self.get_conn() + response = conn.projects().locations().instances().get(name=instance_resource_uri).execute() + return response + + def create_instance(self, instance_id: str, kms_key_uri: str, location_resource_uri: str) -> dict: + """ + Create a Financial Services Anti-Money Laundering AI instance. + + :param instance_id: Identifier for the instance to create + :param kms_key_uri: URI of the KMS key to that will be used for instance encryption + (format: 'projects/<Project ID>/locations/<Location>/keyRings/<Key Ring>/ + cryptoKeys/<Key>') + + :returns: A dictionary containing metadata for the create instance operation + """ + conn = self.get_conn() + response = ( + conn.projects() + .locations() + .instances() + .create( + parent=location_resource_uri, + instanceId=instance_id, + body={"kmsKey": kms_key_uri}, + ) + .execute() + ) + return response + + def delete_instance(self, instance_resource_uri: str) -> dict: + """ + Delete a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to delete (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) Review Comment: The same `instance_resource_uri` should be divided on parameters. ########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: + raise RuntimeError( + "The `delegate_to` parameter has been deprecated before and finally" + " removed in this version" + " of Google Provider. You MUST convert it to `impersonate_chain`" + ) + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=None, + ) + self.discovery_doc = discovery_doc + + def get_conn(self) -> Resource: + """ + Establish a connection to the Google Financial Services API. + + :return: A Google Cloud Financial Services API service resource. + """ + if not self.connection: + self.connection = build_from_document(self.discovery_doc) + + return self.connection + + def get_instance(self, instance_resource_uri: str) -> dict: + """ + Get a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to get (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) + + :returns: A dictionary containing the instance metadata + """ + conn = self.get_conn() + response = conn.projects().locations().instances().get(name=instance_resource_uri).execute() + return response + + def create_instance(self, instance_id: str, kms_key_uri: str, location_resource_uri: str) -> dict: + """ + Create a Financial Services Anti-Money Laundering AI instance. + + :param instance_id: Identifier for the instance to create + :param kms_key_uri: URI of the KMS key to that will be used for instance encryption Review Comment: The same as mentioned above for `location_resource_uri` and `kms_key_uri`. These URIs should be divided on parameters. ########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: + raise RuntimeError( + "The `delegate_to` parameter has been deprecated before and finally" + " removed in this version" + " of Google Provider. You MUST convert it to `impersonate_chain`" + ) + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=None, + ) + self.discovery_doc = discovery_doc + + def get_conn(self) -> Resource: + """ + Establish a connection to the Google Financial Services API. + + :return: A Google Cloud Financial Services API service resource. + """ + if not self.connection: + self.connection = build_from_document(self.discovery_doc) + + return self.connection + + def get_instance(self, instance_resource_uri: str) -> dict: + """ + Get a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to get (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) + + :returns: A dictionary containing the instance metadata + """ + conn = self.get_conn() + response = conn.projects().locations().instances().get(name=instance_resource_uri).execute() Review Comment: For all our methods in hooks we divide `name` for parameters: `project_id`, `location_id` and `instance_id` and then unite these parameters in the `name` inside the method. For example https://github.com/apache/airflow/blob/main/providers/src/airflow/providers/google/cloud/hooks/dataproc_metastore.py#L117 ########## providers/src/airflow/providers/google/cloud/operators/financial_services.py: ########## @@ -0,0 +1,158 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +from airflow.providers.google.cloud.hooks.financial_services import FinancialServicesHook +from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator + +if TYPE_CHECKING: + from airflow.utils.context import Context + Review Comment: All Operators below should be changed after changes in Hook will be made. Example how the operator should look: https://github.com/apache/airflow/blob/main/providers/src/airflow/providers/google/cloud/operators/dataproc_metastore.py#L196C1-L224C1 ########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: + raise RuntimeError( + "The `delegate_to` parameter has been deprecated before and finally" + " removed in this version" + " of Google Provider. You MUST convert it to `impersonate_chain`" + ) + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=None, + ) + self.discovery_doc = discovery_doc + + def get_conn(self) -> Resource: Review Comment: Could we try to download a discovery document for an authorized user? Firstly authorize the user, then download a discovery doc and after that build a connection. Similar to this [code](https://github.com/apache/airflow/blob/main/providers/src/airflow/providers/google/firebase/hooks/firestore.py#L67C2-L84C1) for firebase ########## providers/src/airflow/providers/google/cloud/hooks/financial_services.py: ########## @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from googleapiclient.discovery import Resource, build_from_document + +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class FinancialServicesHook(GoogleBaseHook): + """ + Hook for interacting with the Google Financial Services API. + + :param discovery_doc: Discovery document for building the Financial Services API + as described `here <https://cloud.google.com/financial-services/anti-money-laundering/docs/reference/rest#discovery-document>`__ + :param gcp_conn_id: Identifier of connection to Google Cloud Platform. + Defaults to "google_cloud_default". + """ + + connection: Resource | None = None + + def __init__( + self, + discovery_doc: dict, + gcp_conn_id: str = "google_cloud_default", + **kwargs, + ) -> None: + if kwargs.get("delegate_to") is not None: + raise RuntimeError( + "The `delegate_to` parameter has been deprecated before and finally" + " removed in this version" + " of Google Provider. You MUST convert it to `impersonate_chain`" + ) + super().__init__( + gcp_conn_id=gcp_conn_id, + impersonation_chain=None, + ) + self.discovery_doc = discovery_doc + + def get_conn(self) -> Resource: + """ + Establish a connection to the Google Financial Services API. + + :return: A Google Cloud Financial Services API service resource. + """ + if not self.connection: + self.connection = build_from_document(self.discovery_doc) + + return self.connection + + def get_instance(self, instance_resource_uri: str) -> dict: + """ + Get a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to get (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) + + :returns: A dictionary containing the instance metadata + """ + conn = self.get_conn() + response = conn.projects().locations().instances().get(name=instance_resource_uri).execute() + return response + + def create_instance(self, instance_id: str, kms_key_uri: str, location_resource_uri: str) -> dict: + """ + Create a Financial Services Anti-Money Laundering AI instance. + + :param instance_id: Identifier for the instance to create + :param kms_key_uri: URI of the KMS key to that will be used for instance encryption + (format: 'projects/<Project ID>/locations/<Location>/keyRings/<Key Ring>/ + cryptoKeys/<Key>') + + :returns: A dictionary containing metadata for the create instance operation + """ + conn = self.get_conn() + response = ( + conn.projects() + .locations() + .instances() + .create( + parent=location_resource_uri, + instanceId=instance_id, + body={"kmsKey": kms_key_uri}, + ) + .execute() + ) + return response + + def delete_instance(self, instance_resource_uri: str) -> dict: + """ + Delete a Financial Services Anti-Money Laundering AI instance. + + :param instance_resource_uri: URI of the instance to delete (format: + 'projects/<Project ID>/locations/<Location>/instances/<Instance ID>) + + :returns: A dictionary containing metadata for the delete instance + operation + """ + conn = self.get_conn() + response = conn.projects().locations().instances().delete(name=instance_resource_uri).execute() + return response + + def get_operation(self, operation_resource_uri: str) -> dict: + """ + Get a Financial Services Anti-Money Laundering AI operation. + + :param operation_resource_uri: URI of the operation to get (format: + 'projects/<Project ID>/locations/<Location>/operations/<Operation ID>) Review Comment: The same `operation_resource_uri` should be divided on parameters. ########## providers/src/airflow/providers/google/cloud/sensors/financial_services.py: ########## @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +from airflow.exceptions import AirflowFailException +from airflow.providers.google.cloud.hooks.financial_services import FinancialServicesHook +from airflow.sensors.base import BaseSensorOperator, PokeReturnValue + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class FinancialServicesOperationSensor(BaseSensorOperator): Review Comment: Could you please explain the reason for this Sensor? I see that this Sensor is waiting when Operation is completed. And if it is the purpose of this Sensor then we don't need this Sensor. Because for this reason in our Operators we use the `wait_for_operation` function. Here's the code with the implementation in [Hook](https://github.com/apache/airflow/blob/main/providers/src/airflow/providers/google/cloud/hooks/cloud_composer.py#L84C1-L91C1) and how we use it in the [Operator](https://github.com/apache/airflow/blob/main/providers/src/airflow/providers/google/cloud/operators/cloud_composer.py#L174C1-L188C1). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
