fdemiane commented on code in PR #32606: URL: https://github.com/apache/airflow/pull/32606#discussion_r1269092756
########## airflow/providers/google/cloud/operators/cloud_batch.py: ########## @@ -0,0 +1,327 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Optional, Sequence, Union + +from google.api_core import operation # type: ignore +from google.cloud.batch_v1 import Job, Task + +from airflow.exceptions import AirflowException +from airflow.models.baseoperator import BaseOperator +from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchHook +from airflow.providers.google.cloud.triggers.cloud_batch import \ + CloudBatchJobFinishedTrigger +from airflow.utils.context import Context + + +class CloudBatchSubmitJobOperator(BaseOperator): + """ + Submit a job and wait for its completion. + + :param project_id: Required. The ID of the Google Cloud project that the service belongs to. + :param region: Required. The ID of the Google Cloud region that the service belongs to. + :param job_name: Required. The name of the job to create. + :param job: Required. The job descriptor containing the configuration of the job to submit. + :param polling_period_seconds: Optional: Control the rate of the poll for the result of deferrable run. + By default, the trigger will poll every 10 seconds. + :param timeout: The timeout for this request. + :param gcp_conn_id: The connection ID used to connect to Google Cloud. + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). + :param deferrable: Run operator in the deferrable mode + + """ + + template_fields = ( + 'project_id', + 'region', + 'gcp_conn_id', + 'impersonation_chain', + 'job_name' + ) + + def __init__( + self, + project_id: str, + region: str, + job_name: str, + job: Job, + polling_period_seconds: float = 10, + timeout_seconds: Union[float, None] = None, + gcp_conn_id: str = "google_cloud_default", + impersonation_chain: Optional[Union[str, Sequence[str]]] = None, + deferrable: bool = False, + **kwargs + ) -> None: + super().__init__(**kwargs) + self.project_id = project_id + self.region = region + self.job_name = job_name + self.job = job + self.polling_period_seconds = polling_period_seconds + self.timeout_seconds = timeout_seconds + self.gcp_conn_id = gcp_conn_id + self.impersonation_chain = impersonation_chain + self.deferrable = deferrable + self.polling_period_seconds = polling_period_seconds + + def execute(self, context): + hook: CloudBatchHook = CloudBatchHook( + self.gcp_conn_id, self.impersonation_chain) + job = hook.submit_build_job( + job_name=self.job_name, job=self.job, region=self.region) + + if not self.deferrable: + completed_job = hook.wait_for_job( + job_name=job.name, + polling_period_seconds=self.polling_period_seconds, + timeout=self.timeout_seconds) + + return Job.to_dict(completed_job) + + else: + self.defer( + trigger=CloudBatchJobFinishedTrigger( + job_name=job.name, + project_id=self.project_id, + gcp_conn_id=self.gcp_conn_id, + impersonation_chain=self.impersonation_chain, + location=self.region, + polling_period_seconds=self.polling_period_seconds Review Comment: comment to self: pass timeout_seconds -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
