This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 47a9c8a4a4 Added Datascan Profiling (#35696)
47a9c8a4a4 is described below
commit 47a9c8a4a4ecc4da34bd210d56331e97f9fe8e7e
Author: shourya116 <[email protected]>
AuthorDate: Tue Dec 12 11:53:46 2023 +0530
Added Datascan Profiling (#35696)
---
.../providers/google/cloud/operators/dataplex.py | 531 ++++++++++++++++++++-
airflow/providers/google/cloud/sensors/dataplex.py | 118 +++++
.../providers/google/cloud/triggers/dataplex.py | 82 ++++
.../operators/cloud/dataplex.rst | 96 ++++
.../google/cloud/operators/test_dataplex.py | 126 +++++
.../google/cloud/sensors/test_dataplex.py | 80 ++++
.../google/cloud/dataplex/example_dataplex_dp.py | 341 +++++++++++++
7 files changed, 1373 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/google/cloud/operators/dataplex.py
b/airflow/providers/google/cloud/operators/dataplex.py
index b75024fbf4..66be3e4d9c 100644
--- a/airflow/providers/google/cloud/operators/dataplex.py
+++ b/airflow/providers/google/cloud/operators/dataplex.py
@@ -22,7 +22,10 @@ import time
from typing import TYPE_CHECKING, Any, Sequence
from airflow.exceptions import AirflowException
-from airflow.providers.google.cloud.triggers.dataplex import
DataplexDataQualityJobTrigger
+from airflow.providers.google.cloud.triggers.dataplex import (
+ DataplexDataProfileJobTrigger,
+ DataplexDataQualityJobTrigger,
+)
if TYPE_CHECKING:
from google.protobuf.field_mask_pb2 import FieldMask
@@ -1204,6 +1207,532 @@ class
DataplexGetDataQualityScanResultOperator(GoogleCloudBaseOperator):
return job
+class DataplexCreateOrUpdateDataProfileScanOperator(GoogleCloudBaseOperator):
+ """
+ Creates a DataScan Data Profile resource.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
lake belongs to.
+ :param region: Required. The ID of the Google Cloud region that the lake
belongs to.
+ :param body: Required. The Request body contains an instance of DataScan.
+ :param data_scan_id: Required. Data Profile scan identifier.
+ :param update_mask: Mask of fields to update.
+ :param api_version: The version of the api that will be requested for
example 'v1'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param timeout: The amount of time, in seconds, to wait for the request to
complete.
+ Note that if `retry` is specified, the timeout applies to each
individual attempt.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+
+ :return: Dataplex data profile id
+ """
+
+ template_fields = ("project_id", "data_scan_id", "body",
"impersonation_chain")
+ template_fields_renderers = {"body": "json"}
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ body: dict[str, Any] | DataScan,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ timeout: float | None = None,
+ update_mask: dict | FieldMask | None = None,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.body = body
+ self.update_mask = update_mask
+ self.api_version = api_version
+ self.retry = retry
+ self.timeout = timeout
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+
+ def execute(self, context: Context):
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ self.log.info("Creating Dataplex Data Profile scan %s",
self.data_scan_id)
+ try:
+ operation = hook.create_data_scan(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ body=self.body,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ hook.wait_for_operation(timeout=self.timeout, operation=operation)
+ self.log.info("Dataplex Data Profile scan %s created
successfully!", self.data_scan_id)
+ except AlreadyExists:
+ self.log.info("Dataplex Data Profile scan already exists: %s",
{self.data_scan_id})
+
+ operation = hook.update_data_scan(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ body=self.body,
+ update_mask=self.update_mask,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ hook.wait_for_operation(timeout=self.timeout, operation=operation)
+ self.log.info("Dataplex Data Profile scan %s updated
successfully!", self.data_scan_id)
+ except GoogleAPICallError as e:
+ raise AirflowException(f"Error creating Data Profile scan
{self.data_scan_id}", e)
+
+ return self.data_scan_id
+
+
+class DataplexGetDataProfileScanOperator(GoogleCloudBaseOperator):
+ """
+ Gets a DataScan DataProfile resource.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
lake belongs to.
+ :param region: Required. The ID of the Google Cloud region that the lake
belongs to.
+ :param data_scan_id: Required. Data Profile scan identifier.
+ :param api_version: The version of the api that will be requested for
example 'v1'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param timeout: The amount of time, in seconds, to wait for the request to
complete.
+ Note that if `retry` is specified, the timeout applies to each
individual attempt.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+
+ :return: Dataplex data profile
+ """
+
+ template_fields = ("project_id", "data_scan_id", "impersonation_chain")
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ timeout: float | None = None,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.api_version = api_version
+ self.retry = retry
+ self.timeout = timeout
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+
+ def execute(self, context: Context):
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ self.log.info("Retrieving the details of Dataplex Data Profile scan
%s", self.data_scan_id)
+ data_profile_scan = hook.get_data_scan(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+
+ return DataScan.to_dict(data_profile_scan)
+
+
+class DataplexDeleteDataProfileScanOperator(GoogleCloudBaseOperator):
+ """
+ Deletes a DataScan DataProfile resource.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
lake belongs to.
+ :param region: Required. The ID of the Google Cloud region that the lake
belongs to.
+ :param data_scan_id: Required. Data Profile scan identifier.
+ :param api_version: The version of the api that will be requested for
example 'v1'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param timeout: The amount of time, in seconds, to wait for the request to
complete.
+ Note that if `retry` is specified, the timeout applies to each
individual attempt.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+ :return: None
+ """
+
+ template_fields = ("project_id", "data_scan_id", "impersonation_chain")
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ timeout: float | None = None,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.api_version = api_version
+ self.retry = retry
+ self.timeout = timeout
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+
+ def execute(self, context: Context) -> None:
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ self.log.info("Deleting Dataplex Data Profile Scan: %s",
self.data_scan_id)
+
+ operation = hook.delete_data_scan(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ hook.wait_for_operation(timeout=self.timeout, operation=operation)
+ self.log.info("Dataplex Data Profile scan %s deleted successfully!",
self.data_scan_id)
+
+
+class DataplexRunDataProfileScanOperator(GoogleCloudBaseOperator):
+ """
+ Runs an on-demand execution of a DataScan Data Profile Scan.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
lake belongs to.
+ :param region: Required. The ID of the Google Cloud region that the lake
belongs to.
+ :param data_scan_id: Required. Data Profile scan identifier.
+ :param api_version: The version of the api that will be requested for
example 'v1'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param timeout: The amount of time, in seconds, to wait for the request to
complete.
+ Note that if `retry` is specified, the timeout applies to each
individual attempt.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+ :param asynchronous: Flag informing that the Dataplex job should be run
asynchronously.
+ This is useful for submitting long-running jobs and
+ waiting on them asynchronously using the
DataplexDataProfileJobStatusSensor
+ :param result_timeout: Value in seconds for which operator will wait for
the Data Profile scan result
+ when the flag `asynchronous = False`.
+ Throws exception if there is no result found after specified amount of
seconds.
+ :param polling_interval_seconds: time in seconds between polling for job
completion.
+ The value is considered only when running in deferrable mode. Must be
greater than 0.
+ :param deferrable: Run operator in the deferrable mode.
+
+ :return: Dataplex Data Profile scan job id.
+ """
+
+ template_fields = ("project_id", "data_scan_id", "impersonation_chain")
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ timeout: float | None = None,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ asynchronous: bool = False,
+ result_timeout: float = 60.0 * 10,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ polling_interval_seconds: int = 10,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.api_version = api_version
+ self.retry = retry
+ self.timeout = timeout
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+ self.asynchronous = asynchronous
+ self.result_timeout = result_timeout
+ self.deferrable = deferrable
+ self.polling_interval_seconds = polling_interval_seconds
+
+ def execute(self, context: Context) -> dict:
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ result = hook.run_data_scan(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ job_id = result.job.name.split("/")[-1]
+
+ if self.deferrable:
+ if self.asynchronous:
+ raise AirflowException(
+ "Both asynchronous and deferrable parameters were passed.
Please, provide only one."
+ )
+ self.defer(
+ trigger=DataplexDataProfileJobTrigger(
+ job_id=job_id,
+ data_scan_id=self.data_scan_id,
+ project_id=self.project_id,
+ region=self.region,
+ gcp_conn_id=self.gcp_conn_id,
+ impersonation_chain=self.impersonation_chain,
+ polling_interval_seconds=self.polling_interval_seconds,
+ ),
+ method_name="execute_complete",
+ )
+ if not self.asynchronous:
+ job = hook.wait_for_data_scan_job(
+ job_id=job_id,
+ data_scan_id=self.data_scan_id,
+ project_id=self.project_id,
+ region=self.region,
+ result_timeout=self.result_timeout,
+ )
+
+ if job.state == DataScanJob.State.FAILED:
+ raise AirflowException(f"Data Profile job failed: {job_id}")
+ if job.state == DataScanJob.State.SUCCEEDED:
+ self.log.info("Data Profile job executed successfully.")
+ else:
+ self.log.info("Data Profile job execution returned status:
%s", job.status)
+
+ return job_id
+
+ def execute_complete(self, context, event=None) -> None:
+ """
+ Callback for when the trigger fires - returns immediately.
+
+ Relies on trigger to throw an exception, otherwise it assumes
execution was
+ successful.
+ """
+ job_state = event["job_state"]
+ job_id = event["job_id"]
+ if job_state == DataScanJob.State.FAILED:
+ raise AirflowException(f"Job failed:\n{job_id}")
+ if job_state == DataScanJob.State.CANCELLED:
+ raise AirflowException(f"Job was cancelled:\n{job_id}")
+ if job_state == DataScanJob.State.SUCCEEDED:
+ self.log.info("Data Profile job executed successfully.")
+ return job_id
+
+
+class DataplexGetDataProfileScanResultOperator(GoogleCloudBaseOperator):
+ """
+ Gets a DataScan Data Profile Job resource.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
lake belongs to.
+ :param region: Required. The ID of the Google Cloud region that the lake
belongs to.
+ :param data_scan_id: Required. Data Profile scan identifier.
+ :param job_id: Optional. Data Profile scan job identifier.
+ :param api_version: The version of the api that will be requested for
example 'v1'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param timeout: The amount of time, in seconds, to wait for the request to
complete. Note that if
+ ``retry`` is specified, the timeout applies to each individual attempt.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+ :param wait_for_results: Flag indicating whether to wait for the result of
a job execution
+ or to return the job in its current state.
+ :param result_timeout: Value in seconds for which operator will wait for
the Data Profile scan result
+ when the flag `wait_for_results = True`.
+ Throws exception if there is no result found after specified amount of
seconds.
+
+ :return: Dict representing DataScanJob.
+ When the job completes with a successful status, information about the
Data Profile result
+ is available.
+ """
+
+ template_fields = ("project_id", "data_scan_id", "impersonation_chain")
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ job_id: str | None = None,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ timeout: float | None = None,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ wait_for_results: bool = True,
+ result_timeout: float = 60.0 * 10,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.job_id = job_id
+ self.api_version = api_version
+ self.retry = retry
+ self.timeout = timeout
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+ self.wait_for_results = wait_for_results
+ self.result_timeout = result_timeout
+
+ def execute(self, context: Context) -> dict:
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+ # fetch the last job
+ if not self.job_id:
+ jobs = hook.list_data_scan_jobs(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ job_ids = [DataScanJob.to_dict(job) for job in jobs]
+ if not job_ids:
+ raise AirflowException("There are no jobs, you should create
one before.")
+ job_id = job_ids[0]["name"]
+ self.job_id = job_id.split("/")[-1]
+
+ if self.wait_for_results:
+ job = hook.wait_for_data_scan_job(
+ job_id=self.job_id,
+ data_scan_id=self.data_scan_id,
+ project_id=self.project_id,
+ region=self.region,
+ result_timeout=self.result_timeout,
+ )
+ else:
+ job = hook.get_data_scan_job(
+ project_id=self.project_id,
+ region=self.region,
+ job_id=self.job_id,
+ data_scan_id=self.data_scan_id,
+ retry=self.retry,
+ timeout=self.timeout,
+ metadata=self.metadata,
+ )
+ if job.state == DataScanJob.State.SUCCEEDED:
+ self.log.info("Data Profile job executed successfully")
+ else:
+ self.log.info("Data Profile job execution returned status: %s",
job.state)
+
+ result = DataScanJob.to_dict(job)
+ result["state"] = DataScanJob.State(result["state"]).name
+
+ return result
+
+ def execute_complete(self, context, event=None) -> None:
+ """
+ Callback for when the trigger fires - returns immediately.
+
+ Relies on trigger to throw an exception, otherwise it assumes
execution was
+ successful.
+ """
+ job_state = event["job_state"]
+ job_id = event["job_id"]
+ job = event["job"]
+ if job_state == DataScanJob.State.FAILED:
+ raise AirflowException(f"Job failed:\n{job_id}")
+ if job_state == DataScanJob.State.CANCELLED:
+ raise AirflowException(f"Job was cancelled:\n{job_id}")
+ if job_state == DataScanJob.State.SUCCEEDED:
+ self.log.info("Data Profile job executed successfully")
+ else:
+ self.log.info("Data Profile job execution returned status: %s",
job_state)
+
+ return job
+
+
class DataplexCreateZoneOperator(GoogleCloudBaseOperator):
"""
Creates a Zone resource within a Lake.
diff --git a/airflow/providers/google/cloud/sensors/dataplex.py
b/airflow/providers/google/cloud/sensors/dataplex.py
index ee0ffc7410..05d25316bb 100644
--- a/airflow/providers/google/cloud/sensors/dataplex.py
+++ b/airflow/providers/google/cloud/sensors/dataplex.py
@@ -259,3 +259,121 @@ class
DataplexDataQualityJobStatusSensor(BaseSensorOperator):
raise AirflowSkipException(message)
raise AirflowDataQualityScanException(message)
return job_status == DataScanJob.State.SUCCEEDED
+
+
+class DataplexDataProfileJobStatusSensor(BaseSensorOperator):
+ """
+ Check the status of the Dataplex DataProfile job.
+
+ :param project_id: Required. The ID of the Google Cloud project that the
task belongs to.
+ :param region: Required. The ID of the Google Cloud region that the task
belongs to.
+ :param data_scan_id: Required. Data Quality scan identifier.
+ :param job_id: Required. Job ID.
+ :param api_version: The version of the api that will be requested for
example 'v3'.
+ :param retry: A retry object used to retry requests. If `None` is
specified, requests
+ will not be retried.
+ :param metadata: Additional metadata that is provided to the method.
+ :param gcp_conn_id: The connection ID to use when fetching connection info.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+ :param result_timeout: Value in seconds for which operator will wait for
the Data Quality scan result.
+ Throws exception if there is no result found after specified amount of
seconds.
+
+ :return: Boolean indicating if the job run has reached the
``DataScanJob.State.SUCCEEDED``.
+ """
+
+ template_fields = ["job_id"]
+
+ def __init__(
+ self,
+ project_id: str,
+ region: str,
+ data_scan_id: str,
+ job_id: str,
+ api_version: str = "v1",
+ retry: Retry | _MethodDefault = DEFAULT,
+ metadata: Sequence[tuple[str, str]] = (),
+ gcp_conn_id: str = "google_cloud_default",
+ impersonation_chain: str | Sequence[str] | None = None,
+ result_timeout: float = 60.0 * 10,
+ start_sensor_time: float | None = None,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.project_id = project_id
+ self.region = region
+ self.data_scan_id = data_scan_id
+ self.job_id = job_id
+ self.api_version = api_version
+ self.retry = retry
+ self.metadata = metadata
+ self.gcp_conn_id = gcp_conn_id
+ self.impersonation_chain = impersonation_chain
+ self.result_timeout = result_timeout
+ self.start_sensor_time = start_sensor_time
+
+ def _duration(self):
+ if not self.start_sensor_time:
+ self.start_sensor_time = time.monotonic()
+ return time.monotonic() - self.start_sensor_time
+
+ def poke(self, context: Context) -> bool:
+ self.log.info("Waiting for job %s to be %s", self.job_id,
DataScanJob.State.SUCCEEDED)
+ if self.result_timeout:
+ duration = self._duration()
+ if duration > self.result_timeout:
+ # TODO: remove this if check when min_airflow_version is set
to higher than 2.7.1
+ message = (
+ f"Timeout: Data Profile scan {self.job_id} is not ready
after {self.result_timeout}s"
+ )
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AirflowDataQualityScanResultTimeoutException(message)
+
+ hook = DataplexHook(
+ gcp_conn_id=self.gcp_conn_id,
+ api_version=self.api_version,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ try:
+ job = hook.get_data_scan_job(
+ project_id=self.project_id,
+ region=self.region,
+ data_scan_id=self.data_scan_id,
+ job_id=self.job_id,
+ timeout=self.timeout,
+ retry=self.retry,
+ metadata=self.metadata,
+ )
+ except GoogleAPICallError as e:
+ # TODO: remove this if check when min_airflow_version is set to
higher than 2.7.1
+ message = f"Error occurred when trying to retrieve Data Profile
scan job: {self.data_scan_id}"
+ if self.soft_fail:
+ raise AirflowSkipException(message, e)
+ raise AirflowException(message, e)
+
+ job_status = job.state
+ self.log.info(
+ "Current status of the Dataplex Data Profile scan job %s => %s",
self.job_id, job_status
+ )
+ if job_status == DataScanJob.State.FAILED:
+ # TODO: remove this if check when min_airflow_version is set to
higher than 2.7.1
+ message = f"Data Profile scan job failed: {self.job_id}"
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AirflowException(message)
+ if job_status == DataScanJob.State.CANCELLED:
+ # TODO: remove this if check when min_airflow_version is set to
higher than 2.7.1
+ message = f"Data Profile scan job cancelled: {self.job_id}"
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AirflowException(message)
+ return job_status == DataScanJob.State.SUCCEEDED
diff --git a/airflow/providers/google/cloud/triggers/dataplex.py
b/airflow/providers/google/cloud/triggers/dataplex.py
index 5f7907cad0..ae03023926 100644
--- a/airflow/providers/google/cloud/triggers/dataplex.py
+++ b/airflow/providers/google/cloud/triggers/dataplex.py
@@ -107,3 +107,85 @@ class DataplexDataQualityJobTrigger(BaseTrigger):
def _convert_to_dict(self, job: DataScanJob) -> dict:
"""Returns a representation of a DataScanJob instance as a dict."""
return DataScanJob.to_dict(job)
+
+
+class DataplexDataProfileJobTrigger(BaseTrigger):
+ """
+ DataplexDataProfileJobTrigger runs on the trigger worker and waits for the
job to be `SUCCEEDED` state.
+
+ :param job_id: Optional. The ID of a Dataplex job.
+ :param data_scan_id: Required. DataScan identifier.
+ :param project_id: Google Cloud Project where the job is running.
+ :param region: The ID of the Google Cloud region that the job belongs to.
+ :param gcp_conn_id: Optional, the connection ID used to connect to Google
Cloud Platform.
+ :param impersonation_chain: Optional service account to impersonate using
short-term
+ credentials, or chained list of accounts required to get the
access_token
+ of the last account in the list, which will be impersonated in the
request.
+ If set as a string, the account must grant the originating account
+ the Service Account Token Creator IAM role.
+ If set as a sequence, the identities from the list must grant
+ Service Account Token Creator IAM role to the directly preceding
identity, with first
+ account from the list granting this role to the originating account
(templated).
+ :param polling_interval_seconds: polling period in seconds to check for
the status.
+ """
+
+ def __init__(
+ self,
+ job_id: str | None,
+ data_scan_id: str,
+ project_id: str | None,
+ region: str,
+ gcp_conn_id: str = "google_cloud_default",
+ polling_interval_seconds: int = 10,
+ impersonation_chain: str | Sequence[str] | None = None,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.job_id = job_id
+ self.data_scan_id = data_scan_id
+ self.project_id = project_id
+ self.region = region
+ self.gcp_conn_id = gcp_conn_id
+ self.polling_interval_seconds = polling_interval_seconds
+ self.impersonation_chain = impersonation_chain
+
+ def serialize(self):
+ return (
+
"airflow.providers.google.cloud.triggers.dataplex.DataplexDataProfileJobTrigger",
+ {
+ "job_id": self.job_id,
+ "data_scan_id": self.data_scan_id,
+ "project_id": self.project_id,
+ "region": self.region,
+ "gcp_conn_id": self.gcp_conn_id,
+ "impersonation_chain": self.impersonation_chain,
+ "polling_interval_seconds": self.polling_interval_seconds,
+ },
+ )
+
+ async def run(self) -> AsyncIterator[TriggerEvent]:
+ hook = DataplexAsyncHook(
+ gcp_conn_id=self.gcp_conn_id,
+ impersonation_chain=self.impersonation_chain,
+ )
+ while True:
+ job = await hook.get_data_scan_job(
+ project_id=self.project_id,
+ region=self.region,
+ job_id=self.job_id,
+ data_scan_id=self.data_scan_id,
+ )
+ state = job.state
+ if state in (DataScanJob.State.FAILED,
DataScanJob.State.SUCCEEDED, DataScanJob.State.CANCELLED):
+ break
+ self.log.info(
+ "Current state is: %s, sleeping for %s seconds.",
+ DataScanJob.State(state).name,
+ self.polling_interval_seconds,
+ )
+ await asyncio.sleep(self.polling_interval_seconds)
+ yield TriggerEvent({"job_id": self.job_id, "job_state": state, "job":
self._convert_to_dict(job)})
+
+ def _convert_to_dict(self, job: DataScanJob) -> dict:
+ """Returns a representation of a DataScanJob instance as a dict."""
+ return DataScanJob.to_dict(job)
diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst
b/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst
index 8c0da7d686..1b8cc56dcf 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/dataplex.rst
@@ -321,3 +321,99 @@ To delete a asset you can use:
:dedent: 4
:start-after: [START howto_dataplex_delete_asset_operator]
:end-before: [END howto_dataplex_delete_asset_operator]
+
+Create or update a Data Profile scan
+------------------------------------
+
+Before you create a Dataplex Data Profile scan you need to define its body.
+For more information about the available fields to pass when creating a Data
Profile scan, visit `Dataplex create data profile API.
<https://cloud.google.com/dataplex/docs/reference/rest/v1/projects.locations.dataScans#DataScan>`__
+
+A simple Data Profile scan configuration can look as followed:
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 0
+ :start-after: [START howto_dataplex_data_profile_configuration]
+ :end-before: [END howto_dataplex_data_profile_configuration]
+
+With this configuration we can create or update the Data Profile scan:
+
+:class:`~airflow.providers.google.cloud.operators.dataplex.DataplexCreateOrUpdateDataProfileScanOperator`
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_create_data_profile_operator]
+ :end-before: [END howto_dataplex_create_data_profile_operator]
+
+Get a Data Profile scan
+-----------------------
+
+To get a Data Profile scan you can use:
+
+:class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanOperator`
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_get_data_profile_operator]
+ :end-before: [END howto_dataplex_get_data_profile_operator]
+
+
+
+Delete a Data Profile scan
+--------------------------
+
+To delete a Data Profile scan you can use:
+
+:class:`~airflow.providers.google.cloud.operators.dataplex.DataplexDeleteDataProfileScanOperator`
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_delete_data_profile_operator]
+ :end-before: [END howto_dataplex_delete_data_profile_operator]
+
+Run a Data Profile scan
+-----------------------
+
+You can run Dataplex Data Profile scan in asynchronous modes to later check
its status using sensor:
+
+:class:`~airflow.providers.google.cloud.operators.dataplex.DataplexRunDataProfileScanOperator`
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_run_data_profile_operator]
+ :end-before: [END howto_dataplex_run_data_profile_operator]
+
+To check that running Dataplex Data Profile scan succeeded you can use:
+
+:class:`~airflow.providers.google.cloud.sensors.dataplex.DataplexDataProfileJobStatusSensor`.
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_data_scan_job_state_sensor]
+ :end-before: [END howto_dataplex_data_scan_job_state_sensor]
+
+Also for this action you can use operator in the deferrable mode:
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_run_data_profile_def_operator]
+ :end-before: [END howto_dataplex_run_data_profile_def_operator]
+
+Get a Data Profile scan job
+---------------------------
+
+To get a Data Profile scan job you can use:
+
+:class:`~airflow.providers.google.cloud.operators.dataplex.DataplexGetDataProfileScanResultOperator`
+
+.. exampleinclude::
/../../tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_dataplex_get_data_profile_job_operator]
+ :end-before: [END howto_dataplex_get_data_profile_job_operator]
diff --git a/tests/providers/google/cloud/operators/test_dataplex.py
b/tests/providers/google/cloud/operators/test_dataplex.py
index 2cddabb4a7..67c9b8ca10 100644
--- a/tests/providers/google/cloud/operators/test_dataplex.py
+++ b/tests/providers/google/cloud/operators/test_dataplex.py
@@ -25,17 +25,21 @@ from airflow.exceptions import TaskDeferred
from airflow.providers.google.cloud.operators.dataplex import (
DataplexCreateAssetOperator,
DataplexCreateLakeOperator,
+ DataplexCreateOrUpdateDataProfileScanOperator,
DataplexCreateOrUpdateDataQualityScanOperator,
DataplexCreateTaskOperator,
DataplexCreateZoneOperator,
DataplexDeleteAssetOperator,
+ DataplexDeleteDataProfileScanOperator,
DataplexDeleteDataQualityScanOperator,
DataplexDeleteLakeOperator,
DataplexDeleteTaskOperator,
DataplexDeleteZoneOperator,
+ DataplexGetDataProfileScanResultOperator,
DataplexGetDataQualityScanResultOperator,
DataplexGetTaskOperator,
DataplexListTasksOperator,
+ DataplexRunDataProfileScanOperator,
DataplexRunDataQualityScanOperator,
)
from airflow.providers.google.cloud.triggers.dataplex import
DataplexDataQualityJobTrigger
@@ -329,6 +333,35 @@ class TestDataplexRunDataQualityScanOperator:
assert exc.value.method_name == GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
+class TestDataplexRunDataProfileScanOperator:
+ @mock.patch(HOOK_STR)
+ @mock.patch(DATASCANJOB_STR)
+ def test_execute(self, mock_data_scan_job, hook_mock):
+ op = DataplexRunDataProfileScanOperator(
+ task_id="execute_data_scan",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+ op.execute(context=mock.MagicMock())
+ hook_mock.assert_called_once_with(
+ gcp_conn_id=GCP_CONN_ID,
+ api_version=API_VERSION,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+ hook_mock.return_value.run_data_scan.assert_called_once_with(
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ retry=DEFAULT,
+ timeout=None,
+ metadata=(),
+ )
+
+
class TestDataplexGetDataQualityScanResultOperator:
@mock.patch(HOOK_STR)
@mock.patch(DATASCANJOB_STR)
@@ -392,6 +425,39 @@ class TestDataplexGetDataQualityScanResultOperator:
assert exc.value.method_name == GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
+class TestDataplexGetDataProfileScanResultOperator:
+ @mock.patch(HOOK_STR)
+ @mock.patch(DATASCANJOB_STR)
+ def test_execute(self, mock_data_scan_job, hook_mock):
+ op = DataplexGetDataProfileScanResultOperator(
+ task_id="get_data_scan_result",
+ project_id=PROJECT_ID,
+ region=REGION,
+ job_id=JOB_ID,
+ data_scan_id=DATA_SCAN_ID,
+ api_version=API_VERSION,
+ wait_for_results=False,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+
+ op.execute(context=mock.MagicMock())
+ hook_mock.assert_called_once_with(
+ gcp_conn_id=GCP_CONN_ID,
+ api_version=API_VERSION,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+ hook_mock.return_value.get_data_scan_job.assert_called_once_with(
+ project_id=PROJECT_ID,
+ region=REGION,
+ job_id=JOB_ID,
+ data_scan_id=DATA_SCAN_ID,
+ retry=DEFAULT,
+ timeout=None,
+ metadata=(),
+ )
+
+
class TestDataplexCreateAssetOperator:
@mock.patch(HOOK_STR)
@mock.patch(ASSET_STR)
@@ -557,6 +623,35 @@ class TestDataplexDeleteDataQualityScanOperator:
)
+class TestDataplexDeleteDataProfileScanOperator:
+ @mock.patch(HOOK_STR)
+ def test_execute(self, hook_mock):
+ op = DataplexDeleteDataProfileScanOperator(
+ task_id=TASK_ID,
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+
+ op.execute(context=mock.MagicMock())
+ hook_mock.assert_called_once_with(
+ gcp_conn_id=GCP_CONN_ID,
+ api_version=API_VERSION,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+ hook_mock.return_value.delete_data_scan.assert_called_once_with(
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ retry=DEFAULT,
+ timeout=None,
+ metadata=(),
+ )
+
+
class TestDataplexCreateDataQualityScanOperator:
@mock.patch(HOOK_STR)
def test_execute(self, hook_mock):
@@ -586,3 +681,34 @@ class TestDataplexCreateDataQualityScanOperator:
timeout=None,
metadata=(),
)
+
+
+class TestDataplexCreateDataProfileScanOperator:
+ @mock.patch(HOOK_STR)
+ def test_execute(self, hook_mock):
+ op = DataplexCreateOrUpdateDataProfileScanOperator(
+ task_id=TASK_ID,
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ body={},
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+
+ op.execute(context=mock.MagicMock())
+ hook_mock.assert_called_once_with(
+ gcp_conn_id=GCP_CONN_ID,
+ api_version=API_VERSION,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ )
+ hook_mock.return_value.create_data_scan.assert_called_once_with(
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ body={},
+ retry=DEFAULT,
+ timeout=None,
+ metadata=(),
+ )
diff --git a/tests/providers/google/cloud/sensors/test_dataplex.py
b/tests/providers/google/cloud/sensors/test_dataplex.py
index 20a4de4ff0..4d5e72da71 100644
--- a/tests/providers/google/cloud/sensors/test_dataplex.py
+++ b/tests/providers/google/cloud/sensors/test_dataplex.py
@@ -25,6 +25,7 @@ from google.cloud.dataplex_v1.types import DataScanJob
from airflow.exceptions import AirflowException, AirflowSkipException
from airflow.providers.google.cloud.hooks.dataplex import
AirflowDataQualityScanResultTimeoutException
from airflow.providers.google.cloud.sensors.dataplex import (
+ DataplexDataProfileJobStatusSensor,
DataplexDataQualityJobStatusSensor,
DataplexTaskStateSensor,
TaskState,
@@ -191,3 +192,82 @@ class TestDataplexDataQualityJobStatusSensor:
with pytest.raises(AirflowDataQualityScanResultTimeoutException):
sensor.poke(context={})
+
+
+class TestDataplexDataProfileJobStatusSensor:
+ def run_job(self, state: int):
+ job = mock.Mock()
+ job.state = state
+ return job
+
+ @mock.patch(DATAPLEX_HOOK)
+ def test_done(self, mock_hook):
+ job = self.run_job(DataScanJob.State.SUCCEEDED)
+ mock_hook.return_value.get_data_scan_job.return_value = job
+
+ sensor = DataplexDataProfileJobStatusSensor(
+ task_id=TASK_ID,
+ project_id=PROJECT_ID,
+ job_id=TEST_JOB_ID,
+ data_scan_id=TEST_DATA_SCAN_ID,
+ region=REGION,
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ timeout=TIMEOUT,
+ )
+ result = sensor.poke(context={})
+
+ mock_hook.return_value.get_data_scan_job.assert_called_once_with(
+ project_id=PROJECT_ID,
+ region=REGION,
+ job_id=TEST_JOB_ID,
+ data_scan_id=TEST_DATA_SCAN_ID,
+ timeout=TIMEOUT,
+ retry=DEFAULT,
+ metadata=(),
+ )
+
+ assert result
+
+ def test_start_sensor_time(self):
+ sensor = DataplexDataProfileJobStatusSensor(
+ task_id=TASK_ID,
+ project_id=PROJECT_ID,
+ job_id=TEST_JOB_ID,
+ data_scan_id=TEST_DATA_SCAN_ID,
+ region=REGION,
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ timeout=TIMEOUT,
+ )
+
+ assert sensor.start_sensor_time is None
+
+ duration_1 = sensor._duration()
+ duration_2 = sensor._duration()
+
+ assert sensor.start_sensor_time
+ assert 0 < duration_1 < duration_2
+
+ @mock.patch.object(DataplexDataProfileJobStatusSensor, "_duration")
+ def test_start_sensor_time_timeout(self, mock_duration):
+ result_timeout = 100
+ mock_duration.return_value = result_timeout + 1
+
+ sensor = DataplexDataProfileJobStatusSensor(
+ task_id=TASK_ID,
+ project_id=PROJECT_ID,
+ job_id=TEST_JOB_ID,
+ data_scan_id=TEST_DATA_SCAN_ID,
+ region=REGION,
+ api_version=API_VERSION,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ timeout=TIMEOUT,
+ result_timeout=result_timeout,
+ )
+
+ with pytest.raises(AirflowDataQualityScanResultTimeoutException):
+ sensor.poke(context={})
diff --git
a/tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
b/tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
new file mode 100644
index 0000000000..aaf6669464
--- /dev/null
+++ b/tests/system/providers/google/cloud/dataplex/example_dataplex_dp.py
@@ -0,0 +1,341 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Example Airflow DAG that shows how to use Dataplex Scan Data.
+"""
+from __future__ import annotations
+
+import os
+from datetime import datetime
+
+from google.cloud import dataplex_v1
+from google.cloud.dataplex_v1 import DataProfileSpec
+from google.protobuf.field_mask_pb2 import FieldMask
+
+from airflow.models.baseoperator import chain
+from airflow.models.dag import DAG
+from airflow.providers.google.cloud.operators.bigquery import (
+ BigQueryCreateEmptyDatasetOperator,
+ BigQueryCreateEmptyTableOperator,
+ BigQueryDeleteDatasetOperator,
+ BigQueryInsertJobOperator,
+)
+from airflow.providers.google.cloud.operators.dataplex import (
+ DataplexCreateAssetOperator,
+ DataplexCreateLakeOperator,
+ DataplexCreateOrUpdateDataProfileScanOperator,
+ DataplexCreateZoneOperator,
+ DataplexDeleteAssetOperator,
+ DataplexDeleteDataProfileScanOperator,
+ DataplexDeleteLakeOperator,
+ DataplexDeleteZoneOperator,
+ DataplexGetDataProfileScanOperator,
+ DataplexGetDataProfileScanResultOperator,
+ DataplexRunDataProfileScanOperator,
+)
+from airflow.providers.google.cloud.sensors.dataplex import
DataplexDataProfileJobStatusSensor
+from airflow.utils.trigger_rule import TriggerRule
+
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
+
+DAG_ID = "example_dataplex_data_profile"
+
+LAKE_ID = f"test-lake-{ENV_ID}"
+REGION = "us-central1"
+
+DATASET_NAME = f"dataset_bq_{ENV_ID}"
+
+TABLE_1 = "table0"
+TABLE_2 = "table1"
+
+SCHEMA = [
+ {"name": "value", "type": "INTEGER", "mode": "REQUIRED"},
+ {"name": "name", "type": "STRING", "mode": "NULLABLE"},
+ {"name": "dt", "type": "STRING", "mode": "NULLABLE"},
+]
+
+DATASET = DATASET_NAME
+INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
+INSERT_ROWS_QUERY = f"INSERT {DATASET}.{TABLE_1} VALUES (1, 'test test2',
'{INSERT_DATE}');"
+LOCATION = "us"
+
+TRIGGER_SPEC_TYPE = "ON_DEMAND"
+
+ZONE_ID = "test-zone-id"
+DATA_SCAN_ID = "test-data-scan-id"
+
+EXAMPLE_LAKE_BODY = {
+ "display_name": "test_display_name",
+ "labels": [],
+ "description": "test_description",
+ "metastore": {"service": ""},
+}
+
+# [START howto_dataplex_zone_configuration]
+EXAMPLE_ZONE = {
+ "type_": "RAW",
+ "resource_spec": {"location_type": "SINGLE_REGION"},
+}
+# [END howto_dataplex_zone_configuration]
+
+ASSET_ID = "test-asset-id"
+
+# [START howto_dataplex_asset_configuration]
+EXAMPLE_ASSET = {
+ "resource_spec": {"name":
f"projects/{PROJECT_ID}/datasets/{DATASET_NAME}", "type_": "BIGQUERY_DATASET"},
+ "discovery_spec": {"enabled": True},
+}
+# [END howto_dataplex_asset_configuration]
+
+# [START howto_dataplex_data_profile_configuration]
+EXAMPLE_DATA_SCAN = dataplex_v1.DataScan()
+EXAMPLE_DATA_SCAN.data.entity = (
+
f"projects/{PROJECT_ID}/locations/{REGION}/lakes/{LAKE_ID}/zones/{ZONE_ID}/entities/{TABLE_1}"
+)
+EXAMPLE_DATA_SCAN.data.resource = (
+
f"//bigquery.googleapis.com/projects/{PROJECT_ID}/datasets/{DATASET}/tables/{TABLE_1}"
+)
+EXAMPLE_DATA_SCAN.data_profile_spec = DataProfileSpec({})
+# [END howto_dataplex_data_profile_configuration]
+UPDATE_MASK = FieldMask(paths=["data_profile_spec"])
+ENTITY =
f"projects/{PROJECT_ID}/locations/{REGION}/lakes/{LAKE_ID}/zones/{ZONE_ID}/entities/{TABLE_1}"
+EXAMPLE_DATA_SCAN_UPDATE = {
+ "data": {
+ "entity": ENTITY,
+ "resource":
f"//bigquery.googleapis.com/projects/{PROJECT_ID}/datasets/{DATASET}/tables/{TABLE_1}",
+ },
+ "data_profile_spec": {},
+}
+
+
+with DAG(
+ DAG_ID,
+ start_date=datetime(2021, 1, 1),
+ schedule="@once",
+ tags=["example", "dataplex", "data_profile"],
+) as dag:
+ create_dataset =
BigQueryCreateEmptyDatasetOperator(task_id="create_dataset",
dataset_id=DATASET_NAME)
+ create_table_1 = BigQueryCreateEmptyTableOperator(
+ task_id="create_table_1",
+ dataset_id=DATASET_NAME,
+ table_id=TABLE_1,
+ schema_fields=SCHEMA,
+ location=LOCATION,
+ )
+ create_table_2 = BigQueryCreateEmptyTableOperator(
+ task_id="create_table_2",
+ dataset_id=DATASET_NAME,
+ table_id=TABLE_2,
+ schema_fields=SCHEMA,
+ location=LOCATION,
+ )
+ insert_query_job = BigQueryInsertJobOperator(
+ task_id="insert_query_job",
+ configuration={
+ "query": {
+ "query": INSERT_ROWS_QUERY,
+ "useLegacySql": False,
+ }
+ },
+ )
+ create_lake = DataplexCreateLakeOperator(
+ task_id="create_lake", project_id=PROJECT_ID, region=REGION,
body=EXAMPLE_LAKE_BODY, lake_id=LAKE_ID
+ )
+ # [START howto_dataplex_create_zone_operator]
+ create_zone = DataplexCreateZoneOperator(
+ task_id="create_zone",
+ project_id=PROJECT_ID,
+ region=REGION,
+ lake_id=LAKE_ID,
+ body=EXAMPLE_ZONE,
+ zone_id=ZONE_ID,
+ )
+ # [END howto_dataplex_create_zone_operator]
+ # [START howto_dataplex_create_asset_operator]
+ create_asset = DataplexCreateAssetOperator(
+ task_id="create_asset",
+ project_id=PROJECT_ID,
+ region=REGION,
+ body=EXAMPLE_ASSET,
+ lake_id=LAKE_ID,
+ zone_id=ZONE_ID,
+ asset_id=ASSET_ID,
+ )
+ # [END howto_dataplex_create_asset_operator]
+ # [START howto_dataplex_create_data_profile_operator]
+ create_data_scan = DataplexCreateOrUpdateDataProfileScanOperator(
+ task_id="create_data_scan",
+ project_id=PROJECT_ID,
+ region=REGION,
+ body=EXAMPLE_DATA_SCAN,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [END howto_dataplex_create_data_profile_operator]
+ update_data_scan = DataplexCreateOrUpdateDataProfileScanOperator(
+ task_id="update_data_scan",
+ project_id=PROJECT_ID,
+ region=REGION,
+ update_mask=UPDATE_MASK,
+ body=EXAMPLE_DATA_SCAN_UPDATE,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [START howto_dataplex_get_data_profile_operator]
+ get_data_scan = DataplexGetDataProfileScanOperator(
+ task_id="get_data_scan",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [END howto_dataplex_get_data_profile_operator]
+ run_data_scan_sync = DataplexRunDataProfileScanOperator(
+ task_id="run_data_scan_sync",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ get_data_scan_job_result = DataplexGetDataProfileScanResultOperator(
+ task_id="get_data_scan_job_result",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [START howto_dataplex_run_data_profile_operator]
+ run_data_scan_async = DataplexRunDataProfileScanOperator(
+ task_id="run_data_scan_async",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ asynchronous=True,
+ )
+ # [END howto_dataplex_run_data_profile_operator]
+ # [START howto_dataplex_data_scan_job_state_sensor]
+ get_data_scan_job_status = DataplexDataProfileJobStatusSensor(
+ task_id="get_data_scan_job_status",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ job_id="{{ task_instance.xcom_pull('run_data_scan_async') }}",
+ )
+ # [END howto_dataplex_data_scan_job_state_sensor]
+ # [START howto_dataplex_get_data_profile_job_operator]
+ get_data_scan_job_result_2 = DataplexGetDataProfileScanResultOperator(
+ task_id="get_data_scan_job_result_2",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [END howto_dataplex_get_data_profile_job_operator]
+ # [START howto_dataplex_run_data_profile_def_operator]
+ run_data_scan_def = DataplexRunDataProfileScanOperator(
+ task_id="run_data_scan_def",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ deferrable=True,
+ )
+ # [END howto_dataplex_run_data_profile_def_operator]
+ run_data_scan_async_2 = DataplexRunDataProfileScanOperator(
+ task_id="run_data_scan_async_2",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ asynchronous=True,
+ )
+ # [START howto_dataplex_delete_asset_operator]
+ delete_asset = DataplexDeleteAssetOperator(
+ task_id="delete_asset",
+ project_id=PROJECT_ID,
+ region=REGION,
+ lake_id=LAKE_ID,
+ zone_id=ZONE_ID,
+ asset_id=ASSET_ID,
+ )
+ # [END howto_dataplex_delete_asset_operator]
+ delete_asset.trigger_rule = TriggerRule.ALL_DONE
+ # [START howto_dataplex_delete_zone_operator]
+ delete_zone = DataplexDeleteZoneOperator(
+ task_id="delete_zone",
+ project_id=PROJECT_ID,
+ region=REGION,
+ lake_id=LAKE_ID,
+ zone_id=ZONE_ID,
+ )
+ # [END howto_dataplex_delete_zone_operator]
+ delete_zone.trigger_rule = TriggerRule.ALL_DONE
+ # [START howto_dataplex_delete_data_profile_operator]
+ delete_data_scan = DataplexDeleteDataProfileScanOperator(
+ task_id="delete_data_scan",
+ project_id=PROJECT_ID,
+ region=REGION,
+ data_scan_id=DATA_SCAN_ID,
+ )
+ # [END howto_dataplex_delete_data_profile_operator]
+ delete_data_scan.trigger_rule = TriggerRule.ALL_DONE
+ delete_lake = DataplexDeleteLakeOperator(
+ project_id=PROJECT_ID,
+ region=REGION,
+ lake_id=LAKE_ID,
+ task_id="delete_lake",
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+ delete_dataset = BigQueryDeleteDatasetOperator(
+ task_id="delete_dataset",
+ dataset_id=DATASET_NAME,
+ project_id=PROJECT_ID,
+ delete_contents=True,
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+
+ chain(
+ # TEST SETUP
+ create_dataset,
+ [create_table_1, create_table_2],
+ insert_query_job,
+ create_lake,
+ create_zone,
+ create_asset,
+ # TEST BODY
+ create_data_scan,
+ update_data_scan,
+ get_data_scan,
+ run_data_scan_sync,
+ get_data_scan_job_result,
+ run_data_scan_async,
+ get_data_scan_job_status,
+ get_data_scan_job_result_2,
+ run_data_scan_def,
+ run_data_scan_async_2,
+ # get_data_scan_job_result_def,
+ # TEST TEARDOWN
+ delete_asset,
+ delete_zone,
+ delete_data_scan,
+ [delete_lake, delete_dataset],
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see:
tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)