This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57c09e59ee Display Video 360 cleanup v1 API usage (#30577)
57c09e59ee is described below
commit 57c09e59ee9273ff64cd4a85b020a4df9b1d9eca
Author: Łukasz Wyszomirski <[email protected]>
AuthorDate: Fri Apr 14 13:10:49 2023 +0200
Display Video 360 cleanup v1 API usage (#30577)
* Display Video 360 cleanup v1 API usage
* Update docs
---
.../example_dags/example_display_video.py | 64 -----
.../marketing_platform/hooks/display_video.py | 45 +---
.../marketing_platform/operators/display_video.py | 284 ---------------------
.../marketing_platform/sensors/display_video.py | 65 -----
.../operators/marketing_platform/display_video.rst | 86 +------
scripts/in_container/verify_providers.py | 4 -
.../marketing_platform/hooks/test_display_video.py | 24 +-
.../operators/test_display_video.py | 192 --------------
.../sensors/test_display_video.py | 19 --
9 files changed, 25 insertions(+), 758 deletions(-)
diff --git
a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py
b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py
index 7e6c3c38ac..9caf6eb509 100644
---
a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py
+++
b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py
@@ -30,20 +30,16 @@ from
airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQue
from airflow.providers.google.marketing_platform.hooks.display_video import
GoogleDisplayVideo360Hook
from airflow.providers.google.marketing_platform.operators.display_video
import (
GoogleDisplayVideo360CreateQueryOperator,
- GoogleDisplayVideo360CreateReportOperator,
GoogleDisplayVideo360CreateSDFDownloadTaskOperator,
GoogleDisplayVideo360DeleteReportOperator,
GoogleDisplayVideo360DownloadLineItemsOperator,
- GoogleDisplayVideo360DownloadReportOperator,
GoogleDisplayVideo360DownloadReportV2Operator,
GoogleDisplayVideo360RunQueryOperator,
- GoogleDisplayVideo360RunReportOperator,
GoogleDisplayVideo360SDFtoGCSOperator,
GoogleDisplayVideo360UploadLineItemsOperator,
)
from airflow.providers.google.marketing_platform.sensors.display_video import (
GoogleDisplayVideo360GetSDFDownloadOperationSensor,
- GoogleDisplayVideo360ReportSensor,
GoogleDisplayVideo360RunQuerySensor,
)
@@ -60,24 +56,6 @@ GMP_PARTNER_ID = os.environ.get("GMP_PARTNER_ID", 123)
ENTITY_TYPE = os.environ.get("GMP_ENTITY_TYPE", "LineItem")
ERF_SOURCE_OBJECT = GoogleDisplayVideo360Hook.erf_uri(GMP_PARTNER_ID,
ENTITY_TYPE)
-REPORT = {
- "kind": "doubleclickbidmanager#query",
- "metadata": {
- "title": "Polidea Test Report",
- "dataRange": "LAST_7_DAYS",
- "format": "CSV",
- "sendNotification": False,
- },
- "params": {
- "type": "TYPE_GENERAL",
- "groupBys": ["FILTER_DATE", "FILTER_PARTNER"],
- "filters": [{"type": "FILTER_PARTNER", "value": 1486931}],
- "metrics": ["METRIC_IMPRESSIONS", "METRIC_CLICKS"],
- "includeInviteData": True,
- },
- "schedule": {"frequency": "ONE_TIME"},
-}
-
REPORT_V2 = {
"metadata": {
"title": "Airflow Test Report",
@@ -109,48 +87,6 @@ DOWNLOAD_LINE_ITEMS_REQUEST: dict = {"filterType":
ADVERTISER_ID, "format": "CSV
START_DATE = datetime(2021, 1, 1)
-with models.DAG(
- "example_display_video",
- start_date=START_DATE,
- catchup=False,
-) as dag1:
- # [START howto_google_display_video_createquery_report_operator]
- create_report = GoogleDisplayVideo360CreateReportOperator(body=REPORT,
task_id="create_report")
- report_id = cast(str, XComArg(create_report, key="report_id"))
- # [END howto_google_display_video_createquery_report_operator]
-
- # [START howto_google_display_video_runquery_report_operator]
- run_report = GoogleDisplayVideo360RunReportOperator(
- report_id=report_id, parameters=PARAMETERS, task_id="run_report"
- )
- # [END howto_google_display_video_runquery_report_operator]
-
- # [START howto_google_display_video_wait_report_operator]
- wait_for_report =
GoogleDisplayVideo360ReportSensor(task_id="wait_for_report",
report_id=report_id)
- # [END howto_google_display_video_wait_report_operator]
-
- # [START howto_google_display_video_getquery_report_operator]
- get_report = GoogleDisplayVideo360DownloadReportOperator(
- report_id=report_id,
- task_id="get_report",
- bucket_name=BUCKET,
- report_name="test1.csv",
- )
- # [END howto_google_display_video_getquery_report_operator]
-
- # [START howto_google_display_video_deletequery_report_operator]
- delete_report =
GoogleDisplayVideo360DeleteReportOperator(report_id=report_id,
task_id="delete_report")
- # [END howto_google_display_video_deletequery_report_operator]
-
- run_report >> wait_for_report >> get_report >> delete_report
-
- # Task dependencies created via `XComArgs`:
- # create_report >> run_report
- # create_report >> wait_for_report
- # create_report >> get_report
- # create_report >> delete_report
-
-
with models.DAG(
"example_display_video_misc",
start_date=START_DATE,
diff --git a/airflow/providers/google/marketing_platform/hooks/display_video.py
b/airflow/providers/google/marketing_platform/hooks/display_video.py
index 9422426fbe..d7927f709e 100644
--- a/airflow/providers/google/marketing_platform/hooks/display_video.py
+++ b/airflow/providers/google/marketing_platform/hooks/display_video.py
@@ -18,7 +18,6 @@
"""This module contains Google DisplayVideo hook."""
from __future__ import annotations
-import warnings
from typing import Any, Sequence
from googleapiclient.discovery import Resource, build
@@ -43,11 +42,6 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
delegate_to=delegate_to,
impersonation_chain=impersonation_chain,
)
- if api_version in ["v1", "v1.1"]:
- warnings.warn(
- f"API {api_version} is deprecated and shortly will be removed
please use v2",
- DeprecationWarning,
- )
self.api_version = api_version
def get_conn(self) -> Resource:
@@ -99,10 +93,7 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
:param query: Query object to be passed to request body.
"""
- if self.api_version in ["v1", "v1.1"]:
- response =
self.get_conn().queries().createquery(body=query).execute(num_retries=self.num_retries)
- else:
- response =
self.get_conn().queries().create(body=query).execute(num_retries=self.num_retries)
+ response =
self.get_conn().queries().create(body=query).execute(num_retries=self.num_retries)
return response
def delete_query(self, query_id: str) -> None:
@@ -111,10 +102,7 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
:param query_id: Query ID to delete.
"""
- if self.api_version in ["v1", "v1.1"]:
-
self.get_conn().queries().deletequery(queryId=query_id).execute(num_retries=self.num_retries)
- else:
-
self.get_conn().queries().delete(queryId=query_id).execute(num_retries=self.num_retries)
+
self.get_conn().queries().delete(queryId=query_id).execute(num_retries=self.num_retries)
def get_query(self, query_id: str) -> dict:
"""
@@ -122,20 +110,12 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
:param query_id: Query ID to retrieve.
"""
- if self.api_version in ["v1", "v1.1"]:
- response = (
-
self.get_conn().queries().getquery(queryId=query_id).execute(num_retries=self.num_retries)
- )
- else:
- response =
self.get_conn().queries().get(queryId=query_id).execute(num_retries=self.num_retries)
+ response =
self.get_conn().queries().get(queryId=query_id).execute(num_retries=self.num_retries)
return response
def list_queries(self) -> list[dict]:
"""Retrieves stored queries."""
- if self.api_version in ["v1", "v1.1"]:
- response =
self.get_conn().queries().listqueries().execute(num_retries=self.num_retries)
- else:
- response =
self.get_conn().queries().list().execute(num_retries=self.num_retries)
+ response =
self.get_conn().queries().list().execute(num_retries=self.num_retries)
return response.get("queries", [])
def run_query(self, query_id: str, params: dict[str, Any] | None) -> dict:
@@ -145,20 +125,9 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
:param query_id: Query ID to run.
:param params: Parameters for the report.
"""
- if self.api_version in ["v1", "v1.1"]:
- return (
- self.get_conn()
- .queries()
- .runquery(queryId=query_id, body=params)
- .execute(num_retries=self.num_retries)
- )
- else:
- return (
- self.get_conn()
- .queries()
- .run(queryId=query_id, body=params)
- .execute(num_retries=self.num_retries)
- )
+ return (
+ self.get_conn().queries().run(queryId=query_id,
body=params).execute(num_retries=self.num_retries)
+ )
def get_report(self, query_id: str, report_id: str) -> dict:
"""
diff --git
a/airflow/providers/google/marketing_platform/operators/display_video.py
b/airflow/providers/google/marketing_platform/operators/display_video.py
index 738d734bad..003ad4723c 100644
--- a/airflow/providers/google/marketing_platform/operators/display_video.py
+++ b/airflow/providers/google/marketing_platform/operators/display_video.py
@@ -23,7 +23,6 @@ import json
import shutil
import tempfile
import urllib.request
-import warnings
from typing import TYPE_CHECKING, Any, Sequence
from urllib.parse import urlsplit
@@ -36,84 +35,6 @@ if TYPE_CHECKING:
from airflow.utils.context import Context
-class GoogleDisplayVideo360CreateReportOperator(BaseOperator):
- """
- Creates a query.
-
- .. seealso::
- For more information on how to use this operator, take a look at the
guide:
- :ref:`howto/operator:GoogleDisplayVideo360CreateReportOperator`
-
- .. seealso::
- Check also the official API docs:
- `https://developers.google.com/bid-manager/v1/queries/createquery`
-
- :param body: Report object passed to the request's body as described here:
- https://developers.google.com/bid-manager/v1/queries#resource
- :param api_version: The version of the api that will be requested for
example 'v3'.
- :param gcp_conn_id: The connection ID to use when fetching connection info.
- :param delegate_to: The account to impersonate using domain-wide
delegation of authority,
- if any. For this to work, the service account making the request must
have
- domain-wide delegation enabled.
- :param impersonation_chain: Optional service account to impersonate using
short-term
- credentials, or chained list of accounts required to get the
access_token
- of the last account in the list, which will be impersonated in the
request.
- If set as a string, the account must grant the originating account
- the Service Account Token Creator IAM role.
- If set as a sequence, the identities from the list must grant
- Service Account Token Creator IAM role to the directly preceding
identity, with first
- account from the list granting this role to the originating account
(templated).
- """
-
- template_fields: Sequence[str] = (
- "body",
- "impersonation_chain",
- )
- template_ext: Sequence[str] = (".json",)
-
- def __init__(
- self,
- *,
- body: dict[str, Any],
- api_version: str = "v1",
- gcp_conn_id: str = "google_cloud_default",
- delegate_to: str | None = None,
- impersonation_chain: str | Sequence[str] | None = None,
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.body = body
-
- warnings.warn(
- "This operator is deprecated. Please use
`GoogleDisplayVideo360CreateQueryOperator`",
- DeprecationWarning,
- )
- self.api_version = api_version
- self.gcp_conn_id = gcp_conn_id
- self.delegate_to = delegate_to
- self.impersonation_chain = impersonation_chain
-
- def prepare_template(self) -> None:
- # If .json is passed then we have to read the file
- if isinstance(self.body, str) and self.body.endswith(".json"):
- with open(self.body) as file:
- self.body = json.load(file)
-
- def execute(self, context: Context) -> dict:
- hook = GoogleDisplayVideo360Hook(
- gcp_conn_id=self.gcp_conn_id,
- delegate_to=self.delegate_to,
- api_version=self.api_version,
- impersonation_chain=self.impersonation_chain,
- )
- self.log.info("Creating Display & Video 360 report.")
- response = hook.create_query(query=self.body)
- report_id = response["queryId"]
- self.xcom_push(context, key="report_id", value=report_id)
- self.log.info("Created report with ID: %s", report_id)
- return response
-
-
class GoogleDisplayVideo360CreateQueryOperator(BaseOperator):
"""
Creates a query.
@@ -235,11 +156,6 @@ class
GoogleDisplayVideo360DeleteReportOperator(BaseOperator):
super().__init__(**kwargs)
self.report_id = report_id
self.report_name = report_name
- if api_version in ["v1", "v1.1"]:
- warnings.warn(
- f"API {api_version} is deprecated and shortly will be removed
please use v2",
- DeprecationWarning,
- )
self.api_version = api_version
self.gcp_conn_id = gcp_conn_id
self.delegate_to = delegate_to
@@ -272,132 +188,6 @@ class
GoogleDisplayVideo360DeleteReportOperator(BaseOperator):
self.log.info("Report deleted.")
-class GoogleDisplayVideo360DownloadReportOperator(BaseOperator):
- """
- Retrieves a stored query.
-
- .. seealso::
- For more information on how to use this operator, take a look at the
guide:
- :ref:`howto/operator:GoogleDisplayVideo360DownloadReportOperator`
-
- .. seealso::
- Check also the official API docs:
- `https://developers.google.com/bid-manager/v1/queries/getquery`
-
- :param report_id: Report ID to retrieve.
- :param bucket_name: The bucket to upload to.
- :param report_name: The report name to set when uploading the local file.
- :param chunk_size: File will be downloaded in chunks of this many bytes.
- :param gzip: Option to compress local file or file data for upload
- :param api_version: The version of the api that will be requested for
example 'v3'.
- :param gcp_conn_id: The connection ID to use when fetching connection info.
- :param delegate_to: The account to impersonate using domain-wide
delegation of authority,
- if any. For this to work, the service account making the request must
have
- domain-wide delegation enabled.
- :param impersonation_chain: Optional service account to impersonate using
short-term
- credentials, or chained list of accounts required to get the
access_token
- of the last account in the list, which will be impersonated in the
request.
- If set as a string, the account must grant the originating account
- the Service Account Token Creator IAM role.
- If set as a sequence, the identities from the list must grant
- Service Account Token Creator IAM role to the directly preceding
identity, with first
- account from the list granting this role to the originating account
(templated).
- """
-
- template_fields: Sequence[str] = (
- "report_id",
- "bucket_name",
- "report_name",
- "impersonation_chain",
- )
-
- def __init__(
- self,
- *,
- report_id: str,
- bucket_name: str,
- report_name: str | None = None,
- gzip: bool = True,
- chunk_size: int = 10 * 1024 * 1024,
- api_version: str = "v1",
- gcp_conn_id: str = "google_cloud_default",
- delegate_to: str | None = None,
- impersonation_chain: str | Sequence[str] | None = None,
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- warnings.warn(
- "This operator is deprecated. Please use
`GoogleDisplayVideo360DownloadReportV2Operator`",
- DeprecationWarning,
- )
- self.report_id = report_id
- self.chunk_size = chunk_size
- self.gzip = gzip
- self.bucket_name = bucket_name
- self.report_name = report_name
- self.api_version = api_version
- self.gcp_conn_id = gcp_conn_id
- self.delegate_to = delegate_to
- self.impersonation_chain = impersonation_chain
-
- def _resolve_file_name(self, name: str) -> str:
- new_name = name if name.endswith(".csv") else f"{name}.csv"
- new_name = f"{new_name}.gz" if self.gzip else new_name
- return new_name
-
- @staticmethod
- def _set_bucket_name(name: str) -> str:
- bucket = name if not name.startswith("gs://") else name[5:]
- return bucket.strip("/")
-
- def execute(self, context: Context):
- hook = GoogleDisplayVideo360Hook(
- gcp_conn_id=self.gcp_conn_id,
- delegate_to=self.delegate_to,
- api_version=self.api_version,
- impersonation_chain=self.impersonation_chain,
- )
- gcs_hook = GCSHook(
- gcp_conn_id=self.gcp_conn_id,
- delegate_to=self.delegate_to,
- impersonation_chain=self.impersonation_chain,
- )
-
- resource = hook.get_query(query_id=self.report_id)
- # Check if report is ready
- if resource["metadata"]["running"]:
- raise AirflowException(f"Report {self.report_id} is still running")
-
- # If no custom report_name provided, use DV360 name
- file_url =
resource["metadata"]["googleCloudStoragePathForLatestReport"]
- report_name = self.report_name or
urlsplit(file_url).path.split("/")[-1]
- report_name = self._resolve_file_name(report_name)
-
- # Download the report
- self.log.info("Starting downloading report %s", self.report_id)
- with tempfile.NamedTemporaryFile(delete=False) as temp_file:
- with urllib.request.urlopen(file_url) as response:
- shutil.copyfileobj(response, temp_file, length=self.chunk_size)
-
- temp_file.flush()
- # Upload the local file to bucket
- bucket_name = self._set_bucket_name(self.bucket_name)
- gcs_hook.upload(
- bucket_name=bucket_name,
- object_name=report_name,
- gzip=self.gzip,
- filename=temp_file.name,
- mime_type="text/csv",
- )
- self.log.info(
- "Report %s was saved in bucket %s as %s.",
- self.report_id,
- self.bucket_name,
- report_name,
- )
- self.xcom_push(context, key="report_name", value=report_name)
-
-
class GoogleDisplayVideo360DownloadReportV2Operator(BaseOperator):
"""
Retrieves a stored query.
@@ -523,80 +313,6 @@ class
GoogleDisplayVideo360DownloadReportV2Operator(BaseOperator):
self.xcom_push(context, key="report_name", value=report_name)
-class GoogleDisplayVideo360RunReportOperator(BaseOperator):
- """
- Runs a stored query to generate a report.
-
- .. seealso::
- For more information on how to use this operator, take a look at the
guide:
- :ref:`howto/operator:GoogleDisplayVideo360RunReportOperator`
-
- .. seealso::
- Check also the official API docs:
- `https://developers.google.com/bid-manager/v1/queries/runquery`
-
- :param report_id: Report ID to run.
- :param parameters: Parameters for running a report as described here:
- https://developers.google.com/bid-manager/v1/queries/runquery
- :param api_version: The version of the api that will be requested for
example 'v3'.
- :param gcp_conn_id: The connection ID to use when fetching connection info.
- :param delegate_to: The account to impersonate using domain-wide
delegation of authority,
- if any. For this to work, the service account making the request must
have
- domain-wide delegation enabled.
- :param impersonation_chain: Optional service account to impersonate using
short-term
- credentials, or chained list of accounts required to get the
access_token
- of the last account in the list, which will be impersonated in the
request.
- If set as a string, the account must grant the originating account
- the Service Account Token Creator IAM role.
- If set as a sequence, the identities from the list must grant
- Service Account Token Creator IAM role to the directly preceding
identity, with first
- account from the list granting this role to the originating account
(templated).
- """
-
- template_fields: Sequence[str] = (
- "report_id",
- "parameters",
- "impersonation_chain",
- )
-
- def __init__(
- self,
- *,
- report_id: str,
- parameters: dict[str, Any] | None = None,
- api_version: str = "v1",
- gcp_conn_id: str = "google_cloud_default",
- delegate_to: str | None = None,
- impersonation_chain: str | Sequence[str] | None = None,
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.report_id = report_id
- warnings.warn(
- "This operator is deprecated. Please use
`GoogleDisplayVideo360RunQueryOperator`",
- DeprecationWarning,
- )
- self.api_version = api_version
- self.gcp_conn_id = gcp_conn_id
- self.delegate_to = delegate_to
- self.parameters = parameters
- self.impersonation_chain = impersonation_chain
-
- def execute(self, context: Context) -> None:
- hook = GoogleDisplayVideo360Hook(
- gcp_conn_id=self.gcp_conn_id,
- delegate_to=self.delegate_to,
- api_version=self.api_version,
- impersonation_chain=self.impersonation_chain,
- )
- self.log.info(
- "Running report %s with the following parameters:\n %s",
- self.report_id,
- self.parameters,
- )
- hook.run_query(query_id=self.report_id, params=self.parameters)
-
-
class GoogleDisplayVideo360RunQueryOperator(BaseOperator):
"""
Runs a stored query to generate a report.
diff --git
a/airflow/providers/google/marketing_platform/sensors/display_video.py
b/airflow/providers/google/marketing_platform/sensors/display_video.py
index d4d10ba955..bcbb407af8 100644
--- a/airflow/providers/google/marketing_platform/sensors/display_video.py
+++ b/airflow/providers/google/marketing_platform/sensors/display_video.py
@@ -17,7 +17,6 @@
"""Sensor for detecting the completion of DV360 reports."""
from __future__ import annotations
-import warnings
from typing import TYPE_CHECKING, Sequence
from airflow import AirflowException
@@ -28,70 +27,6 @@ if TYPE_CHECKING:
from airflow.utils.context import Context
-class GoogleDisplayVideo360ReportSensor(BaseSensorOperator):
- """
- Sensor for detecting the completion of DV360 reports.
-
- .. seealso::
- For more information on how to use this operator, take a look at the
guide:
- :ref:`howto/operator:GoogleDisplayVideo360ReportSensor`
-
- :param report_id: Report ID to delete.
- :param api_version: The version of the api that will be requested for
example 'v3'.
- :param gcp_conn_id: The connection ID to use when fetching connection info.
- :param delegate_to: The account to impersonate using domain-wide
delegation of authority,
- if any. For this to work, the service account making the request must
have
- domain-wide delegation enabled.
- :param impersonation_chain: Optional service account to impersonate using
short-term
- credentials, or chained list of accounts required to get the
access_token
- of the last account in the list, which will be impersonated in the
request.
- If set as a string, the account must grant the originating account
- the Service Account Token Creator IAM role.
- If set as a sequence, the identities from the list must grant
- Service Account Token Creator IAM role to the directly preceding
identity, with first
- account from the list granting this role to the originating account
(templated).
- """
-
- template_fields: Sequence[str] = (
- "report_id",
- "impersonation_chain",
- )
-
- def __init__(
- self,
- *,
- report_id: str,
- api_version: str = "v1",
- gcp_conn_id: str = "google_cloud_default",
- delegate_to: str | None = None,
- impersonation_chain: str | Sequence[str] | None = None,
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- warnings.warn(
- "This operator is deprecated. Please use
`GoogleDisplayVideo360RunQuerySensor`",
- DeprecationWarning,
- )
- self.report_id = report_id
- self.api_version = api_version
- self.gcp_conn_id = gcp_conn_id
- self.delegate_to = delegate_to
- self.impersonation_chain = impersonation_chain
-
- def poke(self, context: Context) -> bool:
- hook = GoogleDisplayVideo360Hook(
- gcp_conn_id=self.gcp_conn_id,
- delegate_to=self.delegate_to,
- api_version=self.api_version,
- impersonation_chain=self.impersonation_chain,
- )
-
- response = hook.get_query(query_id=self.report_id)
- if response and not response.get("metadata", {}).get("running"):
- return True
- return False
-
-
class GoogleDisplayVideo360GetSDFDownloadOperationSensor(BaseSensorOperator):
"""
Sensor for detecting the completion of SDF operation.
diff --git
a/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst
b/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst
index c0e225f86e..8a834c885b 100644
---
a/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst
+++
b/docs/apache-airflow-providers-google/operators/marketing_platform/display_video.rst
@@ -27,29 +27,6 @@ Prerequisite Tasks
.. _howto/operator:GoogleDisplayVideo360CreateReportOperator:
-Creating a report
-^^^^^^^^^^^^^^^^^
-
-This Operator is Deprecated and will be removed soon. Please look at
``GoogleDisplayVideo360CreateQueryOperator``.
-
-To create Display&Video 360 report use
-:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateReportOperator`.
-
-.. exampleinclude::
/../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
- :language: python
- :dedent: 4
- :start-after: [START
howto_google_display_video_createquery_report_operator]
- :end-before: [END howto_google_display_video_createquery_report_operator]
-
-Use :ref:`Jinja templating <concepts:jinja-templating>` with
-:template-fields:`airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360CreateReportOperator`
-parameters which allow you to dynamically determine values. You can provide
body definition using ``
-.json`` file as this operator supports this template extension.
-The result is saved to :ref:`XCom <concepts:xcom>`, which allows the result to
be used by other operators.
-
-.. _howto/operator:GoogleDisplayVideo360DeleteReportOperator:
-
-
Creating a Query
^^^^^^^^^^^^^^^^
@@ -86,7 +63,7 @@ You can use :ref:`Jinja templating
<concepts:jinja-templating>` with
parameters which allow you to dynamically determine values.
The result is saved to :ref:`XCom <concepts:xcom>`, which allows the result to
be used by other operators.
-.. _howto/operator:GoogleDisplayVideo360RunQuerySensor:
+.. _howto/operator:GoogleDisplayVideo360DeleteReportOperator:
Deleting a report
^^^^^^^^^^^^^^^^^
@@ -97,35 +74,14 @@ To delete Display&Video 360 report use
.. exampleinclude::
/../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
- :start-after: [START
howto_google_display_video_deletequery_report_operator]
- :end-before: [END howto_google_display_video_deletequery_report_operator]
+ :start-after: [START
howto_google_display_video_delete_query_report_operator]
+ :end-before: [END howto_google_display_video_delete_query_report_operator]
You can use :ref:`Jinja templating <concepts:jinja-templating>` with
:template-fields:`airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DeleteReportOperator`
parameters which allow you to dynamically determine values.
-.. _howto/operator:GoogleDisplayVideo360ReportSensor:
-
-Waiting for report
-^^^^^^^^^^^^^^^^^^
-
-This Operator is Deprecated and will be removed soon. Please look:
-.. _howto/operators:GoogleDisplayVideo360RunQuerySensor
-
-To wait for the report use
-:class:`~airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360ReportSensor`.
-
-.. exampleinclude::
/../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
- :language: python
- :dedent: 4
- :start-after: [START howto_google_display_video_wait_report_operator]
- :end-before: [END howto_google_display_video_wait_report_operator]
-
-Use :ref:`Jinja templating <concepts:jinja-templating>` with
-:template-fields:`airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360ReportSensor`
-parameters which allow you to dynamically determine values.
-
-.. _howto/operator:GoogleDisplayVideo360DownloadReportOperator:
+.. _howto/operator:GoogleDisplayVideo360RunQuerySensor:
Waiting for query
^^^^^^^^^^^^^^^^^
@@ -148,45 +104,19 @@ parameters which allow you to dynamically determine
values.
Downloading a report
^^^^^^^^^^^^^^^^^^^^
-This Operator is Deprecated and will be removed soon. Please look:
-.. _howto/operators:GoogleDisplayVideo360DownloadReportV2Operator
-
To download a report to GCS bucket use
-:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportOperator`.
+:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportV2Operator`.
.. exampleinclude::
/../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
:language: python
:dedent: 4
- :start-after: [START howto_google_display_video_getquery_report_operator]
- :end-before: [END howto_google_display_video_getquery_report_operator]
+ :start-after: [START howto_google_display_video_get_report_operator]
+ :end-before: [END howto_google_display_video_get_report_operator]
Use :ref:`Jinja templating <concepts:jinja-templating>` with
-:template-fields:`airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportOperator`
+:template-fields:`airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360DownloadReportV2Operator`
parameters which allow you to dynamically determine values.
-
-.. _howto/operator:GoogleDisplayVideo360RunReportOperator:
-
-Running a report
-^^^^^^^^^^^^^^^^
-
-This Operator is Deprecated and will be removed soon. Please look:
-.. _howto/operators:GoogleDisplayVideo360RunQueryOperator
-
-To run Display&Video 360 report use
-:class:`~airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360RunReportOperator`.
-
-.. exampleinclude::
/../../airflow/providers/google/marketing_platform/example_dags/example_display_video.py
- :language: python
- :dedent: 4
- :start-after: [START howto_google_display_video_runquery_report_operator]
- :end-before: [END howto_google_display_video_runquery_report_operator]
-
-Use :ref:`Jinja templating <concepts:jinja-templating>` with
-:template-fields:`airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360RunReportOperator`
-parameters which allow you to dynamically determine values.
-
-
.. _howto/operator:GoogleDisplayVideo360DownloadLineItemsOperator:
Downloading Line Items
diff --git a/scripts/in_container/verify_providers.py
b/scripts/in_container/verify_providers.py
index e7153548b9..dd09e2f7c2 100755
--- a/scripts/in_container/verify_providers.py
+++ b/scripts/in_container/verify_providers.py
@@ -150,10 +150,6 @@ KNOWN_COMMON_DEPRECATED_MESSAGES: set[str] = {
"Implementing implicit namespace packages (as specified in PEP 420) is "
"preferred to `pkg_resources.declare_namespace`",
"This module is deprecated. Please use
`airflow.providers.cncf.kubernetes.operators.pod` instead.",
- "This operator is deprecated. Please use
`GoogleDisplayVideo360CreateQueryOperator`",
- "This operator is deprecated. Please use
`GoogleDisplayVideo360RunQueryOperator`",
- "This operator is deprecated. Please use
`GoogleDisplayVideo360RunQuerySensor`",
- "This operator is deprecated. Please use
`GoogleDisplayVideo360DownloadReportV2Operator`",
"urllib3 (1.26.6) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
"urllib3 (1.26.9) or chardet (5.1.0)/charset_normalizer (2.0.12) doesn't
match a supported version!",
}
diff --git
a/tests/providers/google/marketing_platform/hooks/test_display_video.py
b/tests/providers/google/marketing_platform/hooks/test_display_video.py
index 55a3c91308..d46becb3db 100644
--- a/tests/providers/google/marketing_platform/hooks/test_display_video.py
+++ b/tests/providers/google/marketing_platform/hooks/test_display_video.py
@@ -22,7 +22,7 @@ from unittest import mock
from airflow.providers.google.marketing_platform.hooks.display_video import
GoogleDisplayVideo360Hook
from tests.providers.google.cloud.utils.base_gcp_mock import
mock_base_gcp_hook_default_project_id
-API_VERSION = "v1"
+API_VERSION = "v2"
GCP_CONN_ID = "google_cloud_default"
@@ -72,13 +72,13 @@ class TestGoogleDisplayVideo360Hook:
body = {"body": "test"}
return_value = "TEST"
-
get_conn_mock.return_value.queries.return_value.createquery.return_value.execute.return_value
= (
+
get_conn_mock.return_value.queries.return_value.create.return_value.execute.return_value
= (
return_value
)
result = self.hook.create_query(query=body)
-
get_conn_mock.return_value.queries.return_value.createquery.assert_called_once_with(body=body)
+
get_conn_mock.return_value.queries.return_value.create.assert_called_once_with(body=body)
assert return_value == result
@@ -90,13 +90,13 @@ class TestGoogleDisplayVideo360Hook:
query_id = "QUERY_ID"
return_value = "TEST"
-
get_conn_mock.return_value.queries.return_value.deletequery.return_value.execute.return_value
= (
+
get_conn_mock.return_value.queries.return_value.delete.return_value.execute.return_value
= (
return_value
)
self.hook.delete_query(query_id=query_id)
-
get_conn_mock.return_value.queries.return_value.deletequery.assert_called_once_with(queryId=query_id)
+
get_conn_mock.return_value.queries.return_value.delete.assert_called_once_with(queryId=query_id)
@mock.patch(
"airflow.providers.google.marketing_platform.hooks."
@@ -106,13 +106,11 @@ class TestGoogleDisplayVideo360Hook:
query_id = "QUERY_ID"
return_value = "TEST"
-
get_conn_mock.return_value.queries.return_value.getquery.return_value.execute.return_value
= (
- return_value
- )
+
get_conn_mock.return_value.queries.return_value.get.return_value.execute.return_value
= return_value
result = self.hook.get_query(query_id=query_id)
-
get_conn_mock.return_value.queries.return_value.getquery.assert_called_once_with(queryId=query_id)
+
get_conn_mock.return_value.queries.return_value.get.assert_called_once_with(queryId=query_id)
assert return_value == result
@@ -123,13 +121,11 @@ class TestGoogleDisplayVideo360Hook:
def test_list_queries(self, get_conn_mock):
queries = ["test"]
return_value = {"queries": queries}
-
get_conn_mock.return_value.queries.return_value.listqueries.return_value.execute.return_value
= (
- return_value
- )
+
get_conn_mock.return_value.queries.return_value.list.return_value.execute.return_value
= return_value
result = self.hook.list_queries()
-
get_conn_mock.return_value.queries.return_value.listqueries.assert_called_once_with()
+
get_conn_mock.return_value.queries.return_value.list.assert_called_once_with()
assert queries == result
@@ -143,7 +139,7 @@ class TestGoogleDisplayVideo360Hook:
self.hook.run_query(query_id=query_id, params=params)
-
get_conn_mock.return_value.queries.return_value.runquery.assert_called_once_with(
+
get_conn_mock.return_value.queries.return_value.run.assert_called_once_with(
queryId=query_id, body=params
)
diff --git
a/tests/providers/google/marketing_platform/operators/test_display_video.py
b/tests/providers/google/marketing_platform/operators/test_display_video.py
index d55a9a729a..0e5dc190a5 100644
--- a/tests/providers/google/marketing_platform/operators/test_display_video.py
+++ b/tests/providers/google/marketing_platform/operators/test_display_video.py
@@ -26,14 +26,11 @@ import pytest
from airflow.models import DAG, TaskInstance as TI
from airflow.providers.google.marketing_platform.operators.display_video
import (
GoogleDisplayVideo360CreateQueryOperator,
- GoogleDisplayVideo360CreateReportOperator,
GoogleDisplayVideo360CreateSDFDownloadTaskOperator,
GoogleDisplayVideo360DeleteReportOperator,
GoogleDisplayVideo360DownloadLineItemsOperator,
- GoogleDisplayVideo360DownloadReportOperator,
GoogleDisplayVideo360DownloadReportV2Operator,
GoogleDisplayVideo360RunQueryOperator,
- GoogleDisplayVideo360RunReportOperator,
GoogleDisplayVideo360SDFtoGCSOperator,
GoogleDisplayVideo360UploadLineItemsOperator,
)
@@ -53,44 +50,6 @@ QUERY_ID = FILENAME = "test"
OBJECT_NAME = "object_name"
-class TestGoogleDisplayVideo360CreateReportOperator:
- @mock.patch(
- "airflow.providers.google.marketing_platform.operators."
- "display_video.GoogleDisplayVideo360CreateReportOperator.xcom_push"
- )
- @mock.patch(
-
"airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360Hook"
- )
- def test_execute(self, hook_mock, xcom_mock):
- body = {"body": "test"}
- hook_mock.return_value.create_query.return_value = {"queryId":
QUERY_ID}
- op = GoogleDisplayVideo360CreateReportOperator(
- body=body, api_version=API_VERSION, task_id="test_task"
- )
- op.execute(context=None)
- hook_mock.assert_called_once_with(
- gcp_conn_id=GCP_CONN_ID,
- delegate_to=None,
- api_version=API_VERSION,
- impersonation_chain=None,
- )
- hook_mock.return_value.create_query.assert_called_once_with(query=body)
- xcom_mock.assert_called_once_with(None, key="report_id",
value=QUERY_ID)
-
- def test_prepare_template(self):
- body = {"key": "value"}
- with NamedTemporaryFile("w+", suffix=".json") as f:
- f.write(json.dumps(body))
- f.flush()
- op = GoogleDisplayVideo360CreateReportOperator(
- body=body, api_version=API_VERSION, task_id="test_task"
- )
- op.prepare_template()
-
- assert isinstance(op.body, dict)
- assert op.body == body
-
-
class TestGoogleDisplayVideo360DeleteReportOperator:
@mock.patch(
"airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360Hook"
@@ -109,135 +68,6 @@ class TestGoogleDisplayVideo360DeleteReportOperator:
hook_mock.return_value.delete_query.assert_called_once_with(query_id=QUERY_ID)
-class TestGoogleDisplayVideo360DownloadReportOperator:
- def setup_method(self):
- with create_session() as session:
- session.query(TI).delete()
-
- def teardown_method(self):
- with create_session() as session:
- session.query(TI).delete()
-
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.shutil")
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.urllib.request")
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.tempfile")
- @mock.patch(
- "airflow.providers.google.marketing_platform.operators."
- "display_video.GoogleDisplayVideo360DownloadReportOperator.xcom_push"
- )
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.GCSHook")
- @mock.patch(
-
"airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360Hook"
- )
- def test_execute(
- self,
- mock_hook,
- mock_gcs_hook,
- mock_xcom,
- mock_temp,
- mock_request,
- mock_shutil,
- ):
- mock_temp.NamedTemporaryFile.return_value.__enter__.return_value.name
= FILENAME
- mock_hook.return_value.get_query.return_value = {
- "metadata": {
- "running": False,
- "googleCloudStoragePathForLatestReport": "test",
- }
- }
- op = GoogleDisplayVideo360DownloadReportOperator(
- report_id=REPORT_ID,
- api_version=API_VERSION,
- bucket_name=BUCKET_NAME,
- report_name=REPORT_NAME,
- task_id="test_task",
- )
- op.execute(context=None)
- mock_hook.assert_called_once_with(
- gcp_conn_id=GCP_CONN_ID,
- delegate_to=None,
- api_version=API_VERSION,
- impersonation_chain=None,
- )
-
mock_hook.return_value.get_query.assert_called_once_with(query_id=REPORT_ID)
-
- mock_gcs_hook.assert_called_once_with(
- gcp_conn_id=GCP_CONN_ID,
- delegate_to=None,
- impersonation_chain=None,
- )
- mock_gcs_hook.return_value.upload.assert_called_once_with(
- bucket_name=BUCKET_NAME,
- filename=FILENAME,
- gzip=True,
- mime_type="text/csv",
- object_name=REPORT_NAME + ".gz",
- )
- mock_xcom.assert_called_once_with(None, key="report_name",
value=REPORT_NAME + ".gz")
-
- @pytest.mark.parametrize(
- "test_bucket_name",
- [BUCKET_NAME, f"gs://{BUCKET_NAME}", "XComArg", "{{
ti.xcom_pull(task_ids='f') }}"],
- )
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.shutil")
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.urllib.request")
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.tempfile")
-
@mock.patch("airflow.providers.google.marketing_platform.operators.display_video.GCSHook")
- @mock.patch(
-
"airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360Hook"
- )
- def test_set_bucket_name(
- self,
- mock_hook,
- mock_gcs_hook,
- mock_temp,
- mock_request,
- mock_shutil,
- test_bucket_name,
- ):
- mock_temp.NamedTemporaryFile.return_value.__enter__.return_value.name
= FILENAME
- mock_hook.return_value.get_query.return_value = {
- "metadata": {
- "running": False,
- "googleCloudStoragePathForLatestReport": "test",
- }
- }
-
- dag = DAG(
- dag_id="test_set_bucket_name",
- start_date=DEFAULT_DATE,
- schedule=None,
- catchup=False,
- )
-
- if BUCKET_NAME not in test_bucket_name:
-
- @dag.task
- def f():
- return BUCKET_NAME
-
- taskflow_op = f()
- taskflow_op.operator.run(start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE)
-
- op = GoogleDisplayVideo360DownloadReportOperator(
- report_id=REPORT_ID,
- api_version=API_VERSION,
- bucket_name=test_bucket_name if test_bucket_name != "XComArg" else
taskflow_op,
- report_name=REPORT_NAME,
- task_id="test_task",
- dag=dag,
- )
- op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
-
- mock_gcs_hook.return_value.upload.assert_called_once_with(
- bucket_name=BUCKET_NAME,
- filename=FILENAME,
- gzip=True,
- mime_type="text/csv",
- object_name=REPORT_NAME + ".gz",
- )
-
-
class TestGoogleDisplayVideo360DownloadReportV2Operator:
def setup_method(self):
with create_session() as session:
@@ -364,28 +194,6 @@ class TestGoogleDisplayVideo360DownloadReportV2Operator:
)
-class TestGoogleDisplayVideo360RunReportOperator:
- @mock.patch(
-
"airflow.providers.google.marketing_platform.operators.display_video.GoogleDisplayVideo360Hook"
- )
- def test_execute(self, hook_mock):
- parameters = {"param": "test"}
- op = GoogleDisplayVideo360RunReportOperator(
- report_id=REPORT_ID,
- parameters=parameters,
- api_version=API_VERSION,
- task_id="test_task",
- )
- op.execute(context=None)
- hook_mock.assert_called_once_with(
- gcp_conn_id=GCP_CONN_ID,
- delegate_to=None,
- api_version=API_VERSION,
- impersonation_chain=None,
- )
-
hook_mock.return_value.run_query.assert_called_once_with(query_id=REPORT_ID,
params=parameters)
-
-
class TestGoogleDisplayVideo360RunQueryOperator:
@mock.patch(
"airflow.providers.google.marketing_platform.operators."
diff --git
a/tests/providers/google/marketing_platform/sensors/test_display_video.py
b/tests/providers/google/marketing_platform/sensors/test_display_video.py
index 193ba18e6b..6000334b86 100644
--- a/tests/providers/google/marketing_platform/sensors/test_display_video.py
+++ b/tests/providers/google/marketing_platform/sensors/test_display_video.py
@@ -21,7 +21,6 @@ from unittest import mock
from airflow.providers.google.marketing_platform.sensors.display_video import (
GoogleDisplayVideo360GetSDFDownloadOperationSensor,
- GoogleDisplayVideo360ReportSensor,
GoogleDisplayVideo360RunQuerySensor,
)
@@ -29,24 +28,6 @@ API_VERSION = "api_version"
GCP_CONN_ID = "google_cloud_default"
-class TestGoogleDisplayVideo360ReportSensor:
-
@mock.patch("airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360Hook")
-
@mock.patch("airflow.providers.google.marketing_platform.sensors.display_video.BaseSensorOperator")
- def test_poke(self, mock_base_op, hook_mock):
- report_id = "REPORT_ID"
- op = GoogleDisplayVideo360ReportSensor(
- report_id=report_id, api_version=API_VERSION, task_id="test_task"
- )
- op.poke(context=None)
- hook_mock.assert_called_once_with(
- gcp_conn_id=GCP_CONN_ID,
- delegate_to=None,
- api_version=API_VERSION,
- impersonation_chain=None,
- )
-
hook_mock.return_value.get_query.assert_called_once_with(query_id=report_id)
-
-
class TestGoogleDisplayVideo360RunQuerySensor:
@mock.patch("airflow.providers.google.marketing_platform.sensors.display_video.GoogleDisplayVideo360Hook")
@mock.patch("airflow.providers.google.marketing_platform.sensors.display_video.BaseSensorOperator")
