This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 584087f087 Add templated fields to `BigQueryToSqlBaseOperator` from
`BigQueryToPostgresOperator` (#36663)
584087f087 is described below
commit 584087f0874453d8e44ae4cd6ec282340dbf6c84
Author: rom sharon <[email protected]>
AuthorDate: Tue Jan 9 08:23:22 2024 +0200
Add templated fields to `BigQueryToSqlBaseOperator` from
`BigQueryToPostgresOperator` (#36663)
---
airflow/providers/google/cloud/transfers/bigquery_to_sql.py | 6 ++++++
airflow/providers/google/cloud/utils/bigquery_get_data.py | 4 ++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_sql.py
b/airflow/providers/google/cloud/transfers/bigquery_to_sql.py
index 736880726c..e3a24fab5c 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_sql.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_sql.py
@@ -68,6 +68,8 @@ class BigQueryToSqlBaseOperator(BaseOperator):
template_fields: Sequence[str] = (
"target_table_name",
"impersonation_chain",
+ "dataset_id",
+ "table_id",
)
def __init__(
@@ -82,6 +84,8 @@ class BigQueryToSqlBaseOperator(BaseOperator):
batch_size: int = 1000,
location: str | None = None,
impersonation_chain: str | Sequence[str] | None = None,
+ dataset_id: str | None = None,
+ table_id: str | None = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
@@ -93,6 +97,8 @@ class BigQueryToSqlBaseOperator(BaseOperator):
self.batch_size = batch_size
self.location = location
self.impersonation_chain = impersonation_chain
+ self.dataset_id = dataset_id
+ self.table_id = table_id
try:
self.dataset_id, self.table_id = dataset_table.split(".")
except ValueError:
diff --git a/airflow/providers/google/cloud/utils/bigquery_get_data.py
b/airflow/providers/google/cloud/utils/bigquery_get_data.py
index d178aee963..6e1df55286 100644
--- a/airflow/providers/google/cloud/utils/bigquery_get_data.py
+++ b/airflow/providers/google/cloud/utils/bigquery_get_data.py
@@ -30,8 +30,8 @@ if TYPE_CHECKING:
def bigquery_get_data(
logger: Logger,
- dataset_id: str,
- table_id: str,
+ dataset_id: str | None,
+ table_id: str | None,
big_query_hook: BigQueryHook,
batch_size: int,
selected_fields: list[str] | str | None,
