This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f89ab4c0005 fix: add required arguments when creating an external
table (#58199)
f89ab4c0005 is described below
commit f89ab4c000542d78ba95b5b453f792624b882b28
Author: Nathan Hadfield <[email protected]>
AuthorDate: Thu Nov 13 13:59:26 2025 +0000
fix: add required arguments when creating an external table (#58199)
---
.../google/cloud/transfers/gcs_to_bigquery.py | 2 +
.../google/cloud/transfers/test_gcs_to_bigquery.py | 47 ++++++++++++++++++++++
2 files changed, 49 insertions(+)
diff --git
a/providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
b/providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
index dd1fb8e59ba..97a8efbd439 100644
---
a/providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
+++
b/providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
@@ -593,6 +593,8 @@ class GCSToBigQueryOperator(BaseOperator):
self.hook.create_table(
table_resource=table_obj_api_repr,
project_id=self.project_id or self.hook.project_id,
+ dataset_id=table.dataset_id,
+ table_id=table.table_id,
location=self.location,
exists_ok=True,
)
diff --git
a/providers/google/tests/unit/google/cloud/transfers/test_gcs_to_bigquery.py
b/providers/google/tests/unit/google/cloud/transfers/test_gcs_to_bigquery.py
index ad6f3a08a6f..4383ce7ddc9 100644
--- a/providers/google/tests/unit/google/cloud/transfers/test_gcs_to_bigquery.py
+++ b/providers/google/tests/unit/google/cloud/transfers/test_gcs_to_bigquery.py
@@ -123,6 +123,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -155,6 +157,33 @@ class TestGCSToBigQueryOperator:
project_id=JOB_PROJECT_ID,
)
+ @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook"))
+ def test_external_table_explicitly_passes_dataset_and_table_ids(self,
hook):
+ hook.return_value.insert_job.side_effect = [
+ MagicMock(job_id=REAL_JOB_ID, error_result=False),
+ REAL_JOB_ID,
+ ]
+ hook.return_value.generate_job_id.return_value = REAL_JOB_ID
+ hook.return_value.split_tablename.return_value = (PROJECT_ID, DATASET,
TABLE)
+
+ def _validate_create_table(**kwargs):
+ assert kwargs["dataset_id"] == DATASET
+ assert kwargs["table_id"] == TABLE
+
+ hook.return_value.create_table.side_effect = _validate_create_table
+
+ operator = GCSToBigQueryOperator(
+ task_id=TASK_ID,
+ bucket=TEST_BUCKET,
+ source_objects=TEST_SOURCE_OBJECTS,
+ destination_project_dataset_table=TEST_EXPLICIT_DEST,
+ schema_fields=SCHEMA_FIELDS,
+ external_table=True,
+ project_id=JOB_PROJECT_ID,
+ )
+
+ operator.execute(context=MagicMock())
+
@mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook"))
def
test_max_value_without_external_table_should_execute_successfully(self, hook):
hook.return_value.insert_job.side_effect = [
@@ -333,6 +362,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": LABELS,
@@ -434,6 +465,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -536,6 +569,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -640,6 +675,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -745,6 +782,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -849,6 +888,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -1051,6 +1092,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -1235,6 +1278,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {"projectId": PROJECT_ID, "datasetId":
DATASET, "tableId": TABLE},
"labels": {},
@@ -1616,6 +1661,8 @@ class TestGCSToBigQueryOperator:
exists_ok=True,
location=None,
project_id=JOB_PROJECT_ID,
+ dataset_id=DATASET,
+ table_id=TABLE,
table_resource={
"tableReference": {
"projectId": PROJECT_ID,
