mik-laj commented on a change in pull request #6777: [AIRFLOW-6223] BigQuery - 
improve hook tests [depends on AIRFLOW-6220]
URL: https://github.com/apache/airflow/pull/6777#discussion_r356633037
 
 

 ##########
 File path: tests/gcp/hooks/test_bigquery.py
 ##########
 @@ -177,105 +119,146 @@ def test_invalid_syntax(self, table_input, var_name, 
exception_message):
 
 
 class TestBigQueryHookSourceFormat(unittest.TestCase):
-    def test_invalid_source_format(self):
-        with self.assertRaises(Exception) as context:
-            hook.BigQueryBaseCursor("test", "test").run_load(
+    @mock.patch(
+        'airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id',
+        return_value=("CREDENTIALS", "PROJECT_ID",)
+    )
+    @mock.patch("airflow.gcp.hooks.bigquery.BigQueryHook.get_service")
+    def test_invalid_source_format(self, mock_get_service, 
mock_get_creds_and_proj_id):
+        with self.assertRaisesRegex(
+            Exception,
+            r"JSON is not a valid source format. Please use one of the 
following types: \['CSV', "
+            r"'NEWLINE_DELIMITED_JSON', 'AVRO', 'GOOGLE_SHEETS', 
'DATASTORE_BACKUP', 'PARQUET'\]"
+        ):
+            bq_hook = hook.BigQueryHook()
+            cursor = bq_hook.get_cursor()
+            cursor.run_load(
                 "test.test", "test_schema.json", ["test_data.json"], 
source_format="json"
             )
 
-        # since we passed 'json' in, and it's not valid, make sure it's 
present in the
-        # error string.
-        self.assertIn("JSON", str(context.exception))
-
 
 class TestBigQueryExternalTableSourceFormat(unittest.TestCase):
-    def test_invalid_source_format(self):
-        with self.assertRaises(Exception) as context:
-            hook.BigQueryBaseCursor("test", "test").create_external_table(
+    @mock.patch(
+        'airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id',
+        return_value=("CREDENTIALS", "PROJECT_ID",)
+    )
+    @mock.patch("airflow.gcp.hooks.bigquery.BigQueryHook.get_service")
+    def test_invalid_source_format(self, mock_get_service, 
mock_get_creds_and_proj_id):
+        with self.assertRaisesRegex(
+            Exception,
+            r"JSON is not a valid source format. Please use one of the 
following types: \['CSV', "
+            r"'NEWLINE_DELIMITED_JSON', 'AVRO', 'GOOGLE_SHEETS', 
'DATASTORE_BACKUP', 'PARQUET'\]"
+        ):
+            bq_hook = hook.BigQueryHook()
+            cursor = bq_hook.get_cursor()
+            cursor.create_external_table(
                 external_project_dataset_table='test.test',
                 schema_fields='test_schema.json',
                 source_uris=['test_data.json'],
                 source_format='json'
             )
 
-        # since we passed 'csv' in, and it's not valid, make sure it's present 
in the
-        # error string.
-        self.assertIn("JSON", str(context.exception))
-
-
-# Helpers to test_cancel_queries that have mock_poll_job_complete returning 
false,
-# unless mock_job_cancel was called with the same job_id
-mock_canceled_jobs = []  # type: List
-
-
-def mock_poll_job_complete(job_id):
-    return job_id in mock_canceled_jobs
-
-
-# pylint: disable=invalid-name
-# noinspection PyUnusedLocal
-def mock_job_cancel(projectId, jobId):  # pylint: disable=unused-argument
-    mock_canceled_jobs.append(jobId)
-    return mock.Mock()
-
 
 class TestBigQueryBaseCursor(unittest.TestCase):
-    def test_invalid_schema_update_options(self):
-        with self.assertRaises(Exception) as context:
-            hook.BigQueryBaseCursor("test", "test").run_load(
+    @mock.patch(
+        'airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id',
+        return_value=("CREDENTIALS", "PROJECT_ID",)
+    )
+    @mock.patch("airflow.gcp.hooks.bigquery.BigQueryHook.get_service")
+    def test_invalid_schema_update_options(self, mock_get_service, 
mock_get_creds_and_proj_id):
+        with self.assertRaisesRegex(
+            Exception,
+            r"\['THIS IS NOT VALID'\] contains invalid schema update 
options.Please only use one or more of "
+            r"the following options: \['ALLOW_FIELD_ADDITION', 
'ALLOW_FIELD_RELAXATION'\]"
+        ):
+            bq_hook = hook.BigQueryHook()
+            cursor = bq_hook.get_cursor()
+            cursor.run_load(
                 "test.test",
                 "test_schema.json",
                 ["test_data.json"],
                 schema_update_options=["THIS IS NOT VALID"]
             )
-        self.assertIn("THIS IS NOT VALID", str(context.exception))
 
-    def test_invalid_schema_update_and_write_disposition(self):
-        with self.assertRaises(Exception) as context:
-            hook.BigQueryBaseCursor("test", "test").run_load(
+    @mock.patch(
+        'airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id',
+        return_value=("CREDENTIALS", "PROJECT_ID",)
+    )
+    @mock.patch("airflow.gcp.hooks.bigquery.BigQueryHook.get_service")
+    def test_invalid_schema_update_and_write_disposition(self, 
mock_get_service, mock_get_creds_and_proj_id):
+        with self.assertRaisesRegex(Exception, "schema_update_options is only 
allowed if"
+                                               " write_disposition is 
'WRITE_APPEND' or 'WRITE_TRUNCATE'."):
+            bq_hook = hook.BigQueryHook()
+            cursor = bq_hook.get_cursor()
+            cursor.run_load(
                 "test.test",
                 "test_schema.json",
                 ["test_data.json"],
                 schema_update_options=['ALLOW_FIELD_ADDITION'],
                 write_disposition='WRITE_EMPTY'
             )
-        self.assertIn("schema_update_options is only", str(context.exception))
 
-    def test_cancel_queries(self):
+    
@mock.patch("airflow.gcp.hooks.bigquery.BigQueryBaseCursor.poll_job_complete", 
side_effect=[False, True])
+    
@mock.patch('airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id')
 
 Review comment:
   ```suggestion
       
@mock.patch('airflow.gcp.hooks.base.CloudBaseHook._get_credentials_and_project_id',
 return_value=("CREDENTIALS", "PROJECT_ID",))
   ```

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[email protected]


With regards,
Apache Git Services

Reply via email to