kaxil closed pull request #4213: [AIRFLOW-3371] BigQueryHook's Ability to
Create View
URL: https://github.com/apache/incubator-airflow/pull/4213
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/contrib/hooks/bigquery_hook.py
b/airflow/contrib/hooks/bigquery_hook.py
index a03429e155..c7324adde4 100644
--- a/airflow/contrib/hooks/bigquery_hook.py
+++ b/airflow/contrib/hooks/bigquery_hook.py
@@ -218,10 +218,11 @@ def create_empty_table(self,
table_id,
schema_fields=None,
time_partitioning=None,
- labels=None
- ):
+ labels=None,
+ view=None):
"""
Creates a new, empty table in the dataset.
+ To create a view, which is defined by a SQL query, parse a dictionary
to 'view' kwarg
:param project_id: The project to create the table into.
:type project_id: str
@@ -246,6 +247,17 @@ def create_empty_table(self,
.. seealso::
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning
:type time_partitioning: dict
+ :param view: [Optional] A dictionary containing definition for the
view.
+ If set, it will create a view instead of a table:
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view
+ :type view: dict
+
+ **Example**: ::
+
+ view = {
+ "query": "SELECT * FROM
`test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000",
+ "useLegacySql": False
+ }
:return:
"""
@@ -267,6 +279,9 @@ def create_empty_table(self,
if labels:
table_resource['labels'] = labels
+ if view:
+ table_resource['view'] = view
+
self.log.info('Creating Table %s:%s.%s',
project_id, dataset_id, table_id)
diff --git a/tests/contrib/hooks/test_bigquery_hook.py
b/tests/contrib/hooks/test_bigquery_hook.py
index 9099dcbbb7..8c59116c85 100644
--- a/tests/contrib/hooks/test_bigquery_hook.py
+++ b/tests/contrib/hooks/test_bigquery_hook.py
@@ -22,6 +22,7 @@
from google.auth.exceptions import GoogleAuthError
import mock
+from apiclient.errors import HttpError
from airflow.contrib.hooks import bigquery_hook as hook
from airflow.contrib.hooks.bigquery_hook import _cleanse_time_partitioning, \
@@ -344,6 +345,48 @@ def test_insert_all_fail(self, run_with_config):
cursor.insert_all(project_id, dataset_id, table_id,
rows, fail_on_error=True)
+ @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+ def test_create_view_fails_on_exception(self, run_with_config):
+ project_id = 'bq-project'
+ dataset_id = 'bq_dataset'
+ table_id = 'bq_table_view'
+ view = {
+ 'incorrect_key': 'SELECT * FROM
`test-project-id.test_dataset_id.test_table_prefix*`',
+ "useLegacySql": False
+ }
+
+ mock_service = mock.Mock()
+ method = (mock_service.tables.return_value.insert)
+ method.return_value.execute.side_effect = HttpError(
+ resp={'status': '400'}, content=b'Query is required for views')
+ cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+ with self.assertRaises(Exception):
+ cursor.create_empty_table(project_id, dataset_id, table_id,
+ view=view)
+
+ @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
+ def test_create_view(self, run_with_config):
+ project_id = 'bq-project'
+ dataset_id = 'bq_dataset'
+ table_id = 'bq_table_view'
+ view = {
+ 'query': 'SELECT * FROM
`test-project-id.test_dataset_id.test_table_prefix*`',
+ "useLegacySql": False
+ }
+
+ mock_service = mock.Mock()
+ method = (mock_service.tables.return_value.insert)
+ cursor = hook.BigQueryBaseCursor(mock_service, project_id)
+ cursor.create_empty_table(project_id, dataset_id, table_id,
+ view=view)
+ body = {
+ 'tableReference': {
+ 'tableId': table_id
+ },
+ 'view': view
+ }
+ method.assert_called_once_with(projectId=project_id,
datasetId=dataset_id, body=body)
+
class TestBigQueryCursor(unittest.TestCase):
@mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
