[ https://issues.apache.org/jira/browse/AIRFLOW-3371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16695349#comment-16695349 ]
ASF GitHub Bot commented on AIRFLOW-3371: ----------------------------------------- kaxil closed pull request #4213: [AIRFLOW-3371] BigQueryHook's Ability to Create View URL: https://github.com/apache/incubator-airflow/pull/4213 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py index a03429e155..c7324adde4 100644 --- a/airflow/contrib/hooks/bigquery_hook.py +++ b/airflow/contrib/hooks/bigquery_hook.py @@ -218,10 +218,11 @@ def create_empty_table(self, table_id, schema_fields=None, time_partitioning=None, - labels=None - ): + labels=None, + view=None): """ Creates a new, empty table in the dataset. + To create a view, which is defined by a SQL query, parse a dictionary to 'view' kwarg :param project_id: The project to create the table into. :type project_id: str @@ -246,6 +247,17 @@ def create_empty_table(self, .. seealso:: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning :type time_partitioning: dict + :param view: [Optional] A dictionary containing definition for the view. + If set, it will create a view instead of a table: + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#view + :type view: dict + + **Example**: :: + + view = { + "query": "SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*` LIMIT 1000", + "useLegacySql": False + } :return: """ @@ -267,6 +279,9 @@ def create_empty_table(self, if labels: table_resource['labels'] = labels + if view: + table_resource['view'] = view + self.log.info('Creating Table %s:%s.%s', project_id, dataset_id, table_id) diff --git a/tests/contrib/hooks/test_bigquery_hook.py b/tests/contrib/hooks/test_bigquery_hook.py index 9099dcbbb7..8c59116c85 100644 --- a/tests/contrib/hooks/test_bigquery_hook.py +++ b/tests/contrib/hooks/test_bigquery_hook.py @@ -22,6 +22,7 @@ from google.auth.exceptions import GoogleAuthError import mock +from apiclient.errors import HttpError from airflow.contrib.hooks import bigquery_hook as hook from airflow.contrib.hooks.bigquery_hook import _cleanse_time_partitioning, \ @@ -344,6 +345,48 @@ def test_insert_all_fail(self, run_with_config): cursor.insert_all(project_id, dataset_id, table_id, rows, fail_on_error=True) + @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration') + def test_create_view_fails_on_exception(self, run_with_config): + project_id = 'bq-project' + dataset_id = 'bq_dataset' + table_id = 'bq_table_view' + view = { + 'incorrect_key': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`', + "useLegacySql": False + } + + mock_service = mock.Mock() + method = (mock_service.tables.return_value.insert) + method.return_value.execute.side_effect = HttpError( + resp={'status': '400'}, content=b'Query is required for views') + cursor = hook.BigQueryBaseCursor(mock_service, project_id) + with self.assertRaises(Exception): + cursor.create_empty_table(project_id, dataset_id, table_id, + view=view) + + @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration') + def test_create_view(self, run_with_config): + project_id = 'bq-project' + dataset_id = 'bq_dataset' + table_id = 'bq_table_view' + view = { + 'query': 'SELECT * FROM `test-project-id.test_dataset_id.test_table_prefix*`', + "useLegacySql": False + } + + mock_service = mock.Mock() + method = (mock_service.tables.return_value.insert) + cursor = hook.BigQueryBaseCursor(mock_service, project_id) + cursor.create_empty_table(project_id, dataset_id, table_id, + view=view) + body = { + 'tableReference': { + 'tableId': table_id + }, + 'view': view + } + method.assert_called_once_with(projectId=project_id, datasetId=dataset_id, body=body) + class TestBigQueryCursor(unittest.TestCase): @mock.patch.object(hook.BigQueryBaseCursor, 'run_with_configuration') ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > BigQueryHook's Ability to Create View > ------------------------------------- > > Key: AIRFLOW-3371 > URL: https://issues.apache.org/jira/browse/AIRFLOW-3371 > Project: Apache Airflow > Issue Type: New Feature > Reporter: Ryan Yuan > Assignee: Ryan Yuan > Priority: Major > > Modify *BigQueryBaseCursor.create_empty_table()* to take in an optional > 'view' parameter to create view in BigQuery. -- This message was sent by Atlassian JIRA (v7.6.3#76005)