kaxil commented on a change in pull request #3894: [AIRFLOW-3055] add
get_dataset and get_datasets_list to bigquery_hook
URL: https://github.com/apache/incubator-airflow/pull/3894#discussion_r219678497
##########
File path: airflow/contrib/hooks/bigquery_hook.py
##########
@@ -1375,6 +1375,74 @@ def delete_dataset(self, project_id, dataset_id):
'BigQuery job failed. Error was: {}'.format(err.content)
)
+ def get_dataset(self, dataset_id="", project_id=""):
+ """
+ Method returned dataset_resource if dataset exist
+ and raised 404 error if dataset does not exist
+
+ :param dataset_id: dataset_id to get
+ :type dataset_id: str
+ :param project_id: Google Cloud Project for which you try to get
dataset
+ :type project_id: str
+ :return dataset_resource
+
+ .. seealso::
+ For more information, see Dataset Resource content:
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
+ """
+
+ dataset_project_id = project_id if project_id else self.project_id
+
+ try:
+ dataset_resource = self.service.datasets().get(
+ datasetId=dataset_id, projectId=dataset_project_id).execute()
+ self.log.info(dataset_resource)
+ except HttpError as err:
+ raise AirflowException(
+ 'BigQuery job failed. Error was: {}'.format(err.content))
+
+ return dataset_resource
+
+ def get_datasets_list(self, project_id=""):
+ """
+ Method returned full list of BigQuery datasets into current project
+
+ .. seealso::
+ For more information, see:
+
https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/list
+
+ :param project_id: gcp project_id for which you try to get all datasets
+ :type project_id: str
+ :return datasets_list:
+
+ Example of returned datasets_list:
+
+ [
+ {u'kind': u'bigquery#dataset', u'location': u'US',
+ u'id': u'your-project:dataset_2_test',
+ u'datasetReference': {u'projectId': u'your-project',
+ u'datasetId': u'dataset_2_test'}},
+
+ {u'kind': u'bigquery#dataset', u'location': u'US',
+ u'id': u'your-project:dataset_1_test',
+ u'datasetReference': {u'projectId': u'your-project',
+ u'datasetId': u'dataset_1_test'}}
+ ]
Review comment:
I think the above JSON array can be properly formatted.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
