Repository: incubator-airflow Updated Branches: refs/heads/master 2fa1ae989 -> 6b9126733
[AIRFLOW-2258] Allow import of Parquet-format files into BigQuery Closes #3164 from stevesoundcloud/stevesoundcloud /allow-import-of-parquet-format-files-into- bigquery-AIRFLOW-2258 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6b912673 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6b912673 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6b912673 Branch: refs/heads/master Commit: 6b91267338bead35fb65e1545171c362a9d24dd1 Parents: 2fa1ae9 Author: Steve Conover <[email protected]> Authored: Tue Mar 27 07:40:46 2018 -0700 Committer: Chris Riccomini <[email protected]> Committed: Tue Mar 27 07:40:46 2018 -0700 ---------------------------------------------------------------------- airflow/contrib/hooks/bigquery_hook.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6b912673/airflow/contrib/hooks/bigquery_hook.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py index c6499d3..aac221f 100644 --- a/airflow/contrib/hooks/bigquery_hook.py +++ b/airflow/contrib/hooks/bigquery_hook.py @@ -351,7 +351,7 @@ class BigQueryBaseCursor(LoggingMixin): source_format = source_format.upper() allowed_formats = [ "CSV", "NEWLINE_DELIMITED_JSON", "AVRO", "GOOGLE_SHEETS", - "DATASTORE_BACKUP" + "DATASTORE_BACKUP", "PARQUET" ] if source_format not in allowed_formats: raise ValueError("{0} is not a valid source format. " @@ -807,7 +807,7 @@ class BigQueryBaseCursor(LoggingMixin): source_format = source_format.upper() allowed_formats = [ "CSV", "NEWLINE_DELIMITED_JSON", "AVRO", "GOOGLE_SHEETS", - "DATASTORE_BACKUP" + "DATASTORE_BACKUP", "PARQUET" ] if source_format not in allowed_formats: raise ValueError("{0} is not a valid source format. " @@ -903,6 +903,7 @@ class BigQueryBaseCursor(LoggingMixin): ], 'DATASTORE_BACKUP': ['projectionFields'], 'NEWLINE_DELIMITED_JSON': ['autodetect', 'ignoreUnknownValues'], + 'PARQUET': ['autodetect', 'ignoreUnknownValues'], 'AVRO': [], } valid_configs = src_fmt_to_configs_mapping[source_format]
