Repository: incubator-beam Updated Branches: refs/heads/python-sdk 6ac6e420f -> c4208a899
Allow for passing format so that we can migrate to BQ Avro export later Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d0e31218 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d0e31218 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d0e31218 Branch: refs/heads/python-sdk Commit: d0e312184e319050baa02abff2c08348b6cfb651 Parents: 6ac6e42 Author: Sourabh Bajaj <sourabhba...@google.com> Authored: Mon Nov 7 18:15:17 2016 -0800 Committer: Robert Bradshaw <rober...@gmail.com> Committed: Tue Nov 15 08:53:06 2016 -0800 ---------------------------------------------------------------------- sdks/python/apache_beam/runners/dataflow_runner.py | 1 + sdks/python/apache_beam/utils/names.py | 1 + 2 files changed, 2 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d0e31218/sdks/python/apache_beam/runners/dataflow_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow_runner.py index 57867fa..00b466b 100644 --- a/sdks/python/apache_beam/runners/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow_runner.py @@ -515,6 +515,7 @@ class DataflowPipelineRunner(PipelineRunner): elif transform.source.format == 'text': step.add_property(PropertyNames.FILE_PATTERN, transform.source.path) elif transform.source.format == 'bigquery': + step.add_property(PropertyNames.BIGQUERY_EXPORT_FORMAT, 'FORMAT_JSON') # TODO(silviuc): Add table validation if transform.source.validate. if transform.source.table_reference is not None: step.add_property(PropertyNames.BIGQUERY_DATASET, http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d0e31218/sdks/python/apache_beam/utils/names.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/names.py b/sdks/python/apache_beam/utils/names.py index be8c92a..3edde3c 100644 --- a/sdks/python/apache_beam/utils/names.py +++ b/sdks/python/apache_beam/utils/names.py @@ -46,6 +46,7 @@ class PropertyNames(object): BIGQUERY_DATASET = 'dataset' BIGQUERY_QUERY = 'bigquery_query' BIGQUERY_USE_LEGACY_SQL = 'bigquery_use_legacy_sql' + BIGQUERY_EXPORT_FORMAT = 'bigquery_export_format' BIGQUERY_TABLE = 'table' BIGQUERY_PROJECT = 'project' BIGQUERY_SCHEMA = 'schema'
