Repository: beam Updated Branches: refs/heads/master 13e1be2f0 -> 69522fe66
Fix documentation for the shard_template_name Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b3f74044 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b3f74044 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b3f74044 Branch: refs/heads/master Commit: b3f74044436f4b4a92f3135411b5d5093437ec3e Parents: 13e1be2 Author: Sourabh Bajaj <[email protected]> Authored: Mon May 15 14:34:45 2017 -0700 Committer: Sourabh Bajaj <[email protected]> Committed: Mon May 15 14:34:45 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/io/avroio.py | 13 ++++++------- sdks/python/apache_beam/io/textio.py | 13 ++++++------- sdks/python/apache_beam/io/tfrecordio.py | 15 +++++++-------- 3 files changed, 19 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/avroio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py index e02e1f7..7df9983 100644 --- a/sdks/python/apache_beam/io/avroio.py +++ b/sdks/python/apache_beam/io/avroio.py @@ -312,13 +312,12 @@ class WriteToAvro(beam.transforms.PTransform): the performance of a pipeline. Setting this value is not recommended unless you require a specific number of output files. shard_name_template: A template string containing placeholders for - the shard number and shard count. Currently only '' and - '-SSSSS-of-NNNNN' are patterns accepted by the service. - When constructing a filename for a particular shard number, the - upper-case letters 'S' and 'N' are replaced with the 0-padded shard - number and shard count respectively. This argument can be '' in which - case it behaves as if num_shards was set to 1 and only one file will be - generated. The default pattern used is '-SSSSS-of-NNNNN'. + the shard number and shard count. When constructing a filename for a + particular shard number, the upper-case letters 'S' and 'N' are + replaced with the 0-padded shard number and shard count respectively. + This argument can be '' in which case it behaves as if num_shards was + set to 1 and only one file will be generated. The default pattern used + is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template. mime_type: The MIME type to use for the produced files, if the filesystem supports specifying MIME types. http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/textio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/textio.py b/sdks/python/apache_beam/io/textio.py index eeefaf6..60e1512 100644 --- a/sdks/python/apache_beam/io/textio.py +++ b/sdks/python/apache_beam/io/textio.py @@ -291,13 +291,12 @@ class _TextSink(filebasedsink.FileBasedSink): the performance of a pipeline. Setting this value is not recommended unless you require a specific number of output files. shard_name_template: A template string containing placeholders for - the shard number and shard count. Currently only '' and - '-SSSSS-of-NNNNN' are patterns accepted by the service. - When constructing a filename for a particular shard number, the - upper-case letters 'S' and 'N' are replaced with the 0-padded shard - number and shard count respectively. This argument can be '' in which - case it behaves as if num_shards was set to 1 and only one file will be - generated. The default pattern used is '-SSSSS-of-NNNNN'. + the shard number and shard count. When constructing a filename for a + particular shard number, the upper-case letters 'S' and 'N' are + replaced with the 0-padded shard number and shard count respectively. + This argument can be '' in which case it behaves as if num_shards was + set to 1 and only one file will be generated. The default pattern used + is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template. coder: Coder used to encode each line. compression_type: Used to handle compressed output files. Typical value is CompressionTypes.AUTO, in which case the final file path's http://git-wip-us.apache.org/repos/asf/beam/blob/b3f74044/sdks/python/apache_beam/io/tfrecordio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/tfrecordio.py b/sdks/python/apache_beam/io/tfrecordio.py index a8cd1ce..d7eb932 100644 --- a/sdks/python/apache_beam/io/tfrecordio.py +++ b/sdks/python/apache_beam/io/tfrecordio.py @@ -242,7 +242,7 @@ class WriteToTFRecord(PTransform): coder=coders.BytesCoder(), file_name_suffix='', num_shards=0, - shard_name_template=filebasedsink.DEFAULT_SHARD_NAME_TEMPLATE, + shard_name_template=None, compression_type=CompressionTypes.AUTO, **kwargs): """Initialize WriteToTFRecord transform. @@ -256,13 +256,12 @@ class WriteToTFRecord(PTransform): num_shards: The number of files (shards) used for output. If not set, the default value will be used. shard_name_template: A template string containing placeholders for - the shard number and shard count. Currently only '' and - '-SSSSS-of-NNNNN' are patterns allowed. - When constructing a filename for a particular shard number, the - upper-case letters 'S' and 'N' are replaced with the 0-padded shard - number and shard count respectively. This argument can be '' in which - case it behaves as if num_shards was set to 1 and only one file will be - generated. The default pattern is '-SSSSS-of-NNNNN'. + the shard number and shard count. When constructing a filename for a + particular shard number, the upper-case letters 'S' and 'N' are + replaced with the 0-padded shard number and shard count respectively. + This argument can be '' in which case it behaves as if num_shards was + set to 1 and only one file will be generated. The default pattern used + is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template. compression_type: Used to handle compressed output files. Typical value is CompressionTypes.AUTO, in which case the file_path's extension will be used to detect the compression.
