This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8de544a Moved application of lower bound from _get_split_keys to
split to avoid source bundles of weight=0
new 811d41f Merge pull request #12142 from
corvin-quodai/mdbio-zerodivision
8de544a is described below
commit 8de544a2669e5f19987a4b4b8a8213610d8a0dc5
Author: Corvin Deboeser <cor...@quod.ai>
AuthorDate: Wed Jul 1 08:28:18 2020 +0800
Moved application of lower bound from _get_split_keys to split to avoid
source bundles of weight=0
---
sdks/python/apache_beam/io/mongodbio.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/io/mongodbio.py
b/sdks/python/apache_beam/io/mongodbio.py
index 324f1e3..aed0ff2 100644
--- a/sdks/python/apache_beam/io/mongodbio.py
+++ b/sdks/python/apache_beam/io/mongodbio.py
@@ -171,6 +171,12 @@ class _BoundedMongoSource(iobase.BoundedSource):
start_position, stop_position)
desired_bundle_size_in_mb = desired_bundle_size // 1024 // 1024
+
+ # for desired bundle size, if desired chunk size smaller than 1mb, use
+ # mongodb default split size of 1mb.
+ if desired_bundle_size_in_mb < 1:
+ desired_bundle_size_in_mb = 1
+
split_keys = self._get_split_keys(
desired_bundle_size_in_mb, start_position, stop_position)
@@ -221,10 +227,6 @@ class _BoundedMongoSource(iobase.BoundedSource):
def _get_split_keys(self, desired_chunk_size_in_mb, start_pos, end_pos):
# calls mongodb splitVector command to get document ids at split position
- # for desired bundle size, if desired chunk size smaller than 1mb, use
- # mongodb default split size of 1mb.
- if desired_chunk_size_in_mb < 1:
- desired_chunk_size_in_mb = 1
if start_pos >= end_pos:
# single document not splittable
return []
