This is an automated email from the ASF dual-hosted git repository.
robertwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a63e15f Avoid apiary submission of job graph when it is not needed.
(#15458)
a63e15f is described below
commit a63e15f714765f4f89d166c12430ee433a695e51
Author: Robert Bradshaw <[email protected]>
AuthorDate: Mon Sep 13 16:42:20 2021 -0700
Avoid apiary submission of job graph when it is not needed. (#15458)
---
sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index bbaf52c..29a6016 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -508,6 +508,13 @@ class DataflowRunner(PipelineRunner):
# in the proto representation of the graph.
pipeline.replace_all(DataflowRunner._NON_PORTABLE_PTRANSFORM_OVERRIDES)
+ # Always upload graph out-of-band when explicitly using runner v2 with
+ # use_portable_job_submission to avoid irrelevant large graph limits.
+ if (apiclient._use_unified_worker(debug_options) and
+ debug_options.lookup_experiment('use_portable_job_submission') and
+ not debug_options.lookup_experiment('upload_graph')):
+ debug_options.add_experiment("upload_graph")
+
# Add setup_options for all the BeamPlugin imports
setup_options = options.view_as(SetupOptions)
plugins = BeamPlugin.get_all_plugin_paths()
