claudevdm commented on code in PR #39161:
URL: https://github.com/apache/beam/pull/39161#discussion_r3544465749
##########
sdks/python/apache_beam/runners/interactive/recording_manager.py:
##########
@@ -820,17 +830,16 @@ def _wait_for_dependencies(
len(computing_deps),
computing_deps.keys())
- futures_to_wait = list(
- set(comp._future for comp in computing_deps.values()))
+ results_to_wait = list(set(comp for comp in computing_deps.values()))
try:
- for i, future in enumerate(futures_to_wait):
+ for i, comp in enumerate(results_to_wait):
if async_result:
async_result.update_display(
- f'Waiting for dependency {i + 1}/{len(futures_to_wait)}...',
- progress=0.05 + 0.05 * (i / len(futures_to_wait)),
+ f'Waiting for dependency {i + 1}/{len(results_to_wait)}...',
+ progress=0.05 + 0.05 * (i / len(results_to_wait)),
)
- future.result()
+ comp.wait_for_completion()
Review Comment:
Should we check the result of the future completion here? What should happen
if it failed?
##########
sdks/python/apache_beam/runners/interactive/recording_manager.py:
##########
@@ -710,19 +718,13 @@ def task():
return async_result
def _get_pipeline_graph(self):
- """Lazily initializes and returns the PipelineGraph."""
- if self._pipeline_graph is None:
- try:
- # Try to create the graph.
- self._pipeline_graph = PipelineGraph(self.user_pipeline)
- except (ImportError, NameError, AttributeError):
- # If pydot is missing, PipelineGraph() might crash.
- _LOGGER.warning(
- "Could not create PipelineGraph (pydot missing?). " \
- "Async features disabled."
- )
- self._pipeline_graph = None
- return self._pipeline_graph
+ """Initializes and returns the PipelineGraph."""
+ try:
+ # Try to create the graph.
+ return PipelineGraph(self.user_pipeline)
+ except (ImportError, NameError, AttributeError):
+ # If pydot is missing, PipelineGraph() might crash.
+ return None
Review Comment:
Add logging here?
##########
sdks/python/apache_beam/runners/interactive/recording_manager.py:
##########
@@ -151,27 +152,33 @@ def exception(self, timeout=None):
except TimeoutError:
return None
- def _on_done(self, future: Future):
- self._env.unmark_pcollection_computing(self._pcolls)
- self._recording_manager._async_computations.pop(self._display_id, None)
-
- if future.cancelled():
- self.update_display('Computation Cancelled.', 1.0)
- return
+ def wait_for_completion(self):
+ self._completed_event.wait()
Review Comment:
Add a timeout here?
##########
sdks/python/apache_beam/runners/interactive/interactive_beam_test.py:
##########
@@ -853,12 +853,12 @@ def test_compute_dependency_wait_true(self):
spy_wait.assert_called_with({pcoll2}, async_res2)
# Let pcoll1 finish
- async_res1.result(timeout=60)
+ async_res1.wait_for_completion()
self.assertTrue(pcoll1 in self.env.computed_pcollections)
self.assertFalse(self.env.is_pcollection_computing(pcoll1))
# pcoll2 should now run and complete
- async_res2.result(timeout=60)
+ async_res2.wait_for_completion()
Review Comment:
Should test_compute_dependency_wait_false also be updated?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
