liferoad opened a new pull request, #35776:
URL: https://github.com/apache/beam/pull/35776
The `DirectPipelineTemplateTest` used a hardcoded GCS path for the
`template_location`. This can cause test flakiness due to collisions between
concurrent test runs or leftover artifacts from previous runs.
This change uses `time.time()` to generate a unique template path for each
test execution, ensuring test isolation and improving reliability.
This error is strange. Try to use the random location first.
```
2025-08-03T09:19:40.8190919Z
apache_beam/runners/dataflow/dataflow_job_service_test.py:76:
2025-08-03T09:19:40.8191574Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2025-08-03T09:19:40.8192169Z apache_beam/pipeline.py:663: in __exit__
2025-08-03T09:19:40.8192839Z self.result.wait_until_finish()
2025-08-03T09:19:40.8193650Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2025-08-03T09:19:40.8193917Z
2025-08-03T09:19:40.8194208Z self =
<apache_beam.runners.portability.portable_runner.PipelineResult object at
0x7c4117c3f4f0>
2025-08-03T09:19:40.8194651Z duration = None
2025-08-03T09:19:40.8194783Z
2025-08-03T09:19:40.8194906Z def wait_until_finish(self, duration=None):
2025-08-03T09:19:40.8195177Z """
2025-08-03T09:19:40.8195487Z :param duration: The maximum time in
milliseconds to wait for the result of
2025-08-03T09:19:40.8195980Z the execution. If None or zero, will wait
until the pipeline finishes.
2025-08-03T09:19:40.8196391Z :return: The result of the pipeline, i.e.
PipelineResult.
2025-08-03T09:19:40.8196693Z """
2025-08-03T09:19:40.8196919Z def read_messages() -> None:
2025-08-03T09:19:40.8197183Z previous_state = -1
2025-08-03T09:19:40.8197436Z for message in self._message_stream:
2025-08-03T09:19:40.8197855Z if
message.HasField('message_response'):
2025-08-03T09:19:40.8198163Z logging.log(
2025-08-03T09:19:40.8198680Z
MESSAGE_LOG_LEVELS[message.message_response.importance],
2025-08-03T09:19:40.8199263Z "%s",
2025-08-03T09:19:40.8199689Z
message.message_response.message_text)
2025-08-03T09:19:40.8200139Z else:
2025-08-03T09:19:40.8200389Z current_state =
message.state_response.state
2025-08-03T09:19:40.8200707Z if current_state != previous_state:
2025-08-03T09:19:40.8200983Z _LOGGER.info(
2025-08-03T09:19:40.8201220Z "Job state changed to %s",
2025-08-03T09:19:40.8201623Z
self.runner_api_state_to_pipeline_state(current_state))
2025-08-03T09:19:40.8202455Z previous_state = current_state
2025-08-03T09:19:40.8202979Z self._messages.append(message)
2025-08-03T09:19:40.8203394Z
2025-08-03T09:19:40.8203711Z message_thread = threading.Thread(
2025-08-03T09:19:40.8204042Z target=read_messages,
name='wait_until_finish_read')
2025-08-03T09:19:40.8204524Z message_thread.daemon = True
2025-08-03T09:19:40.8204990Z message_thread.start()
2025-08-03T09:19:40.8205382Z
2025-08-03T09:19:40.8205683Z if duration:
2025-08-03T09:19:40.8206044Z state_thread = threading.Thread(
2025-08-03T09:19:40.8206563Z
target=functools.partial(self._observe_state, message_thread),
2025-08-03T09:19:40.8207243Z
name='wait_until_finish_state_observer')
2025-08-03T09:19:40.8207736Z state_thread.daemon = True
2025-08-03T09:19:40.8208161Z state_thread.start()
2025-08-03T09:19:40.8208588Z start_time = time.time()
2025-08-03T09:19:40.8209054Z duration_secs = duration / 1000
2025-08-03T09:19:40.8209566Z while (time.time() - start_time <
duration_secs and
2025-08-03T09:19:40.8209937Z state_thread.is_alive()):
2025-08-03T09:19:40.8210472Z time.sleep(1)
2025-08-03T09:19:40.8210860Z else:
2025-08-03T09:19:40.8211214Z self._observe_state(message_thread)
2025-08-03T09:19:40.8211661Z
2025-08-03T09:19:40.8211972Z if self._runtime_exception:
2025-08-03T09:19:40.8212590Z > raise self._runtime_exception
2025-08-03T09:19:40.8213553Z E RuntimeError: Pipeline
job-581f8b10-982c-4182-abc3-224280c22e09 failed in state FAILED: Error running
pipeline.
2025-08-03T09:19:40.8214299Z E Traceback (most recent call last):
2025-08-03T09:19:40.8215548Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/portability/local_job_service.py",
line 294, in _run_job
2025-08-03T09:19:40.8216715Z E self.result = self._invoke_runner()
2025-08-03T09:19:40.8218060Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/dataflow_job_service.py",
line 36, in _invoke_runner
2025-08-03T09:19:40.8219422Z E self.result = runner.run_pipeline(
2025-08-03T09:19:40.8220606Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",
line 507, in run_pipeline
2025-08-03T09:19:40.8221997Z E
self.dataflow_client.create_job(self.job), self, options)
2025-08-03T09:19:40.8223399Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/utils/retry.py",
line 300, in wrapper
2025-08-03T09:19:40.8224501Z E return fun(*args, **kwargs)
2025-08-03T09:19:40.8225778Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",
line 737, in create_job
2025-08-03T09:19:40.8227092Z E self.create_job_description(job)
2025-08-03T09:19:40.8228317Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",
line 837, in create_job_description
2025-08-03T09:19:40.8229285Z E job.proto.environment = Environment(
2025-08-03T09:19:40.8230604Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",
line 289, in __init__
2025-08-03T09:19:40.8231907Z E items = [item.get_dict() for item
in dd.items]
2025-08-03T09:19:40.8233469Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",
line 289, in <listcomp>
2025-08-03T09:19:40.8234845Z E items = [item.get_dict() for item
in dd.items]
2025-08-03T09:19:40.8236003Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/transforms/display.py",
line 381, in get_dict
2025-08-03T09:19:40.8237126Z E self.is_valid()
2025-08-03T09:19:40.8238214Z E File
"/runner/_work/beam/beam/sdks/python/test-suites/tox/py39/build/srcs/sdks/python/apache_beam/transforms/display.py",
line 344, in is_valid
2025-08-03T09:19:40.8239471Z E raise ValueError(
2025-08-03T09:19:40.8240804Z E ValueError: Invalid DisplayDataItem
DisplayDataItem({"key": "pot_vp_arg1", "namespace":
"apache_beam.options.pipeline_options.PipelineOptions", "type": "STRING",
"value": null}). Value must not be None
2025-08-03T09:19:40.8242045Z
2025-08-03T09:19:40.8242561Z
apache_beam/runners/portability/portable_runner.py:568: RuntimeError
```
------------------------
Thank you for your contribution! Follow this checklist to help us
incorporate your contribution quickly and easily:
- [ ] Mention the appropriate issue in your description (for example:
`addresses #123`), if applicable. This will automatically add a link to the
pull request in the issue. If you would like the issue to automatically close
on merging the pull request, comment `fixes #<ISSUE NUMBER>` instead.
- [ ] Update `CHANGES.md` with noteworthy changes.
- [ ] If this contribution is large, please file an Apache [Individual
Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
See the [Contributor Guide](https://beam.apache.org/contribute) for more
tips on [how to make review process
smoother](https://github.com/apache/beam/blob/master/CONTRIBUTING.md#make-the-reviewers-job-easier).
To check the build health, please visit
[https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md](https://github.com/apache/beam/blob/master/.test-infra/BUILD_STATUS.md)
GitHub Actions Tests Status (on master branch)
------------------------------------------------------------------------------------------------
[](https://github.com/apache/beam/actions?query=workflow%3A%22Build+python+source+distribution+and+wheels%22+branch%3Amaster+event%3Aschedule)
[](https://github.com/apache/beam/actions?query=workflow%3A%22Python+Tests%22+branch%3Amaster+event%3Aschedule)
[](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule)
[](https://github.com/apache/beam/actions?query=workflow%3A%22Go+tests%22+branch%3Amaster+event%3Aschedule)
See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more
information about GitHub Actions CI or the [workflows
README](https://github.com/apache/beam/blob/master/.github/workflows/README.md)
to see a list of phrases to trigger workflows.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
