See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8232/display/redirect?page=changes>
Changes:
[noreply] Port changes from Pub/Sub Lite to beam (#15418)
[heejong] [BEAM-12805] Fix XLang CombinePerKey test by explicitly assigning the
[BenWhitehead] [BEAM-8376] Google Cloud Firestore Connector - Add handling for
[noreply] Decreasing peak memory usage for beam.TupleCombineFn (#15494)
[noreply] [BEAM-12802] Add support for prefetch through data layers down through
[noreply] [BEAM-11097] Add implementation of side input cache (#15483)
------------------------------------------
[...truncated 117.17 KB...]
apache_beam/runners/dataflow/test_dataflow_runner.py:65: in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <DataflowPipelineResult <Job
clientRequestId: '20210911002907951086-7153'
createTime: '2021-09-11T00:29:09.493383Z'
...021-09-11T00:29:09.493383Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)> at 0x7fec863b42b0>
duration = None
def wait_until_finish(self, duration=None):
if not self.is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
# use thread.join() to wait for the polling thread.
thread.daemon = True
thread.start()
while thread.is_alive():
time.sleep(5.0)
# TODO: Merge the termination code in poll_for_job_completion and
# is_in_terminal_state.
terminated = self.is_in_terminal_state()
assert duration or terminated, (
'Job did not reach to a terminal state after waiting indefinitely.')
if terminated and self.state != PipelineState.DONE:
# TODO(BEAM-1290): Consider converting this to an error log based on
# theresolution of the issue.
> raise DataflowRuntimeException(
'Dataflow pipeline failed. State: %s, Error:\n%s' %
(self.state, getattr(self._runner, 'last_error_msg', None)),
self)
E
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow
pipeline failed. State: FAILED, Error:
E Workflow failed.
apache_beam/runners/dataflow/dataflow_runner.py:1632: DataflowRuntimeException
______________ ExerciseMetricsPipelineTest.test_metrics_fnapi_it _______________
[gw0] linux -- Python 3.8.5
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/bin/python3.8>
self =
<apache_beam.runners.dataflow.dataflow_exercise_metrics_pipeline_test.ExerciseMetricsPipelineTest
testMethod=test_metrics_fnapi_it>
@pytest.mark.it_postcommit
@pytest.mark.it_validatescontainer
def test_metrics_fnapi_it(self):
> result = self.run_pipeline(experiment='beam_fn_api')
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py:56:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py:43: in
run_pipeline
return dataflow_exercise_metrics_pipeline.apply_and_run(p)
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline.py:176: in
apply_and_run
result = pipeline.run()
apache_beam/pipeline.py:539: in run
return Pipeline.from_runner_api(
apache_beam/pipeline.py:566: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:65: in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <DataflowPipelineResult <Job
clientRequestId: '20210911003050192824-1145'
createTime: '2021-09-11T00:30:51.423013Z'
...021-09-11T00:30:51.423013Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)> at 0x7fec8fa45e80>
duration = None
def wait_until_finish(self, duration=None):
if not self.is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
# use thread.join() to wait for the polling thread.
thread.daemon = True
thread.start()
while thread.is_alive():
time.sleep(5.0)
# TODO: Merge the termination code in poll_for_job_completion and
# is_in_terminal_state.
terminated = self.is_in_terminal_state()
assert duration or terminated, (
'Job did not reach to a terminal state after waiting indefinitely.')
if terminated and self.state != PipelineState.DONE:
# TODO(BEAM-1290): Consider converting this to an error log based on
# theresolution of the issue.
> raise DataflowRuntimeException(
'Dataflow pipeline failed. State: %s, Error:\n%s' %
(self.state, getattr(self._runner, 'last_error_msg', None)),
self)
E
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow
pipeline failed. State: FAILED, Error:
E Workflow failed.
apache_beam/runners/dataflow/dataflow_runner.py:1632: DataflowRuntimeException
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.8_sdk.xml>
-
============== 3 failed, 1 passed, 3 warnings in 1037.64 seconds ===============
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210911-001238518901961
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:9b8b2415fea99c6c2522257a09dfdccc579407e300c324a99fe20a713b66cc23
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:d9c69f16-4445-4caf-8a8c-537f40c54f3a
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:4869bea8a39734edd1d301b1e67c4b9d78f109be3f892c8055985e07319f64e9
Deleted: sha256:cd4bc9ba385ad1db439c8f433510b138a39145525a681932719db73fcececbc2
Deleted: sha256:1ec5539dc175f15e59df2d0485b653e864ec08ba454bde4ee3fb241d83905843
Deleted: sha256:6c08810988e19af2d506d456c94d6d5495b5291c17b86eed56488468f30d5b44
Deleted: sha256:027b1440360b14bd7a9208d0a3e620591edf7ea56573abf8f1d0cf46a2322b71
Deleted: sha256:e8b90a221c5cedf21ad25bef7920d53449f2060dadd14d94b963152b25555093
Deleted: sha256:b08a1a1d8c9fea7b5fbd4d2a607e9c8ceedcbf1606330a8d5937aaea9dade966
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:9b8b2415fea99c6c2522257a09dfdccc579407e300c324a99fe20a713b66cc23
Associated tags:
- 20210911-001238518901961
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210911-001238518901961
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210911-001238518901961].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:9b8b2415fea99c6c2522257a09dfdccc579407e300c324a99fe20a713b66cc23].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:4869bea8a39734edd1d301b1e67c4b9d78f109be3f892c8055985e07319f64e9
Associated tags:
- d9c69f16-4445-4caf-8a8c-537f40c54f3a
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:d9c69f16-4445-4caf-8a8c-537f40c54f3a].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:4869bea8a39734edd1d301b1e67c4b9d78f109be3f892c8055985e07319f64e9].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:97a9cf9bef483a09678cc0a9c355e32b0c7337275485c0f20a718b7b8bcaaec3
Associated tags:
- 64122b35-9b27-4c0d-8213-8868570ef8a3
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:64122b35-9b27-4c0d-8213-8868570ef8a3].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:97a9cf9bef483a09678cc0a9c355e32b0c7337275485c0f20a718b7b8bcaaec3].
Removed the container
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:3ee6ffb5e118ca444d6a83f85c726fcfef0036c1a5237f797482acfe9f935eec
Associated tags:
- 20210911-001229867151344
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210911-001229867151344
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210911-001229867151344].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:3ee6ffb5e118ca444d6a83f85c726fcfef0036c1a5237f797482acfe9f935eec].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:84ef1662660d37b7f1705b398654a3fa878fd878590a2e9b1c55102a380ea711
Associated tags:
- d372e381-13b6-43e2-990c-7d6caee0df51
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:d372e381-13b6-43e2-990c-7d6caee0df51].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:84ef1662660d37b7f1705b398654a3fa878fd878590a2e9b1c55102a380ea711].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:1d2b0ee8c25227f7d39412f17b47f6c4871a4175e534bcf47b5c2b5c40f51681
Associated tags:
- efb71066-402f-4734-af28-5b2613d944ad
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:efb71066-402f-4734-af28-5b2613d944ad].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:1d2b0ee8c25227f7d39412f17b47f6c4871a4175e534bcf47b5c2b5c40f51681].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 41m 54s
44 actionable tasks: 40 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/44eyrl5i3urf4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
