See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8969/display/redirect?page=changes>
Changes:
[ryanthompson591] fixed typo in typehints
[zyichi] Remove unused prebuild_sdk_container_base_iamge option from validate
[noreply] [BEAM-13888] Add unit testing to ioutilx (#17058)
[noreply] Merge pull request #16822 from [BEAM-13841][Playground] Add
Application
[noreply] Minor: Make serializableCoder warning gramatically correct english
[noreply] [BEAM-14091] Fixing Interactive Beam show/collect for remote runners
------------------------------------------
[...truncated 1.59 MB...]
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:49cd9826ba6c2f5fa5dfada4cda38e40e4f63850661a1150d94891b02985b32b
Associated tags:
- 20220317-002806389983652
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20220317-002806389983652
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20220317-002806389983652].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:49cd9826ba6c2f5fa5dfada4cda38e40e4f63850661a1150d94891b02985b32b].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:a4292f74546507f757dee08c365e3bd73f4f427149628107f3a8b6b06fc1143a
Associated tags:
- 20220317-002806199320419
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20220317-002806199320419
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20220317-002806199320419].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:a4292f74546507f757dee08c365e3bd73f4f427149628107f3a8b6b06fc1143a].
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:4c0620db58e037101c94831826b074e413c673062e26760722684efc184e9777
Associated tags:
- 9e67eef1-8dff-4af0-9803-e91e1515f907
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:9e67eef1-8dff-4af0-9803-e91e1515f907].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:4c0620db58e037101c94831826b074e413c673062e26760722684efc184e9777].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:69763a80793c0c1521a91d4ae701da966cae00a4e93a3d501a341d6a70a8a487
Associated tags:
- 07d0c600-9097-4900-8512-27d1125bad14
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:07d0c600-9097-4900-8512-27d1125bad14].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:69763a80793c0c1521a91d4ae701da966cae00a4e93a3d501a341d6a70a8a487].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:c4226e4857b11e1eec0efb044eb732b0c687b5382982661919560c28b84d72d2
Associated tags:
- cf231698-edbe-41b4-9d7a-dc5b3e77b6e3
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:cf231698-edbe-41b4-9d7a-dc5b3e77b6e3].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:c4226e4857b11e1eec0efb044eb732b0c687b5382982661919560c28b84d72d2].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:6b1ecb9a5f6bc7346e0e47f8aedbdfbce0d07b4df2be008f8198e63e36bbed09
Associated tags:
- b4c6d47f-d7a1-4d3c-996a-1bf13d113635
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:b4c6d47f-d7a1-4d3c-996a-1bf13d113635].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:6b1ecb9a5f6bc7346e0e47f8aedbdfbce0d07b4df2be008f8198e63e36bbed09].
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:59c44570512bebc0f7e1ac2aa6db3a72d29bd14372a27dc0dbead7bdf137249a
Associated tags:
- 20220317-002806384504135
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20220317-002806384504135
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20220317-002806384504135].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:59c44570512bebc0f7e1ac2aa6db3a72d29bd14372a27dc0dbead7bdf137249a].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3264c994409a75ca53d56c44f122267f72c55aacf6df68384b00138bd6f591c9
Associated tags:
- bdc0d5c4-4d48-4cf0-a403-7ff7dd57fe67
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:bdc0d5c4-4d48-4cf0-a403-7ff7dd57fe67].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3264c994409a75ca53d56c44f122267f72c55aacf6df68384b00138bd6f591c9].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:bce6f82daefee9f8822f6c700dbd57364f8a157968e58381c7434849f9163122
Associated tags:
- 2cea5561-7b9b-4dfe-8925-42287e2cd0df
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:2cea5561-7b9b-4dfe-8925-42287e2cd0df].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:bce6f82daefee9f8822f6c700dbd57364f8a157968e58381c7434849f9163122].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
[gw0] [100%] PASSED
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
=================================== FAILURES ===================================
____ WordCountIT.test_wordcount_it_with_prebuilt_sdk_container_cloud_build _____
[gw0] linux -- Python 3.7.12
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967052/bin/python3.7>
self = <apache_beam.examples.wordcount_it_test.WordCountIT
testMethod=test_wordcount_it_with_prebuilt_sdk_container_cloud_build>
@pytest.mark.it_validatescontainer
def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self):
self._run_wordcount_it(
wordcount.run,
experiment='beam_fn_api',
> prebuild_sdk_container_engine='cloud_build')
apache_beam/examples/wordcount_it_test.py:67:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam/examples/wordcount_it_test.py:109: in _run_wordcount_it
save_main_session=False,
apache_beam/examples/wordcount.py:100: in run
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:596: in __exit__
self.result = self.run()
apache_beam/pipeline.py:549: in run
self._options).run(False)
apache_beam/pipeline.py:573: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:64: in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <DataflowPipelineResult <Job
clientRequestId: '20220317004447874521-9986'
createTime: '2022-03-17T00:44:50.846654Z'
...022-03-17T00:44:50.846654Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)> at 0x7fc63ba7d310>
duration = None
def wait_until_finish(self, duration=None):
if not self.is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
# use thread.join() to wait for the polling thread.
thread.daemon = True
thread.start()
while thread.is_alive():
> time.sleep(5.0)
E Failed: Timeout >900.0s
apache_beam/runners/dataflow/dataflow_runner.py:1625: Failed
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.7_sdk.xml>
-
============== 1 failed, 3 passed, 2 warnings in 2218.69 seconds ===============
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20220317-002806223727270
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:a469a1b023bba41f2c543d063764e334b6fc7689451e0d7d4c0a52b656c60374
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:347c081d-5038-44b2-b850-057c35bcafc4
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:ac2a7b86e6cfaf9e1c00c89087bac3a44c212819d9ac9c3e7490310eb6d5f4d8
Deleted: sha256:34120837a69dc1539fa56cb320384a00fd967bd6712b5233598296171c757b2a
Deleted: sha256:f8b7a62ead52ae9d6b9a168184d415d2a1d2f39932b42d4b4971423793dccbc1
Deleted: sha256:e82dab8a94b573fb852d7f39938bf65417494efe288d1e2fec5e76e9985a2a90
Deleted: sha256:de2f7d1dff787250be15d7caba9f96168e40a21c357fc606243aebf8043c186c
Deleted: sha256:992ee08fe17f2c4e8c5797012bfbc9fd7d970051e8824e87c277eccbdf9f966e
Deleted: sha256:ab5539c8d333adaa50e5672d11969e87a1ca55ffae3acf02d2d30d55a8911e3f
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:a469a1b023bba41f2c543d063764e334b6fc7689451e0d7d4c0a52b656c60374
Associated tags:
- 20220317-002806223727270
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20220317-002806223727270
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20220317-002806223727270].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:a469a1b023bba41f2c543d063764e334b6fc7689451e0d7d4c0a52b656c60374].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:ac2a7b86e6cfaf9e1c00c89087bac3a44c212819d9ac9c3e7490310eb6d5f4d8
Associated tags:
- 347c081d-5038-44b2-b850-057c35bcafc4
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:347c081d-5038-44b2-b850-057c35bcafc4].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:ac2a7b86e6cfaf9e1c00c89087bac3a44c212819d9ac9c3e7490310eb6d5f4d8].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3d633a0a1669d80d7b51f53494b477cd2a65e370e1d4c56866386fdee6017730
Associated tags:
- d0b0b9d6-1005-4ebb-9f9a-f1403c89cf39
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:d0b0b9d6-1005-4ebb-9f9a-f1403c89cf39].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3d633a0a1669d80d7b51f53494b477cd2a65e370e1d4c56866386fdee6017730].
Removed the container
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer FAILED
FAILURE: Build failed with an exception.
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 338
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 56m 34s
62 actionable tasks: 51 executed, 4 from cache, 7 up-to-date
Publishing build scan...
https://gradle.com/s/pomlrgezac4w4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
