See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8409/display/redirect?page=changes>
Changes:
[noreply] Fixing BigQueryIO request too big corner case for streaming inserts
------------------------------------------
[...truncated 107.57 KB...]
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:596: in __exit__
self.result = self.run()
apache_beam/pipeline.py:549: in run
self._options).run(False)
apache_beam/pipeline.py:573: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:53: in run_pipeline
self.result = super().run_pipeline(pipeline, options)
apache_beam/runners/dataflow/dataflow_runner.py:459: in run_pipeline
environments.DockerEnvironment.from_options(options))
apache_beam/transforms/environments.py:341: in from_options
options)
apache_beam/runners/portability/sdk_container_builder.py:139: in
build_container_image
return builder._build()
apache_beam/runners/portability/sdk_container_builder.py:90: in _build
self._invoke_docker_build_and_push(container_image_name)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
<apache_beam.runners.portability.sdk_container_builder._SdkContainerImageCloudBuilder
object at 0x7fa3a3f06ac8>
container_image_name =
'us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b609f279-e064-43cb-9250-ef397b61136b'
def _invoke_docker_build_and_push(self, container_image_name):
project_id = self._google_cloud_options.project
temp_location = self._google_cloud_options.temp_location
# google cloud build service expects all the build source file to be
# compressed into a tarball.
tarball_path = os.path.join(self._temp_src_dir, '%s.tgz' % SOURCE_FOLDER)
self._make_tarfile(tarball_path, self._temp_src_dir)
_LOGGER.info(
"Compressed source files for building sdk container at %s" %
tarball_path)
container_image_tag = container_image_name.split(':')[-1]
gcs_location = os.path.join(
temp_location, '%s-%s.tgz' % (SOURCE_FOLDER, container_image_tag))
self._upload_to_gcs(tarball_path, gcs_location)
build = cloudbuild.Build()
build.steps = []
step = cloudbuild.BuildStep()
step.name = 'gcr.io/kaniko-project/executor:latest'
step.args = ['--destination=' + container_image_name, '--cache=true']
step.dir = SOURCE_FOLDER
build.steps.append(step)
source = cloudbuild.Source()
source.storageSource = cloudbuild.StorageSource()
gcs_bucket, gcs_object = self._get_gcs_bucket_and_name(gcs_location)
source.storageSource.bucket = os.path.join(gcs_bucket)
source.storageSource.object = gcs_object
build.source = source
# TODO(zyichi): make timeout configurable
build.timeout = '7200s'
now = time.time()
# operation = client.create_build(project_id=project_id, build=build)
request = cloudbuild.CloudbuildProjectsBuildsCreateRequest(
projectId=project_id, build=build)
build = self._cloudbuild_client.projects_builds.Create(request)
build_id, log_url = self._get_cloud_build_id_and_log_url(build.metadata)
_LOGGER.info(
'Building sdk container with Google Cloud Build, this may '
'take a few minutes, you may check build log at %s' % log_url)
# block until build finish, if build fails exception will be raised and
# stops the job submission.
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
while response.status in [cloudbuild.Build.StatusValueValuesEnum.QUEUED,
cloudbuild.Build.StatusValueValuesEnum.WORKING]:
time.sleep(10)
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
if response.status != cloudbuild.Build.StatusValueValuesEnum.SUCCESS:
raise RuntimeError(
> 'Failed to build python sdk container image on google cloud build, '
'please check build log for error.')
E RuntimeError: Failed to build python sdk container image on google
cloud build, please check build log for error.
apache_beam/runners/portability/sdk_container_builder.py:287: RuntimeError
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.6_sdk.xml>
-
============== 1 failed, 3 passed, 2 warnings in 1289.29 seconds ===============
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2
Deleted: sha256:58a933b867c4bc145172b32a115f559fef771de8fd4a0f511b2a023732d217b4
Deleted: sha256:2ae2c82ddd71fcbec3c53ab3fa4d7113313a14a6f6b46c088aaf41075a8e179b
Deleted: sha256:ac845b2efdbdb3daa0f1f4b4c60552cf8d50cce17ecf99fbcd52e10628fef9aa
Deleted: sha256:e31486d2e4ab63b6edb402393c71e3a3a02f6cf236627fc79a10150b5e6fdb0f
Deleted: sha256:eef27c289ab59fbfa6c01fea1e4a61486f4e34e2681c6efd3ef1db41d0492d10
Deleted: sha256:8de510415ba224ee4e3628842d8064ef42b3d88b8c72dcd98e033ffa9d836cc1
Deleted: sha256:fb0f8f90fa58c30a50466ccc101ae24f7b7105a0415b7afd3c5d022e9cb16f21
Deleted: sha256:f4cbe33a20fd0149616cdf6518e2a4774ba115a051c8cc3da77d6b64d5b2e2ac
Deleted: sha256:93a04db611c1f5829f61633445ba5eed5942879509d2514576091f4c61b38ccb
Deleted: sha256:5ffe31c4f43689f319e6333c0c333dc6145db61ddf9e83a1b4a27d38331510e5
Deleted: sha256:012cce032ec48f10c8e082a196ed976ce1022b4ae5ee075226856c74c1d21f5d
Deleted: sha256:eec6fec9974dce6af41fe7c96290cf9088ec36146b7b4ca2e99b188546e6d319
Deleted: sha256:ef2499c318a1f39f2314c80b16aab1123d95060b56271fefc23ccf7f8ac489a6
Deleted: sha256:c689a6059be9a3a948c4ed1c0e11835dbc910ba66e38f06a289bc8bb3b14b922
Deleted: sha256:c1feb7fb6ddd8139c8e81f22c16abb25a205f963097ea23dd64c501e6008b0e7
Deleted: sha256:e556c76801c604985f19b0d8fe35dd6f45c1208912241f6af1bb7aa2ad92cc8e
Deleted: sha256:cc12d3e966b6daa5944d6b62a5d048afcbe0f6667e6c994c9551e69bd4b92b39
Deleted: sha256:34d4f957c6d1e046481926beff2f9e6fc4feada0b4245e4e0a98031e4ab9a7e9
Deleted: sha256:264c1df281ecb757a35f637a81cdfbfbcb8fa981ecd19c22da38b24181171aa9
Deleted: sha256:62a41178107bb957aa525c75695125b13b51313a5b9bb03e6126dc07037b616c
Deleted: sha256:13fbb4fb296a844e2fed7d88aac5a98b0a544897f873460dfaea499c8eb423c9
Deleted: sha256:32ffd91993c4679c32721590a5159e435e01bab675140ac6b96e286a6eab1889
Deleted: sha256:471cc209d726bc5f4ce9d6d6b5cbdd5c8182cbcc41c49c2eade519ae1839046f
Deleted: sha256:67bfd5936f89a4163a96afc0eb16889fd23943251fba7c56951db1a8004eb20e
Deleted: sha256:ff81648956dd8cb2719ad258356c729726eb0d0924850a0ef819064262c44f4b
Deleted: sha256:52a7446c14d9639982952fb51ead52a6f4f7b15208845d69fe014c62899f26b7
Deleted: sha256:d78812954dbc54d0f3082d91f4aaa3efc59903fdfcd9d8cee546b19dd61798aa
Deleted: sha256:bba01d4564aeb007b39661cfce7cdf35dc21c2adb87cdff57ce777d608993721
Deleted: sha256:19633dbbdffea59a5b39061bc358e0b03f1ed2ae3867bc43d85c8a7f18d42188
Deleted: sha256:a9e2617cc6cd62f79819cb30acf0d945512d18e1a7a706205571b094078ac94a
Deleted: sha256:66e900f6c20f09aa21e24c2dd33d16a5817fda3c185db88a04f6d936619915f6
Deleted: sha256:c83907b7931a0cd71ca583fc3fb071ef808392f72300f5bb2b5688822a424fcc
Deleted: sha256:e5cc5545015eaf3aa35b37381242ecb500d6514b22896946644772aa10f756ee
Deleted: sha256:90550931db8dc570b8af8ba48c111839aafc4fd72966bca85788457fd4e9c0d7
Deleted: sha256:c810ddbd23f9e20a159d23fc1dc2467cc1ba913680ef4e53462770eed4b4b8a6
Deleted: sha256:6adfd84cace6c7e7dd77ba0eeff45e9cd9ec884e440f883a3d2f0966967638d1
Deleted: sha256:c1c570ab24ac3e61c553c58cf4a928b450641075a331928885f190c01759b9cf
Deleted: sha256:9e346c6cbe103d825180f719fd6cf0d42d737775070d4ef48c14ca982e7598d1
Deleted: sha256:0013106ea2956b2d90ce4cefa8e9c447211b4cc90c663426f5ff0a12dd8c0964
Deleted: sha256:5c6173857940106c1ab44794175a52af096e309fb5dac5c41698e05be1a352ac
Deleted: sha256:7d3f42cec70bac9f328b14032fe293769c939148c5f776beffb609aaa4880ca2
Deleted: sha256:b8765545b48b3d3d6dc19de1d17c47977ac3a9ddcf8d40dce830e8b965670688
Deleted: sha256:2039edd92e901e0a91b84435941a54ff51b0ec21586243ee287d0be06d929e87
Deleted: sha256:d84f0fef9a3027f285322b67fc04aee338fb63d502fe9a7936ff3d7721d756dc
Deleted: sha256:bb709dd4c8607895a550918c052dfaaf227060bae8dc9edcf03438680a4a14f1
Deleted: sha256:37060df8fd153bdb8061473a5a1a07fa7144889a16497b2c1bff98c1a7fece7d
Deleted: sha256:1a9e2de5e13509507800438c244680ba8ab99512c86aa064377926b702e08f23
Deleted: sha256:04c73ac1856ad7a11aab5f93ddb448ab2b9c9a3ebcdc9ce4ada2557d5aa8b13d
Deleted: sha256:7bcd503128fbcd517bd0d38437b19495876c7eb381243f745b80a357cc495acc
Deleted: sha256:f5db4263ab95397234be1eec802f346e57b703b67b13fc038bdb88f66eaff3bc
Deleted: sha256:c24700b71dacece76dbeeb09a89db9f76e4b911abe568708dfbc57e0992cbeea
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598
Associated tags:
- 20211025-061538659902022
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2
Associated tags:
- b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2].
Removed the container
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:994b6641cac1c2ad112c6ee4a9807fc0a3158f6b0a35ac87f178d20e9b3af616
Associated tags:
- 20211025-061552046074833
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-061552046074833
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-061552046074833].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:994b6641cac1c2ad112c6ee4a9807fc0a3158f6b0a35ac87f178d20e9b3af616].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a9bcb06737376adb88027fca36ee663cf540daee3ef6bbcb4ed54c166ffdb823
Associated tags:
- 008fc315-df02-4ae5-9547-6ab7647326f0
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:008fc315-df02-4ae5-9547-6ab7647326f0].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a9bcb06737376adb88027fca36ee663cf540daee3ef6bbcb4ed54c166ffdb823].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.9.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 40m 17s
44 actionable tasks: 40 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/bpd7s775z7l2e
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
