See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8392/display/redirect?page=changes>
Changes:
[noreply] Add -XX:+AlwaysActAsServerClassMachine to Java SDK container
[moritz] adhoc: Minor update to flink runner docs
[noreply] [BEAM-11087] Add default WindowMappingFn from Main to Side Input
------------------------------------------
[...truncated 107.77 KB...]
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:596: in __exit__
self.result = self.run()
apache_beam/pipeline.py:549: in run
self._options).run(False)
apache_beam/pipeline.py:573: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:53: in run_pipeline
self.result = super().run_pipeline(pipeline, options)
apache_beam/runners/dataflow/dataflow_runner.py:459: in run_pipeline
environments.DockerEnvironment.from_options(options))
apache_beam/transforms/environments.py:341: in from_options
options)
apache_beam/runners/portability/sdk_container_builder.py:139: in
build_container_image
return builder._build()
apache_beam/runners/portability/sdk_container_builder.py:90: in _build
self._invoke_docker_build_and_push(container_image_name)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
<apache_beam.runners.portability.sdk_container_builder._SdkContainerImageCloudBuilder
object at 0x7f92d144d898>
container_image_name =
'us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:e2f67b5e-24b8-469f-a15d-acea93cdc2cd'
def _invoke_docker_build_and_push(self, container_image_name):
project_id = self._google_cloud_options.project
temp_location = self._google_cloud_options.temp_location
# google cloud build service expects all the build source file to be
# compressed into a tarball.
tarball_path = os.path.join(self._temp_src_dir, '%s.tgz' % SOURCE_FOLDER)
self._make_tarfile(tarball_path, self._temp_src_dir)
_LOGGER.info(
"Compressed source files for building sdk container at %s" %
tarball_path)
container_image_tag = container_image_name.split(':')[-1]
gcs_location = os.path.join(
temp_location, '%s-%s.tgz' % (SOURCE_FOLDER, container_image_tag))
self._upload_to_gcs(tarball_path, gcs_location)
build = cloudbuild.Build()
build.steps = []
step = cloudbuild.BuildStep()
step.name = 'gcr.io/kaniko-project/executor:latest'
step.args = ['--destination=' + container_image_name, '--cache=true']
step.dir = SOURCE_FOLDER
build.steps.append(step)
source = cloudbuild.Source()
source.storageSource = cloudbuild.StorageSource()
gcs_bucket, gcs_object = self._get_gcs_bucket_and_name(gcs_location)
source.storageSource.bucket = os.path.join(gcs_bucket)
source.storageSource.object = gcs_object
build.source = source
# TODO(zyichi): make timeout configurable
build.timeout = '7200s'
now = time.time()
# operation = client.create_build(project_id=project_id, build=build)
request = cloudbuild.CloudbuildProjectsBuildsCreateRequest(
projectId=project_id, build=build)
build = self._cloudbuild_client.projects_builds.Create(request)
build_id, log_url = self._get_cloud_build_id_and_log_url(build.metadata)
_LOGGER.info(
'Building sdk container with Google Cloud Build, this may '
'take a few minutes, you may check build log at %s' % log_url)
# block until build finish, if build fails exception will be raised and
# stops the job submission.
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
while response.status in [cloudbuild.Build.StatusValueValuesEnum.QUEUED,
cloudbuild.Build.StatusValueValuesEnum.WORKING]:
time.sleep(10)
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
if response.status != cloudbuild.Build.StatusValueValuesEnum.SUCCESS:
raise RuntimeError(
> 'Failed to build python sdk container image on google cloud build, '
'please check build log for error.')
E RuntimeError: Failed to build python sdk container image on google
cloud build, please check build log for error.
apache_beam/runners/portability/sdk_container_builder.py:287: RuntimeError
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.6_sdk.xml>
-
============== 1 failed, 3 passed, 2 warnings in 1334.24 seconds ===============
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211021-001451800227461
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:dc7b9e77676ce47d971b9e81ef8f924d788be2103b33a21b60cc18ecf96c623f
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:4fb644b8-2069-494f-a04c-8c6dad229f5c
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:66275e7a79b0df79be5da10cb0940782651efdff836a68781a8084460d51b47a
Deleted: sha256:29cee70b7562cfb75f5255795aa443a6376a4f79752b5318cf963f4ae06861b3
Deleted: sha256:aa86b7fbef3ae25f28de29e99464950eef200c3db2c3ee14c5005175bf44b933
Deleted: sha256:d7ec18a0f7b66201ffd9745b38cc88f22bf7c6be7091c33faabf7634b69a2a25
Deleted: sha256:02e68957410a568611ffba851de63cd30a62721c0a339438efae5ceb9185fc4a
Deleted: sha256:b7d393d4d5699dc0a3ef812eaf7467b8c5c7c5e22477ea2a73fc6f2ee19f2eb6
Deleted: sha256:a85a5450dddf9fb6f1b7e565413d56aa532a60dfbc3c1759d703dfa723a768cb
Deleted: sha256:5073dd409401da66c317fbdee4695893bce1dec63ae05c686c98db61fe930d35
Deleted: sha256:bf4570e632dd73a1ddb8f2e05982304b65da87ff701308f4df2cb6f3713fbdb2
Deleted: sha256:e89e61120d46ac7ebdd9a87fdea6467e1b09630c5abe18053cfa13a023479a65
Deleted: sha256:96c67165877a3371a91210d676a6af5a03ca489bf4cf3ab07d479d00217cfaa4
Deleted: sha256:99544ec40510735cf8f4e162a327d8793072fc5e11ef2ca3440da820437c0e79
Deleted: sha256:1245954d4efd978d1173697d166aa8fdb4bca7f583807b4f1230109bb421e0f6
Deleted: sha256:fe08d06265490d6032cea70ea4ff894ec98924303b3abc978dd3eafcddf0ba97
Deleted: sha256:5454582e6c021e40c722b3a444beca517d5c3f0656e950ba4cd12db08087705a
Deleted: sha256:4f23ccb9177c6c2749d125d08788882460836765d5b24ef4e0d94ff9c0da33ef
Deleted: sha256:46ebbd63bcecf46d2abda1cf20c0cd75896f0af833428828cdca2051af124f14
Deleted: sha256:05221f797d6651876e58503b9dadfdb6918dbd6d0eefa2f4405e234cab7fd027
Deleted: sha256:dd1b186e5ee3de679dd53a50552f8180e4403580053283037eae886a1d6e59f2
Deleted: sha256:d4ee4f6eb5980cb899a953d3dd3fa37738d215c47ac4f2e0965cc5957237470c
Deleted: sha256:3aa46e9a4434fbf76eaa5d884b31de3cd2a9dbd07748110309b03e5d0ce2e656
Deleted: sha256:ebbb056b1d2e68f3e8b7f9d2bfd7372a68dda6aee8932b90bd3dcb574bce9784
Deleted: sha256:467414b38a38ac567ad4c895437b4081b21eb1d2254074cf307bfd7153ed5ad4
Deleted: sha256:472b130e65e5847654ab4876438b434b91588d445baa346ef2b7768d1a4527b5
Deleted: sha256:7ba745204aa7d8252f1cfcff03aca14714a1b5419032b88ec09333a6a66164bd
Deleted: sha256:96b7dea0848a491f6834e2b375d2b98cdc7fb68507dc84d3041c2cee20c6b4ef
Deleted: sha256:8639918e0c3667eb2694d8af919fa6c2b0e0321746c7c285ff19710d72ab5e77
Deleted: sha256:ce7d224044c8ab812b224a17b295bef3df7c3135e63fd2bb0c7ba427cf9df65d
Deleted: sha256:250318c0e46f4295735ca690b53036d95372cec9c1c43cf6921e32297d4796fb
Deleted: sha256:5925e173bb7f066e907c4af6ca165dde900c3c4acc9f2bdd9a4bd723013bdd76
Deleted: sha256:0d4276002b9cdf44daaa6e6800e9be1ad91d6abe4a343ed2becd4b578bd82df1
Deleted: sha256:74a90f9df849405243f1269f567b2f10ac0ec00f03e2e53842a12e0ed19c6659
Deleted: sha256:88a25aaed60a391c03f722124e62adb161bd0fcb871755c40f527cea5bc373a5
Deleted: sha256:2c2b6d664ea6ba95613fb5859d5445270266dd5ec3f5604cf1c71e8cad41d7e2
Deleted: sha256:87434988d5a533d22456cdafb37bcb6ef13e412ad93aa9f50f833784775646cb
Deleted: sha256:5a7490a9efced16971b824d1e1b72442b2267f60abbed4c491d9d5560c4ef780
Deleted: sha256:3fb20cb6820c9566fd45dbb0e1088ce70f889e1437ecfdd6ced547afef75a1f7
Deleted: sha256:2e683ddf22fb6779c10e1fce877759ad23aa5272e85dedf95323554988b01cbc
Deleted: sha256:22ee425af80694ac0e24df48d4531edbbfeedfc078e734a6587ded5093304859
Deleted: sha256:8a872c2b1d018dcad0cb5e4a2a7ff30f00a7dfe7f631b1a76bce51fb4715fc56
Deleted: sha256:0d85e84feb056b9a1a8f38e8e2faf5b7392ab2fca281fda781d8456fcf29f9a3
Deleted: sha256:8b121d6d876180616ccb02902b702d0c369dd198d951781e3068d4e347f697ef
Deleted: sha256:5758f95252161d743be082b567ff891557e00a092edb5b56ea0625fa87f7568e
Deleted: sha256:d3c94c6630b7363d0bb098bbb623c979fae8494bbc9249c95634ea21668af816
Deleted: sha256:8012b544a210c57f7ba00125446f52d63dd3de7174a4704ac23057a7ec091edf
Deleted: sha256:6d93dfb8e196a97dacdbf1fd20a3277199205ef3eb54108e97da1e8de44a0c5c
Deleted: sha256:ac030888dc9652c075dd4bbe81fd5311268e5630e4215786ff27fceec326f628
Deleted: sha256:895a5f3f248d12895509034ec93d39286d5ec8440e98166575ca591e5d5c4200
Deleted: sha256:93f7ceb1b7902ad9d7de4b48923862b073c88c14c2aab7cc7845a765fac96d0f
Deleted: sha256:0057117badf77f6cb5ee04292fba13ef37588cb0e96d66fe75245a488296c2bd
Deleted: sha256:5e1482191b308284dcd99aa335ebf97d2f134f98fb2c997fad6be3c28f5b4e45
Deleted: sha256:68a668c40e42e442f5433d70947113fd6fcc4ef96450ba474b0b2a8d0e240f9e
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:dc7b9e77676ce47d971b9e81ef8f924d788be2103b33a21b60cc18ecf96c623f
Associated tags:
- 20211021-001451800227461
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211021-001451800227461
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211021-001451800227461].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:dc7b9e77676ce47d971b9e81ef8f924d788be2103b33a21b60cc18ecf96c623f].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:66275e7a79b0df79be5da10cb0940782651efdff836a68781a8084460d51b47a
Associated tags:
- 4fb644b8-2069-494f-a04c-8c6dad229f5c
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:4fb644b8-2069-494f-a04c-8c6dad229f5c].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:66275e7a79b0df79be5da10cb0940782651efdff836a68781a8084460d51b47a].
Removed the container
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:3d8dbde23db1720b65ba896e5a26dd5926859413942536c3189850ab8312eddf
Associated tags:
- 20211021-001500841703341
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211021-001500841703341
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211021-001500841703341].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:3d8dbde23db1720b65ba896e5a26dd5926859413942536c3189850ab8312eddf].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:d4cd4787d6c45daa38c2941c87656a8e271cb595e4f15f3eb08643ff3419dbf8
Associated tags:
- 504f546f-8580-457c-9101-bbe3c78bff45
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:504f546f-8580-457c-9101-bbe3c78bff45].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:d4cd4787d6c45daa38c2941c87656a8e271cb595e4f15f3eb08643ff3419dbf8].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.9.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 39m 22s
44 actionable tasks: 40 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/idixklddsua7m
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
