See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8408/display/redirect>
Changes:
------------------------------------------
[...truncated 109.30 KB...]
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:596: in __exit__
self.result = self.run()
apache_beam/pipeline.py:549: in run
self._options).run(False)
apache_beam/pipeline.py:573: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:53: in run_pipeline
self.result = super().run_pipeline(pipeline, options)
apache_beam/runners/dataflow/dataflow_runner.py:459: in run_pipeline
environments.DockerEnvironment.from_options(options))
apache_beam/transforms/environments.py:341: in from_options
options)
apache_beam/runners/portability/sdk_container_builder.py:139: in
build_container_image
return builder._build()
apache_beam/runners/portability/sdk_container_builder.py:90: in _build
self._invoke_docker_build_and_push(container_image_name)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
<apache_beam.runners.portability.sdk_container_builder._SdkContainerImageCloudBuilder
object at 0x7f4f4df9b860>
container_image_name =
'us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:e15e21be-7a6e-493b-bc1c-4b3c5a81ce87'
def _invoke_docker_build_and_push(self, container_image_name):
project_id = self._google_cloud_options.project
temp_location = self._google_cloud_options.temp_location
# google cloud build service expects all the build source file to be
# compressed into a tarball.
tarball_path = os.path.join(self._temp_src_dir, '%s.tgz' % SOURCE_FOLDER)
self._make_tarfile(tarball_path, self._temp_src_dir)
_LOGGER.info(
"Compressed source files for building sdk container at %s" %
tarball_path)
container_image_tag = container_image_name.split(':')[-1]
gcs_location = os.path.join(
temp_location, '%s-%s.tgz' % (SOURCE_FOLDER, container_image_tag))
self._upload_to_gcs(tarball_path, gcs_location)
build = cloudbuild.Build()
build.steps = []
step = cloudbuild.BuildStep()
step.name = 'gcr.io/kaniko-project/executor:latest'
step.args = ['--destination=' + container_image_name, '--cache=true']
step.dir = SOURCE_FOLDER
build.steps.append(step)
source = cloudbuild.Source()
source.storageSource = cloudbuild.StorageSource()
gcs_bucket, gcs_object = self._get_gcs_bucket_and_name(gcs_location)
source.storageSource.bucket = os.path.join(gcs_bucket)
source.storageSource.object = gcs_object
build.source = source
# TODO(zyichi): make timeout configurable
build.timeout = '7200s'
now = time.time()
# operation = client.create_build(project_id=project_id, build=build)
request = cloudbuild.CloudbuildProjectsBuildsCreateRequest(
projectId=project_id, build=build)
build = self._cloudbuild_client.projects_builds.Create(request)
build_id, log_url = self._get_cloud_build_id_and_log_url(build.metadata)
_LOGGER.info(
'Building sdk container with Google Cloud Build, this may '
'take a few minutes, you may check build log at %s' % log_url)
# block until build finish, if build fails exception will be raised and
# stops the job submission.
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
while response.status in [cloudbuild.Build.StatusValueValuesEnum.QUEUED,
cloudbuild.Build.StatusValueValuesEnum.WORKING]:
time.sleep(10)
response = self._cloudbuild_client.projects_builds.Get(
cloudbuild.CloudbuildProjectsBuildsGetRequest(
id=build_id, projectId=project_id))
if response.status != cloudbuild.Build.StatusValueValuesEnum.SUCCESS:
raise RuntimeError(
> 'Failed to build python sdk container image on google cloud build, '
'please check build log for error.')
E RuntimeError: Failed to build python sdk container image on google
cloud build, please check build log for error.
apache_beam/runners/portability/sdk_container_builder.py:287: RuntimeError
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.6_sdk.xml>
-
============== 1 failed, 3 passed, 2 warnings in 1334.18 seconds ===============
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-001435638137091
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:d98a01856491734dd754433b805bbee9df0c5adc01ea0841b11525d54a9f6137
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:e3e1e3ea-f68f-4fd7-b2eb-c95f86373bdd
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:421c3997f60426643e9bcf9eacf27c3f67518a88ee383d1594c7345484280d8e
Deleted: sha256:2adce77e608e7a3539084ac9d3a4d2593451c4005b95a69702ca3ce4299f827e
Deleted: sha256:927f52d81686396a5508cd6dcda80bc4ac9638bff16c5c91bbac81022dfd27b3
Deleted: sha256:0476e1fed49d2d23bc88678882b500f5dc8750e19a08ac95a8cc947a81057a61
Deleted: sha256:74d834371eeb87e41bb44f402cf67e52171ec0c0b2edc7cce1647d8e9eee4087
Deleted: sha256:d1c2f2fde3c815e616452b0bd0bf9fa044427341a668b66c674140865d940678
Deleted: sha256:2aae5ef82736df20122db4ab353062e1a126bb5ef0751924df8f6f3f38c1e4fb
Deleted: sha256:94b8c011d17db8885079bd8ccc6dde304398954065f4e19a8f7e279691cde245
Deleted: sha256:925fe825d2efbceb7985054efae1f143d16b09f5bb11e3f5d4cf136e0e0798bd
Deleted: sha256:91c01eae51cfc7d99bb41d6689dd0e5aa5bfa21e3cd071fe8f973382869175e0
Deleted: sha256:ba9e64b73e198896398a51b0d8711090627f6dbaa8d2483c21a985ccbc37ec46
Deleted: sha256:3e9f90ce6a6f1608c4f0e8afb0e1c7a0a73b9f653926080a2984df8f2bb0869f
Deleted: sha256:641de73974dab55eb174bedb70acf7527f548e9746b85ae5dcaf0cd695da3d6e
Deleted: sha256:32ff0536045f0d9b2c8125632632537e4f243cc4ddc6464431c5e45aa11d837e
Deleted: sha256:5713402ccea9f6fcef05021b9ac0d17a977ca6c4073c441ff29c857f30278791
Deleted: sha256:a84cade2e54cd5feb59b93b34ea9a83ca55cc40d7655d02a413a2799c8215b79
Deleted: sha256:73fcb2ea93dc76dcfcada42433938fd797995607e4b14d25009c74635156c49a
Deleted: sha256:f66e41e19f89cda7594bf969e6c369030688e71616eee5039e097786fcfe8d07
Deleted: sha256:f1ddd7209ef2852816b7f5e86a3e734b06382fa995fdb591ece35752f71f1e2e
Deleted: sha256:c5081b04579acafaff65f1945c8965a4c4f03af2fd4ee9049b3fbbbad03ebc8f
Deleted: sha256:38154c46d5320850bf3be8376dfddadcd2cd03152309fc0c1f4b795c17c05725
Deleted: sha256:d0df386ee60453eb2b75ad73a0e9998cc05184a6c49fb9debb1ebb25240917f8
Deleted: sha256:4ec8250888cd306f073ff86e4c118ab08d123fea2c901a7dc86176a466dc57ea
Deleted: sha256:2507bff5cd5813c9266967f4fcd8f6e593fdb1356f34969ee39c579b377eff3f
Deleted: sha256:2914e5f517b3b73947846c9415a4e39649d01e4f47ae030c8f42aea3906dd7a7
Deleted: sha256:d3fce9a9e7e85dd852ec42e0e5e212921da8e49c9276ddbca0df2be5d942ccd3
Deleted: sha256:0b877a14f6d37e9d13a0a4a6e40dd9c6bbc14fb36476bc9fd1dffc7e08aa8d31
Deleted: sha256:254742466ff364b731dcf3271672830075944a83c2513136347d46c2bb72da43
Deleted: sha256:f8dc1e80b2d467bc45152e925b6335d0ed45a31bbbba85c406bb49d6024d687a
Deleted: sha256:79710723acbe304459ac00264f8d772562c13c3a6b9f3226d7095f83a2f5cfaa
Deleted: sha256:11de1f767f62eca79b7b1d4bc0ead7e12fb60767a657dbf627d3966d5b84acc7
Deleted: sha256:5deea6284accedc9b6ac74a4048c2443c7399bb38786e7716e2bd9a1f5d0dac2
Deleted: sha256:53d60757e1983ce6739115735ccec5cf20bb8d6149e0e47c79048345d6053ca9
Deleted: sha256:03fefc7203807ab7212316c629add623737e7d2fe55fe4b1bf7f123b46af2935
Deleted: sha256:8084495e2ea83680cef59a045d3a0408d8e9cb2b374a16aeebd8314089232ace
Deleted: sha256:8e15534237b80d91d9f88637dddbc5a4ed526af502cc7f6bec6b65f2b10aec45
Deleted: sha256:cc43cc8e99c164c2137cbd6453e7eee041deb93a834ae9bc6e9a72067462b637
Deleted: sha256:fc6b82d69cf82205a7c0414ad2ccd3004c01ff316bcf60120069e315aecdc485
Deleted: sha256:5fa0b7a85c2a67b6a20516331459db36c2cbf29fa669f0ef7522bad45f8cb743
Deleted: sha256:7364810012d05758f93b1a0fc5da48cf900dca4a0ffcb0266d26cc4b8b86a9f3
Deleted: sha256:a08c1787f36857d30bbf422a0684f416c505e2f1f6e002277f6baf2a966cff42
Deleted: sha256:ee401d83ddaf4541be43aaa9320e0e1b8683afb4ac12c4ba8bdeddfcfc0ec3e3
Deleted: sha256:f70e7f9c5489544fef90ddd02113baefa8950b233a22c348f3f6474db138130a
Deleted: sha256:c65f989bc0f86546e158a39c775bb5b25840925f5c0d9f5af785f1d5577e9c20
Deleted: sha256:b1442eaba94a0cbe34ea65a8e9a679a0458631b0a0262f5c91aa34ff9b80fa3a
Deleted: sha256:a34cada4b79c0a81d1870e01a8c8e5f8d7d76be9787bc080ac1b53c5ed5e55ee
Deleted: sha256:68f03157f72d068b901cc58e795b33074fa29ac05b0910920b9714d2aec0f665
Deleted: sha256:0d6b921b74e973ef076e853416dae031dee95e951e7475f6cd66839a5e71146c
Deleted: sha256:5e6163200959f9059472d7d329452a263805cc1fee68cefd08f29c4fae14d8aa
Deleted: sha256:11ce500b65d316935aea0fd02747dec2abebbc080d124ee32242fe4c78e3e2af
Deleted: sha256:a8141ceabbaa183de4370e1dc0a2049dce49448c256cadd101523269d49d3559
Deleted: sha256:8c957a65822a764befbb42c5d06675eb3ed9f184a321829684f6c9a485093aa7
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:d98a01856491734dd754433b805bbee9df0c5adc01ea0841b11525d54a9f6137
Associated tags:
- 20211025-001435638137091
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-001435638137091
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-001435638137091].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:d98a01856491734dd754433b805bbee9df0c5adc01ea0841b11525d54a9f6137].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:421c3997f60426643e9bcf9eacf27c3f67518a88ee383d1594c7345484280d8e
Associated tags:
- e3e1e3ea-f68f-4fd7-b2eb-c95f86373bdd
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:e3e1e3ea-f68f-4fd7-b2eb-c95f86373bdd].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:421c3997f60426643e9bcf9eacf27c3f67518a88ee383d1594c7345484280d8e].
Removed the container
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:021ed179c16acc81ad5cdb7e8f9e305270b9928434ac7fe4a08af62dcbbf6454
Associated tags:
- 20211025-001435631922794
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-001435631922794
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-001435631922794].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:021ed179c16acc81ad5cdb7e8f9e305270b9928434ac7fe4a08af62dcbbf6454].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:685440d5a4f14b08d3ae401d5f2f61aa89af29b659c9e3c7abba8af5f92f3028
Associated tags:
- 8adeac96-3f8c-4470-91a4-734dccfe5f39
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:8adeac96-3f8c-4470-91a4-734dccfe5f39].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:685440d5a4f14b08d3ae401d5f2f61aa89af29b659c9e3c7abba8af5f92f3028].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.9.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 38m 51s
44 actionable tasks: 40 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/zmt2nnjhdyfiu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
