See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/9137/display/redirect?page=changes>
Changes:
[noreply] [BEAM-13983] Small changes to sklearn runinference (#17459)
[chamikaramj] Renames ExternalPythonTransform to PythonExternalTransform
------------------------------------------
[...truncated 91.35 KB...]
/usr/bin/docker
/usr/bin/gcloud
docker -v
Docker version 18.09.4, build d14af54
gcloud -v
Google Cloud SDK 335.0.0
alpha 2021.04.02
beta 2021.04.02
bq 2.0.66
core 2021.04.02
gsutil 4.60
# Verify docker image has been built.
docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
apache/beam_python3.9_sdk
2.39.0.dev 0356a390a5c0 3 seconds ago 3.9GB
TAG=$(date +%Y%m%d-%H%M%S%N)
CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk
Using container us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk
echo "Using container $CONTAINER"
# Tag the docker container.
docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
# Push the container.
gcloud docker -- push $CONTAINER:$TAG
WARNING: `gcloud docker` will not be supported for Docker client versions above
18.03.
As an alternative, use `gcloud auth configure-docker` to configure `docker` to
use `gcloud` as a credential helper, then use `docker` as you would for non-GCR
registries, e.g. `docker pull gcr.io/project-id/my-image`. Add
`--verbosity=error` to silence this warning: `gcloud docker
--verbosity=error -- pull gcr.io/project-id/my-image`.
See:
https://cloud.google.com/container-registry/docs/support/deprecation-notices#gcloud-docker
The push refers to repository
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk]
e386ada84e20: Preparing
1136c4af8298: Preparing
d7076b05c0d8: Preparing
6b59765dfc6c: Preparing
b1cc842bde9d: Preparing
7ea61334ae97: Preparing
7f2a414bdb19: Preparing
297aa7a25c34: Preparing
d64cc20dc6b3: Preparing
c1429d5f2bd2: Preparing
9b261ec5e2c9: Preparing
688e10d43bbe: Preparing
28bb340ff43d: Preparing
80e102589f50: Preparing
d7076b05c0d8: Waiting
425914ff8d5c: Preparing
105e55456415: Preparing
7e8c58e9679c: Preparing
6b59765dfc6c: Waiting
9c26176f6534: Preparing
7ea61334ae97: Waiting
b1cc842bde9d: Waiting
6d95196cbe50: Preparing
5bdcc8e2060c: Preparing
7f2a414bdb19: Waiting
e386ada84e20: Waiting
08fa02ce37eb: Preparing
a037458de4e0: Preparing
bafdbe68e4ae: Preparing
297aa7a25c34: Waiting
1136c4af8298: Waiting
a13c519c6361: Preparing
d64cc20dc6b3: Waiting
28bb340ff43d: Waiting
80e102589f50: Waiting
425914ff8d5c: Waiting
5bdcc8e2060c: Waiting
688e10d43bbe: Waiting
a13c519c6361: Waiting
c1429d5f2bd2: Waiting
105e55456415: Waiting
9b261ec5e2c9: Waiting
6d95196cbe50: Waiting
08fa02ce37eb: Waiting
a037458de4e0: Waiting
9c26176f6534: Waiting
7e8c58e9679c: Waiting
bafdbe68e4ae: Waiting
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:0aebe370b9be3a4bb436a7a416a0f8e272dc22b9bb2c23c7bd52babd43310112
Associated tags:
- 20220428-045210252011936
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20220428-045210252011936
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20220428-045210252011936].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:0aebe370b9be3a4bb436a7a416a0f8e272dc22b9bb2c23c7bd52babd43310112].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3aad235f42f6caa2e5e829e1e985220dc830108e57136b7596d8f3378b1e99a7
Associated tags:
- 66b7d57b-07b8-4f59-a2f6-b2bf25224cdc
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:66b7d57b-07b8-4f59-a2f6-b2bf25224cdc].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3aad235f42f6caa2e5e829e1e985220dc830108e57136b7596d8f3378b1e99a7].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:7eba063731161f9383756b22ba255905eaaadf74f6aed87525dd79950d7688f2
Associated tags:
- d4201b7f-351b-4d03-a2f0-6c8495898149
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:d4201b7f-351b-4d03-a2f0-6c8495898149].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:7eba063731161f9383756b22ba255905eaaadf74f6aed87525dd79950d7688f2].
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:319578e65c122717040a5811056548780118a12c1e9510f6a56948f91f142dc9
Associated tags:
- 20220428-045210507033480
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20220428-045210507033480
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20220428-045210507033480].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:319578e65c122717040a5811056548780118a12c1e9510f6a56948f91f142dc9].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:d89c954262f05b9f19723b23105bbd01434d530695d2bb036821f6fff84667c5
Associated tags:
- 78ca03f2-bc1f-4859-b4f8-ce630dd108f8
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:78ca03f2-bc1f-4859-b4f8-ce630dd108f8].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:d89c954262f05b9f19723b23105bbd01434d530695d2bb036821f6fff84667c5].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:f21a8fe048d8edfb4364967ab1f75c4ee8f0d9119ce6e8761c984eb37f4b787e
Associated tags:
- d79203c9-2f01-42a1-96e9-590fcf23b459
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:d79203c9-2f01-42a1-96e9-590fcf23b459].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:f21a8fe048d8edfb4364967ab1f75c4ee8f0d9119ce6e8761c984eb37f4b787e].
FAILURE: Build failed with an exception.
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 359
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 39m 57s
51 actionable tasks: 41 executed, 4 from cache, 6 up-to-date
Publishing build scan...
https://gradle.com/s/zxpej6xzdt5po
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
