See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/561/display/redirect>
Changes:
------------------------------------------
[...truncated 2.26 MB...]
PROJECT=${PROJECT:-apache-beam-testing}
REGION=${REGION:-us-central1}
IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut
-d'=' -f2)"
SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)"
PY_VERSION=$1
ARCH=${3:-"x86"}
IMAGE_NAME="${IMAGE_PREFIX}python${PY_VERSION}_sdk"
CONTAINER_PROJECT="sdks:python:container:py${PY_VERSION//.}" # Note: we
substitute away the dot in the version.
PY_INTERPRETER="python${PY_VERSION}"
TEST_SUITE_TAG="it_validatescontainer"
XUNIT_FILE="pytest-$IMAGE_NAME.xml"
# Verify in the root of the repository
test -d sdks/python/container
# Verify docker and gcloud commands exist
command -v docker
command -v gcloud
docker -v
gcloud -v
TAG=$(date +%Y%m%d-%H%M%S%N)
CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk
echo "Using container $CONTAINER"
echo "Using CPU architecture $ARCH"
if [[ "$ARCH" == "x86" ]]; then
# Verify docker image has been built.
docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
# Tag the docker container.
docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
# Push the container
gcloud docker -- push $CONTAINER:$TAG
elif [[ "$ARCH" == "ARM" ]]; then
# Note: ARM test suites only run on github actions, where multi-arch Python
SDK containers are already pushed during build.
# Reset the test suite tag to run ARM pipelines.
TEST_SUITE_TAG="it_dataflow_arm"
# Reset the multi-arch Python SDK container image tag.
TAG=$MULTIARCH_TAG
else
printf "Please give a valid CPU architecture, either x86 or ARM."
exit 1
fi
WARNING: `gcloud docker` will not be supported for Docker client versions above
18.03.
As an alternative, use `gcloud auth configure-docker` to configure `docker` to
use `gcloud` as a credential helper, then use `docker` as you would for non-GCR
registries, e.g. `docker pull gcr.io/project-id/my-image`. Add
`--verbosity=error` to silence this warning: `gcloud docker
--verbosity=error -- pull gcr.io/project-id/my-image`.
See:
https://cloud.google.com/container-registry/docs/support/deprecation-notices#gcloud-docker
> Task :sdks:python:test-suites:dataflow:py311:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Built container image was not removed.
Possibly, it was not not saved locally."
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
# Note: we don't delete the multi-arch containers here because this command
only deletes the manifest list with the tag,
# the associated container images can't be deleted because they are not
tagged. However, multi-arch containers that are
# older than 6 weeks old are deleted by
stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
if [[ "$ARCH" == "x86" ]]; then
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
fi
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o log_cli=True -o log_level=Info -o junit_suite_name=$IMAGE_NAME \
-m=$TEST_SUITE_TAG \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Built container image was not removed.
Possibly, it was not not saved locally."
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
# Note: we don't delete the multi-arch containers here because this command
only deletes the manifest list with the tag,
# the associated container images can't be deleted because they are not
tagged. However, multi-arch containers that are
# older than 6 weeks old are deleted by
stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
if [[ "$ARCH" == "x86" ]]; then
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
fi
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o log_cli=True -o log_level=Info -o junit_suite_name=$IMAGE_NAME \
-m=$TEST_SUITE_TAG \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use
sha256 directly.
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:bd320cc22dd62df7fe128a68c9538404d18cb4c0de1c694ac6d8301bb20ccbf7
Associated tags:
- 20230902-082516851394449
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20230902-082516851394449
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20230902-082516851394449].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:bd320cc22dd62df7fe128a68c9538404d18cb4c0de1c694ac6d8301bb20ccbf7].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:64d55dd64cfb5d99c68ea39a70f2b4348cce848b891056fe076175a69abeae3f
Associated tags:
- 647bc738-e785-4954-8161-20a685bb5668
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:647bc738-e785-4954-8161-20a685bb5668].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:64d55dd64cfb5d99c68ea39a70f2b4348cce848b891056fe076175a69abeae3f].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:52ceb16bdd9d3a292e2704fac9cc97b99c0c8b30e4cb998659c2f5022f51fb28
Associated tags:
- 99c1437a-3653-4eec-8639-5f5f7f2e82df
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:99c1437a-3653-4eec-8639-5f5f7f2e82df].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:52ceb16bdd9d3a292e2704fac9cc97b99c0c8b30e4cb998659c2f5022f51fb28].
> Task :sdks:python:test-suites:dataflow:py311:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use
sha256 directly.
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.11_sdk@sha256:6fa06cf37a6ebc6960921eefcfed199e091513fb955417e523501a6e8feaec5a
Associated tags:
- 20230902-082458603706971
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.11_sdk:20230902-082458603706971
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.11_sdk:20230902-082458603706971].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.11_sdk@sha256:6fa06cf37a6ebc6960921eefcfed199e091513fb955417e523501a6e8feaec5a].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk@sha256:09e71178e687bad69a3c68307a6877daea268f49bc3217220259524aa3413f9c
Associated tags:
- f05d1e09-c95d-4e58-83cc-70880bf29f1a
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk:f05d1e09-c95d-4e58-83cc-70880bf29f1a].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk@sha256:09e71178e687bad69a3c68307a6877daea268f49bc3217220259524aa3413f9c].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk@sha256:43e623458cf75dea72904d8ae0ba79c6f87948d1479050a06a075cd77f8a6d6a
Associated tags:
- 52a6c37f-b4d5-409a-8ad0-101bab5ec439
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk:52a6c37f-b4d5-409a-8ad0-101bab5ec439].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python311_sdk/beam_python_prebuilt_sdk@sha256:43e623458cf75dea72904d8ae0ba79c6f87948d1479050a06a075cd77f8a6d6a].
FAILURE: Build completed with 2 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/ws/src/sdks/python/container/common.gradle'>
line: 38
* What went wrong:
Execution failed for task
':sdks:python:container:py310:generatePythonRequirements'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont_with_RC/ws/src/sdks/python/container/common.gradle'>
line: 38
* What went wrong:
Execution failed for task
':sdks:python:container:py39:generatePythonRequirements'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings
and determine if they come from your own scripts or plugins.
See
https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 42m 43s
64 actionable tasks: 53 executed, 4 from cache, 7 up-to-date
Publishing build scan...
https://ge.apache.org/s/g5poqfkxayh7e
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
