See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8223/display/redirect?page=changes>
Changes:
[suztomo] [BEAM-11205] Upgrading the Libraries BOM to v22
[randomstep] [BEAM-12708] Bump arrow-memory-netty
[Etienne Chauchot] [BEAM-12153] implement GroupByKey with CombinePerKey with
Concatenate
[Etienne Chauchot] [BEAM-11023] Increase memory in SS Validates runner tests to
avoid OOM
[vincent.marquez] [BEAM-9008] adds CassandraIO.readAll
[Etienne Chauchot] [BEAM-12727] extract Concatenate CombineFn to runner-core
module to
------------------------------------------
[...truncated 106.91 KB...]
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:a473c406743cb7591019c31447723dc6653411bc50012d5924cd0c658e6d5501
Associated tags:
- 20210908-181402535020906
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210908-181402535020906
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210908-181402535020906].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:a473c406743cb7591019c31447723dc6653411bc50012d5924cd0c658e6d5501].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:7654282e6c115d1167497a3c81ecc6329569f4808c74756f58b630c9d687fb40
Associated tags:
- f9e99158-4879-4862-a454-2dfdc7bd7dd0
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:f9e99158-4879-4862-a454-2dfdc7bd7dd0].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:7654282e6c115d1167497a3c81ecc6329569f4808c74756f58b630c9d687fb40].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:f345a68da934c930bff5c9b5c6c6cd20148740da0b4ff32278533dddc0c862b6
Associated tags:
- dc872069-7062-4916-a902-b4a6a2dc64fe
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:dc872069-7062-4916-a902-b4a6a2dc64fe].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:f345a68da934c930bff5c9b5c6c6cd20148740da0b4ff32278533dddc0c862b6].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:b5ecc72f00c6ceb44eeec602b35687eef10b807972af1cfd39135ce307479200
Associated tags:
- 20210908-181342167085740
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210908-181342167085740
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210908-181342167085740].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:b5ecc72f00c6ceb44eeec602b35687eef10b807972af1cfd39135ce307479200].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:6f82245ac88116bad19036690d32e45045b53bfc860acbc1fe76f077e8ea2d8a
Associated tags:
- bf453b67-abae-45ef-b45a-e5ef7b5d5bea
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:bf453b67-abae-45ef-b45a-e5ef7b5d5bea].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:6f82245ac88116bad19036690d32e45045b53bfc860acbc1fe76f077e8ea2d8a].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:129181354d76f51693c43c8a042ff0a173bf360329193b5128c89b1208a77ac1
Associated tags:
- f3d41e0e-6b03-4764-985f-40d327d61cc9
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:f3d41e0e-6b03-4764-985f-40d327d61cc9].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:129181354d76f51693c43c8a042ff0a173bf360329193b5128c89b1208a77ac1].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer FAILED
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:5d0346705bea3e1669f447c2755861e663ba23ee3e32c6f80b713f91739884b1
Associated tags:
- 20210908-181343292677458
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20210908-181343292677458
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20210908-181343292677458].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:5d0346705bea3e1669f447c2755861e663ba23ee3e32c6f80b713f91739884b1].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:22ee81f24c376dce357e2ab6b1d5ed5398d177819f2599ca6b63bbe501732fb1
Associated tags:
- 16efa03a-61ed-48ad-8b67-33b0ad0b1f42
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:16efa03a-61ed-48ad-8b67-33b0ad0b1f42].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:22ee81f24c376dce357e2ab6b1d5ed5398d177819f2599ca6b63bbe501732fb1].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:8acd39d901ad4e10734373da015ed289818b7fdbb83eb74f9fcfc726633fd5dd
Associated tags:
- beab0ad6-5773-4c96-934c-616b96dd3d9f
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:beab0ad6-5773-4c96-934c-616b96dd3d9f].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:8acd39d901ad4e10734373da015ed289818b7fdbb83eb74f9fcfc726633fd5dd].
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 276
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py36:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug
option to get more log output. Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See
https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 45m 35s
44 actionable tasks: 40 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/pmd7zif5v5ol2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
