See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8762/display/redirect?page=changes>
Changes:
[tuyarer] [BEAM-13577] Beam Select's uniquifyNames function loses nullability of
[Steve Niemitz] [BEAM-13689] Output token elements when BQ batch writes
complete.
[noreply] [BEAM-13699] Replace fnv with maphash. (#16573)
[noreply] [BEAM-13693] Bump
[noreply] [BEAM-10206] Remove Fatalf calls in non-test goroutines for
------------------------------------------
[...truncated 86.53 KB...]
PY_INTERPRETER="python3.6" # Use this in virtualenv command.
elif [[ $1 == "python37" ]]; then
IMAGE_NAME="${IMAGE_PREFIX}python3.7_sdk" # Use this to create
CONTAINER_IMAGE variable.
CONTAINER_PROJECT="sdks:python:container:py37" # Use this to build container
by Gradle.
PY_INTERPRETER="python3.7" # Use this in virtualenv command.
elif [[ $1 == "python38" ]]; then
IMAGE_NAME="${IMAGE_PREFIX}python3.8_sdk" # Use this to create
CONTAINER_IMAGE variable.
CONTAINER_PROJECT="sdks:python:container:py38" # Use this to build container
by Gradle.
PY_INTERPRETER="python3.8" # Use this in virtualenv command.
else
echo "Must set Python version with one of 'python36', 'python37' and
'python38' from commandline."
exit 1
fi
XUNIT_FILE="pytest-$IMAGE_NAME.xml"
# Verify in the root of the repository
test -d sdks/python/container
# Verify docker and gcloud commands exist
command -v docker
command -v gcloud
docker -v
gcloud -v
# Verify docker image has been built.
docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
TAG=$(date +%Y%m%d-%H%M%S%N)
date +%Y%m%d-%H%M%S%N
CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_$1_sdk
echo "Using container $CONTAINER"
# Tag the docker container.
docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
# Push the container.
gcloud docker -- push $CONTAINER:$TAG
WARNING: `gcloud docker` will not be supported for Docker client versions above
18.03.
As an alternative, use `gcloud auth configure-docker` to configure `docker` to
use `gcloud` as a credential helper, then use `docker` as you would for non-GCR
registries, e.g. `docker pull gcr.io/project-id/my-image`. Add
`--verbosity=error` to silence this warning: `gcloud docker
--verbosity=error -- pull gcr.io/project-id/my-image`.
See:
https://cloud.google.com/container-registry/docs/support/deprecation-notices#gcloud-docker
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:484bbbe7331d393798283d2f9652b93fa8e95388cd548f5a616bc8ad07df1f5c
Associated tags:
- 20220121-001238094613300
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20220121-001238094613300
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20220121-001238094613300].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:484bbbe7331d393798283d2f9652b93fa8e95388cd548f5a616bc8ad07df1f5c].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:8d65d8cdfb49432e975ba0d3f632e2c0b5c4ffe821cffae20e7a1452cfb77c74
Associated tags:
- eb40a067-5ed4-47cb-befe-c87a61d5010f
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:eb40a067-5ed4-47cb-befe-c87a61d5010f].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:8d65d8cdfb49432e975ba0d3f632e2c0b5c4ffe821cffae20e7a1452cfb77c74].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3235c52ff9d40681e3938a9671e8214be9a48862ab8237540ea4cddbd05b0435
Associated tags:
- 610e4fa0-ac40-4fe3-b6f1-dda35ddbaddf
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:610e4fa0-ac40-4fe3-b6f1-dda35ddbaddf].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3235c52ff9d40681e3938a9671e8214be9a48862ab8237540ea4cddbd05b0435].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
[gw0] [100%] PASSED
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.8_sdk.xml>
-
=================== 4 passed, 3 warnings in 2092.71 seconds ====================
>>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20220121-001238109933960
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:7626080ff903d863357a90cfcbb51e7dbfe277971758f82bb12e5ce45bfd19ca
docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:b2b0ca07-6e25-4317-8ed5-cf1398652c38
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:3b4132142ca7f79c44ea4fd9d8aadb0e341be8c6ac983203e0ad084f5845a320
Deleted: sha256:78b8e9ae0346f1de261798ea40e2c2b8741ab5930089c6ffaf7b5464cf4f83af
Deleted: sha256:2e4c5b78fe692d6e7cb60c1afa0f783a800b6b3923787f801419ad0028a2d618
Deleted: sha256:254eb71b00f0bf09e33ba8ee2bcb2403663e49ff15925f12af53d77779f2c463
Deleted: sha256:478d2d710e8796d3939d15156b73785fd649364d81f3a63abfe979509ddd6940
Deleted: sha256:8796bf7ecff9f6b159ce8c9cccf1b146a8747d24dee80b04408f77c8c8e0ca30
Deleted: sha256:eb507ee60a475ca14f1d7ff94c1051bb6574bf710fe19c59b1ecd132f361d347
Deleted: sha256:011da0a8ccc9cff385a6f672ac66e7d27c4cac792f2c98b9d92a74c5ef0d89e9
Deleted: sha256:a5d3a2bb4a51d07cd6887f0641fd788bfc977fb8ad41f785f74c289669a09497
Deleted: sha256:cdd3abd58e45a19ad0d93fc9e9483ca810d5046905c04f1927a5a7016a9884ec
Deleted: sha256:5cdb986de5fed82617c15eb1f5dbddf52ade2b0fc51ca2716fe768aa277a10d8
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:7626080ff903d863357a90cfcbb51e7dbfe277971758f82bb12e5ce45bfd19ca
Associated tags:
- 20220121-001238109933960
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20220121-001238109933960
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20220121-001238109933960].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:7626080ff903d863357a90cfcbb51e7dbfe277971758f82bb12e5ce45bfd19ca].
gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)"
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:3b4132142ca7f79c44ea4fd9d8aadb0e341be8c6ac983203e0ad084f5845a320
Associated tags:
- b2b0ca07-6e25-4317-8ed5-cf1398652c38
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:b2b0ca07-6e25-4317-8ed5-cf1398652c38].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:3b4132142ca7f79c44ea4fd9d8aadb0e341be8c6ac983203e0ad084f5845a320].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:bfc5c3b42e8a78e8ba5088bac47d5e55f19c575730efe16d86012bc327c8cdd4
Associated tags:
- dcd86b88-6d0e-4ead-bf4f-ecd6c23a035b
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:dcd86b88-6d0e-4ead-bf4f-ecd6c23a035b].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:bfc5c3b42e8a78e8ba5088bac47d5e55f19c575730efe16d86012bc327c8cdd4].
Removed the container
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:container:py37:docker'.
> Process 'command 'docker'' finished with non-zero exit value 2
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 49m 13s
50 actionable tasks: 40 executed, 4 from cache, 6 up-to-date
Publishing build scan...
https://gradle.com/s/nxd4y5bbxamms
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
