See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/10120/display/redirect?page=changes>
Changes:
[Robert Bradshaw] Add gradle files for typescript containers and update the
version info.
[Robert Bradshaw] Various cleanups to the typescript SDK:
[Robert Bradshaw] Fix issue with typescript sibling sdks.
[Robert Bradshaw] Add timeout to cross-language tests.
[Robert Bradshaw] npm install before build
[Robert Bradshaw] Format go file.
[noreply] [#24789] Spot fix fullvalue wrapping for SDF. (#24826)
------------------------------------------
[...truncated 1.84 MB...]
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py310:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use
sha256 directly.
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:43246e63991151b1db1ab71094661ec23226ae0d9a698263967ae48ae14dee86
Associated tags:
- 20221230-045536663842173
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221230-045536663842173
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221230-045536663842173].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:43246e63991151b1db1ab71094661ec23226ae0d9a698263967ae48ae14dee86].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:4cf35f3c1782aeb90147a4d27e425ef6cf60498ddaf36a9c64570c3a9c51a0d2
Associated tags:
- cfd0a1a8-2a6b-4f02-a73a-e37ff1350c28
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:cfd0a1a8-2a6b-4f02-a73a-e37ff1350c28].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:4cf35f3c1782aeb90147a4d27e425ef6cf60498ddaf36a9c64570c3a9c51a0d2].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:f93d280c875612024d0ecd181845da502493e776c0bcf39550a9093d78020672
Associated tags:
- fa40cb0f-4da1-4b05-9a08-f3ec61d55607
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:fa40cb0f-4da1-4b05-9a08-f3ec61d55607].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:f93d280c875612024d0ecd181845da502493e776c0bcf39550a9093d78020672].
> Task :sdks:python:test-suites:dataflow:py310:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use
sha256 directly.
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk@sha256:ed9ef86a4dcbb8a3fcbc036eed3622250c9677f2a0f19003401f42e6d92eb812
Associated tags:
- 20221230-045537026109496
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk:20221230-045537026109496
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk:20221230-045537026109496].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk@sha256:ed9ef86a4dcbb8a3fcbc036eed3622250c9677f2a0f19003401f42e6d92eb812].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:751884e8568f575e71615bca4824d8c0bb8cc9e290fe5671de016f193296659f
Associated tags:
- 59b4040a-007c-4b9b-b1a8-916271096a98
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk:59b4040a-007c-4b9b-b1a8-916271096a98].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:751884e8568f575e71615bca4824d8c0bb8cc9e290fe5671de016f193296659f].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:85de8bf1b1989ab14e8fa112b506d8ad57e1fd9b3aa0c3ff8a197d4fbf93ce54
Associated tags:
- 3ec6c399-5abd-4a07-abcf-dce3b8da3636
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk:3ec6c399-5abd-4a07-abcf-dce3b8da3636].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:85de8bf1b1989ab14e8fa112b506d8ad57e1fd9b3aa0c3ff8a197d4fbf93ce54].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
[gw0] [ 80%] PASSED
apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it_with_use_sibling_sdk_workers
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
[gw0] [100%] SKIPPED
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
=============================== warnings summary ===============================
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py:15
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py:15
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py>:15:
DeprecationWarning: the imp module is deprecated in favour of importlib; see
the module's documentation for alternative uses
from imp import load_source
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py:42
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
apache_beam/typehints/pandas_type_compatibility_test.py:67
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:67:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
}).set_index(pd.Int64Index(range(123, 223), name='an_index')),
apache_beam/typehints/pandas_type_compatibility_test.py:90
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:90:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(123, 223), name='an_index'),
apache_beam/typehints/pandas_type_compatibility_test.py:91
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:91:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(475, 575), name='another_index'),
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.8_sdk.xml>
-
============ 4 passed, 6 skipped, 6 warnings in 2220.62s (0:37:00) =============
>>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221230-045537134854507
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:c16219116f6ed24e7392abe514504f75ac7f435595ae0657068d794a8f6bd85b
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:3e6edda9-d711-4bf7-bf22-216070fa2808
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:5254c6e4fa599fecd3f21de86ad6704737f9db4b6b3d850b8081d533881d6ae4
Deleted: sha256:1f1571b22eb12affbb1ce55419568112fdf1108b0d09ce0b4422286d7c8c2d7f
Deleted: sha256:ceb2f4eae21476699103876eb38ae6a5d90b6943d2b81e49939c886ecc782945
Deleted: sha256:8e75d03f0e974a4e2c84e4a757724cc3f6f9b7146f4a7bd9859696cdfe60781e
Deleted: sha256:1cf49204e5bde1aa369fe0a022c38ea6f7b5c8d0c348bbce8468e9e76dd07623
Deleted: sha256:ec5f131e010a77ce5b18cd871e65818734ab5e2a20d88d38a1544513feac24fc
Deleted: sha256:c31131ca9c8cea4f9b52d841c59dc17bf1755019919f363c66fd3084d89ceeca
WARNING: Successfully resolved tag to sha256, but it is recommended to use
sha256 directly.
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:c16219116f6ed24e7392abe514504f75ac7f435595ae0657068d794a8f6bd85b
Associated tags:
- 20221230-045537134854507
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221230-045537134854507
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221230-045537134854507].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:c16219116f6ed24e7392abe514504f75ac7f435595ae0657068d794a8f6bd85b].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:5254c6e4fa599fecd3f21de86ad6704737f9db4b6b3d850b8081d533881d6ae4
Associated tags:
- 3e6edda9-d711-4bf7-bf22-216070fa2808
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:3e6edda9-d711-4bf7-bf22-216070fa2808].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:5254c6e4fa599fecd3f21de86ad6704737f9db4b6b3d850b8081d533881d6ae4].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:c11a014d5f2a3f96bd8548113ed03ad3fc6017adce98bededf4efed2335a5520
Associated tags:
- 3ab85d6a-dbc7-48f9-810c-4bbb3e30c01a
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:3ab85d6a-dbc7-48f9-810c-4bbb3e30c01a].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:c11a014d5f2a3f96bd8548113ed03ad3fc6017adce98bededf4efed2335a5520].
Removed the container
FAILURE: Build failed with an exception.
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 331
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py39:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings
and determine if they come from your own scripts or plugins.
See
https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 56m 31s
62 actionable tasks: 51 executed, 4 from cache, 7 up-to-date
Publishing build scan...
https://gradle.com/s/ktdcgvqxyxe3u
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
