See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/9829/display/redirect?page=changes>
Changes:
[Alexey Romanenko] [TPC-DS] Use "nonpartitioned" input for Jenkins jobs
------------------------------------------
[...truncated 1.35 MB...]
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:6fa3e8b08aebf792b0d15aeb4c626f5bab99e3f3a1c4d1fbcdda020830ba7b83
Associated tags:
- 20221018-105412629141695
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221018-105412629141695
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221018-105412629141695].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:6fa3e8b08aebf792b0d15aeb4c626f5bab99e3f3a1c4d1fbcdda020830ba7b83].
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:65b85ec95a0e5289bcc544a0be71685212e24f6820acf26bfddf217e6ef51ca9
Associated tags:
- 20221018-105412666033512
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20221018-105412666033512
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20221018-105412666033512].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:65b85ec95a0e5289bcc544a0be71685212e24f6820acf26bfddf217e6ef51ca9].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:c68b126419c2d2076d381206006edd854dbb4683e4cd814abcb81061d9bb627c
Associated tags:
- 429cf689-518e-40b9-89b4-a5fa2dcd61e3
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:429cf689-518e-40b9-89b4-a5fa2dcd61e3].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:c68b126419c2d2076d381206006edd854dbb4683e4cd814abcb81061d9bb627c].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:1de0e7d10e0c271b8c926a990c7df4c5711d523c87922c9b8f057bea3ac4ab9d
Associated tags:
- 1025ac92-99fc-464e-b079-3eb33fd93b68
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:1025ac92-99fc-464e-b079-3eb33fd93b68].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:1de0e7d10e0c271b8c926a990c7df4c5711d523c87922c9b8f057bea3ac4ab9d].
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:58173ed826925bcfe5a11f9af97b6abf11bb85160f49fb5e18b06ccc329d3c4a
Associated tags:
- fb4d33fc-bf35-4cde-96a3-8e28a3ce0bbf
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:fb4d33fc-bf35-4cde-96a3-8e28a3ce0bbf].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:58173ed826925bcfe5a11f9af97b6abf11bb85160f49fb5e18b06ccc329d3c4a].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:60fed0a13cf1a2c3c27e59624b69d75b28d1412fff7218325ccb62027da1f8db
Associated tags:
- f09adf82-4716-4cae-989e-7e54ef13d98d
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:f09adf82-4716-4cae-989e-7e54ef13d98d].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:60fed0a13cf1a2c3c27e59624b69d75b28d1412fff7218325ccb62027da1f8db].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
[gw0] [100%] PASSED
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
=================================== FAILURES ===================================
____ WordCountIT.test_wordcount_it_with_prebuilt_sdk_container_cloud_build _____
[gw0] linux -- Python 3.8.10
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/bin/python3.8>
self = <apache_beam.examples.wordcount_it_test.WordCountIT
testMethod=test_wordcount_it_with_prebuilt_sdk_container_cloud_build>
@pytest.mark.it_validatescontainer
def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self):
> self._run_wordcount_it(
wordcount.run,
experiment='beam_fn_api',
prebuild_sdk_container_engine='cloud_build')
apache_beam/examples/wordcount_it_test.py:106:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam/examples/wordcount_it_test.py:149: in _run_wordcount_it
run_wordcount(
apache_beam/examples/wordcount.py:106: in run
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:597: in __exit__
self.result = self.run()
apache_beam/pipeline.py:547: in run
return Pipeline.from_runner_api(
apache_beam/pipeline.py:574: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:66: in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <DataflowPipelineResult <Job
clientRequestId: '20221018111232735471-9986'
createTime: '2022-10-18T11:12:33.845021Z'
...022-10-18T11:12:33.845021Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)> at 0x7f6e3abb9970>
duration = None
def wait_until_finish(self, duration=None):
if not self.is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
consoleUrl = (
"Console URL: https://console.cloud.google.com/";
f"dataflow/jobs/<RegionId>/{self.job_id()}"
"?project=<ProjectId>")
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
# use thread.join() to wait for the polling thread.
thread.daemon = True
thread.start()
while thread.is_alive():
> time.sleep(5.0)
E Failed: Timeout >900.0s
apache_beam/runners/dataflow/dataflow_runner.py:1626: Failed
=============================== warnings summary ===============================
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py:15
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py:15
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/hdfs/config.py>:15:
DeprecationWarning: the imp module is deprecated in favour of importlib; see
the module's documentation for alternative uses
from imp import load_source
../../build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py:42
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967051/lib/python3.8/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
apache_beam/typehints/pandas_type_compatibility_test.py:66
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:66:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
}).set_index(pd.Int64Index(range(123, 223), name='an_index')),
apache_beam/typehints/pandas_type_compatibility_test.py:89
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:89:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(123, 223), name='an_index'),
apache_beam/typehints/pandas_type_compatibility_test.py:90
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:90:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(475, 575), name='another_index'),
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.8_sdk.xml>
-
=========================== short test summary info ============================
FAILED
apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it_with_prebuilt_sdk_container_cloud_build
======= 1 failed, 3 passed, 3 skipped, 6 warnings in 2330.70s (0:38:50) ========
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221018-105412467124395
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:675117c94e1455ec637f4cdb40fdc897e91d428fc569bf6caf0b5e0ad034b106
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:d3388398-7198-44f9-b279-ebfbf35eebc4
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:9c016f3b9038151344a2515ec2235cc711cdefcc8e84cab2665ee9e05de67b3d
Deleted: sha256:7d8b2061bbf23d18f7e567df8ce14082463938ef62ba382d5863e26fb0c71a82
Deleted: sha256:1fa53c88d07a93d48b210d47e3de169a8fee664485006fcfa145ba8a975ebc0e
Deleted: sha256:efa8a425bb5c87835c1c2bc0df02743a01f980349cfaedbbe8a119c24cec0a28
Deleted: sha256:36fdd1130f903c1189225eb59639c7b32879a3c2fc4310e9ff78a470aa4c10fb
Deleted: sha256:c448a69cda304441259336e6a34ff5aa20ef3fce86cfcd32460107020508cd2e
Deleted: sha256:93c489ee19831a13bd7234085ff706f2da8ff32f437b7ef5a8b74ada33c1e36e
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:675117c94e1455ec637f4cdb40fdc897e91d428fc569bf6caf0b5e0ad034b106
Associated tags:
- 20221018-105412467124395
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221018-105412467124395
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221018-105412467124395].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:675117c94e1455ec637f4cdb40fdc897e91d428fc569bf6caf0b5e0ad034b106].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:9c016f3b9038151344a2515ec2235cc711cdefcc8e84cab2665ee9e05de67b3d
Associated tags:
- d3388398-7198-44f9-b279-ebfbf35eebc4
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:d3388398-7198-44f9-b279-ebfbf35eebc4].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:9c016f3b9038151344a2515ec2235cc711cdefcc8e84cab2665ee9e05de67b3d].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:31a8af08209ac143306616bcca0c2e082ceb743314a4f712c825e0cbce5608a0
Associated tags:
- a82243e6-2773-451e-840c-d1bbd462f889
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:a82243e6-2773-451e-840c-d1bbd462f889].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:31a8af08209ac143306616bcca0c2e082ceb743314a4f712c825e0cbce5608a0].
Removed the container
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED
FAILURE: Build failed with an exception.
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 359
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py38:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings
and determine if they come from your own scripts or plugins.
See
https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 55m 16s
51 actionable tasks: 41 executed, 4 from cache, 6 up-to-date
Publishing build scan...
https://gradle.com/s/t3f7eohs5ly54
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
