See
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/9807/display/redirect?page=changes>
Changes:
[noreply] (BQ Java) Explicitly set coder for multi-partition batch load writes
[noreply] Fix typo in bootstrap_beam_venv.py (#23574)
[noreply] Bump github.com/spf13/cobra from 1.5.0 to 1.6.0 in /sdks (#23591)
[noreply] [Playground][Tour Of Beam] Datastore entities split by origin (#23088)
[noreply] use write schema only for read api (#23594)
[noreply] [Go SDK]: SingleFlight bundle descriptor requests (#23589)
------------------------------------------
[...truncated 1.33 MB...]
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to
delete container"
for digest in $(gcloud container images list-tags
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk
--format="get(digest)")
do gcloud container images delete
$PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest
--force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container
image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:1565d9e81108aca0383c7d4a553f39ad0c3511e1e5d19e8e645e616206ad6648
Associated tags:
- 20221012-225208885716830
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221012-225208885716830
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20221012-225208885716830].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:1565d9e81108aca0383c7d4a553f39ad0c3511e1e5d19e8e645e616206ad6648].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:304ef826525813a57ffa7ad184601f0da002b5797a7613fe208608fd946ce989
Associated tags:
- f50e72b8-0a2f-4191-b0c1-516df9aef0c9
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:f50e72b8-0a2f-4191-b0c1-516df9aef0c9].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:304ef826525813a57ffa7ad184601f0da002b5797a7613fe208608fd946ce989].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a2be5111ae99c3f900f4b7ff121bdac0ac9fe4409d9f1263ff6325a5af55efca
Associated tags:
- 292b3238-9dcf-43a2-aae6-720514127951
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:292b3238-9dcf-43a2-aae6-720514127951].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a2be5111ae99c3f900f4b7ff121bdac0ac9fe4409d9f1263ff6325a5af55efca].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:7ca3e56d99da8cc17232bdd65061600e25f6c0aa4661c680b2466260650988d4
Associated tags:
- 20221012-225208999722578
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221012-225208999722578
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20221012-225208999722578].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:7ca3e56d99da8cc17232bdd65061600e25f6c0aa4661c680b2466260650988d4].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:e1081ac312ffda4966deb9707f6e8884576437d3b56f5d288be49a5d8fd87bd2
Associated tags:
- feea92b8-bb25-4b89-bda5-3aeff0ad4870
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:feea92b8-bb25-4b89-bda5-3aeff0ad4870].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:e1081ac312ffda4966deb9707f6e8884576437d3b56f5d288be49a5d8fd87bd2].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:f649c9396007f98aef45dbc3c41d0d2d00b67d9d685c32ad86f1bfaab33f16a9
Associated tags:
- ed844153-785c-4cf9-b1ed-b3919a66269a
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:ed844153-785c-4cf9-b1ed-b3919a66269a].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:f649c9396007f98aef45dbc3c41d0d2d00b67d9d685c32ad86f1bfaab33f16a9].
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer
[gw0] [100%] PASSED
apache_beam/runners/dataflow/dataflow_exercise_metrics_pipeline_test.py::ExerciseMetricsPipelineTest::test_metrics_fnapi_it
=================================== FAILURES ===================================
____ WordCountIT.test_wordcount_it_with_prebuilt_sdk_container_cloud_build _____
[gw0] linux -- Python 3.9.10
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967050/bin/python3.9>
self = <apache_beam.examples.wordcount_it_test.WordCountIT
testMethod=test_wordcount_it_with_prebuilt_sdk_container_cloud_build>
@pytest.mark.it_validatescontainer
def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self):
> self._run_wordcount_it(
wordcount.run,
experiment='beam_fn_api',
prebuild_sdk_container_engine='cloud_build')
apache_beam/examples/wordcount_it_test.py:103:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
apache_beam/examples/wordcount_it_test.py:146: in _run_wordcount_it
run_wordcount(
apache_beam/examples/wordcount.py:106: in run
output | 'Write' >> WriteToText(known_args.output)
apache_beam/pipeline.py:597: in __exit__
self.result = self.run()
apache_beam/pipeline.py:547: in run
return Pipeline.from_runner_api(
apache_beam/pipeline.py:574: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/dataflow/test_dataflow_runner.py:66: in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <DataflowPipelineResult <Job
clientRequestId: '20221012231036684975-9986'
createTime: '2022-10-12T23:10:37.811278Z'
...022-10-12T23:10:37.811278Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)> at 0x7fd981ace1c0>
duration = None
def wait_until_finish(self, duration=None):
if not self.is_in_terminal_state():
if not self.has_job:
raise IOError('Failed to get the Dataflow job id.')
consoleUrl = (
"Console URL: https://console.cloud.google.com/";
f"dataflow/jobs/<RegionId>/{self.job_id()}"
"?project=<ProjectId>")
thread = threading.Thread(
target=DataflowRunner.poll_for_job_completion,
args=(self._runner, self, duration))
# Mark the thread as a daemon thread so a keyboard interrupt on the main
# thread will terminate everything. This is also the reason we will not
# use thread.join() to wait for the polling thread.
thread.daemon = True
thread.start()
while thread.is_alive():
> time.sleep(5.0)
E Failed: Timeout >900.0s
apache_beam/runners/dataflow/dataflow_runner.py:1622: Failed
=============================== warnings summary ===============================
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/hdfs/config.py:15
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/hdfs/config.py:15
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967050/lib/python3.9/site-packages/hdfs/config.py>:15:
DeprecationWarning: the imp module is deprecated in favour of importlib; see
the module's documentation for alternative uses
from imp import load_source
../../build/gradleenv/-1734967050/lib/python3.9/site-packages/tenacity/_asyncio.py:42
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/build/gradleenv/-1734967050/lib/python3.9/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
apache_beam/typehints/pandas_type_compatibility_test.py:66
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:66:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
}).set_index(pd.Int64Index(range(123, 223), name='an_index')),
apache_beam/typehints/pandas_type_compatibility_test.py:89
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:89:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(123, 223), name='an_index'),
apache_beam/typehints/pandas_type_compatibility_test.py:90
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:90:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(475, 575), name='another_index'),
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.9_sdk.xml>
-
=========================== short test summary info ============================
FAILED
apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it_with_prebuilt_sdk_container_cloud_build
======= 1 failed, 3 passed, 3 skipped, 6 warnings in 2235.70s (0:37:15) ========
cleanup_container
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20221012-225209208819350
Untagged:
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:9c8ec589063b31e0a96b2f78ce08b2a3c3aa5be4ecd298038eea2c4d27574f96
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:39b818a3-cbc8-4600-9eaf-faee69f835d4
Untagged:
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:86bab95df42af7afa0423824cabd3ccfbe1e88d07ea496454df0cd87b9d13a73
Deleted: sha256:b977d7161f64bde8983a9b05f0f90033bc7b237c2ef6e03f6f6f4088a9fd8caf
Deleted: sha256:b558bf495cad5ff00c68d29b1667808a14388b91cf63a94032dcfb9f0093539b
Deleted: sha256:3545425f8f0a06725441b14a6e3e0b7c83268bf1aad00074db6052eff03db221
Deleted: sha256:54d19c993b9344d9ae6c69661c253c3adedcb1c144a31d8fe25d30954ccdb9ea
Deleted: sha256:16eb5d37913b7ea35a17f7474986cd7a8d0cddb63001b680ce11d3481d82e649
Deleted: sha256:d1290e221de7baa3bb5c36bd316e54d91394f4b794e9940fec499e22baae7c47
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:9c8ec589063b31e0a96b2f78ce08b2a3c3aa5be4ecd298038eea2c4d27574f96
Associated tags:
- 20221012-225209208819350
Tags:
-
us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20221012-225209208819350
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk:20221012-225209208819350].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/beam_python3.9_sdk@sha256:9c8ec589063b31e0a96b2f78ce08b2a3c3aa5be4ecd298038eea2c4d27574f96].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:86bab95df42af7afa0423824cabd3ccfbe1e88d07ea496454df0cd87b9d13a73
Associated tags:
- 39b818a3-cbc8-4600-9eaf-faee69f835d4
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:39b818a3-cbc8-4600-9eaf-faee69f835d4].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:86bab95df42af7afa0423824cabd3ccfbe1e88d07ea496454df0cd87b9d13a73].
Digests:
-
us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:f4ed557ff00fbf12c6e1f602eeaffcb636ee7588b945eae0faff5b0149889ac2
Associated tags:
- ffc7190b-2caf-40d0-bf30-7f6c915b1574
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk:ffc7190b-2caf-40d0-bf30-7f6c915b1574].
Deleted
[us.gcr.io/apache-beam-testing/jenkins/prebuild_python39_sdk/beam_python_prebuilt_sdk@sha256:f4ed557ff00fbf12c6e1f602eeaffcb636ee7588b945eae0faff5b0149889ac2].
Removed the container
> Task :sdks:python:test-suites:dataflow:py39:validatesContainer FAILED
FAILURE: Build failed with an exception.
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'>
line: 359
* What went wrong:
Execution failed for task
':sdks:python:test-suites:dataflow:py39:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with
Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings
and determine if they come from your own scripts or plugins.
See
https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 52m 21s
51 actionable tasks: 41 executed, 4 from cache, 6 up-to-date
Publishing build scan...
https://gradle.com/s/pld4w7ao57g36
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
