See
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/1220/display/redirect?page=changes>
Changes:
[chamikaramj] Fixing a breakage of multi-lang auto Runner v2 enabling
[noreply] interface{} -> any for registration (#24600)
[noreply] Restrict tox to be in 3.x version (#24601)
[noreply] [Playground] support for Kafka-enabled examples (#24459)
[noreply] Fix some small notebook typos (#24616)
------------------------------------------
[...truncated 619.59 KB...]
apache_beam/pipeline.py:577: in run
return self.runner.run_pipeline(self, self._options)
apache_beam/runners/direct/test_direct_runner.py:42: in run_pipeline
self.result = super().run_pipeline(pipeline, options)
apache_beam/runners/direct/direct_runner.py:131: in run_pipeline
return runner.run_pipeline(pipeline, options)
apache_beam/runners/portability/fn_api_runner/fn_runner.py:201: in run_pipeline
self._latest_run_result = self.run_via_runner_api(
apache_beam/runners/portability/fn_api_runner/fn_runner.py:222: in
run_via_runner_api
return self.run_stages(stage_context, stages)
apache_beam/runners/portability/fn_api_runner/fn_runner.py:453: in run_stages
bundle_results = self._execute_bundle(
apache_beam/runners/portability/fn_api_runner/fn_runner.py:781: in
_execute_bundle
self._run_bundle(
apache_beam/runners/portability/fn_api_runner/fn_runner.py:1010: in _run_bundle
result, splits = bundle_manager.process_bundle(
apache_beam/runners/portability/fn_api_runner/fn_runner.py:1346: in
process_bundle
result_future = self._worker_handler.control_conn.push(process_bundle_req)
apache_beam/runners/portability/fn_api_runner/worker_handlers.py:379: in push
response = self.worker.do_instruction(request)
apache_beam/runners/worker/sdk_worker.py:596: in do_instruction
return getattr(self, request_type)(
apache_beam/runners/worker/sdk_worker.py:634: in process_bundle
bundle_processor.process_bundle(instruction_id))
apache_beam/runners/worker/bundle_processor.py:1009: in process_bundle
op.finish()
apache_beam/runners/worker/operations.py:943: in finish
self.dofn_runner.finish()
apache_beam/runners/common.py:1479: in finish
self._invoke_bundle_method(self.do_fn_invoker.invoke_finish_bundle)
apache_beam/runners/common.py:1460: in _invoke_bundle_method
self._reraise_augmented(exn)
apache_beam/runners/common.py:1507: in _reraise_augmented
raise new_exn.with_traceback(tb)
apache_beam/runners/common.py:1458: in _invoke_bundle_method
bundle_method()
apache_beam/runners/common.py:566: in invoke_finish_bundle
self.output_handler.finish_bundle_outputs(
apache_beam/runners/common.py:1730: in finish_bundle_outputs
for result in results:
apache_beam/io/iobase.py:1202: in finish_bundle
self.writer.close(),
apache_beam/io/filebasedsink.py:434: in close
self.sink.close(self.temp_handle)
apache_beam/io/textio.py:516: in close
super().close(file_handle)
apache_beam/io/filebasedsink.py:167: in close
file_handle.close()
apache_beam/io/filesystemio.py:215: in close
self._uploader.finish()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.io.gcp.gcsio.GcsUploader object at 0x7fc632fbca90>
def finish(self):
self._conn.close()
# TODO(udim): Add timeout=DEFAULT_HTTP_TIMEOUT_SECONDS * 2 and raise if
# isAlive is True.
self._upload_thread.join()
# Check for exception since the last put() call.
if self._upload_thread.last_error is not None:
raise type(self._upload_thread.last_error)(
"Error while uploading file %s: %s",
self._path,
> self._upload_thread.last_error.message) # pylint:
> disable=raising-bad-type
E AttributeError: 'HttpError' object has no attribute 'message' [while
running 'write chars/Write/WriteImpl/WriteBundles']
apache_beam/io/gcp/gcsio.py:829: AttributeError
------------------------------ Captured log call -------------------------------
WARNING apache_beam.options.pipeline_options:pipeline_options.py:352
Discarding unparseable args: ['--sleep_secs=20',
'--kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test']
INFO root:environments.py:376 Default Python SDK image for environment is
apache/beam_python3.9_sdk:2.45.0.dev
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function annotate_downstream_side_inputs at
0x7fc684e9c790> ====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function fix_side_input_pcoll_coders at 0x7fc684e9c8b0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function pack_combiners at 0x7fc684e9cdc0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function lift_combiners at 0x7fc684e9ce50>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function expand_sdf at 0x7fc684e9b040>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function expand_gbk at 0x7fc684e9b0d0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function sink_flattens at 0x7fc684e9b1f0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function greedily_fuse at 0x7fc684e9b280>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function read_to_impulse at 0x7fc684e9b310>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function impulse_to_input at 0x7fc684e9b3a0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function sort_stages at 0x7fc684e9b5e0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function add_impulse_to_dangling_transforms at
0x7fc684e9b700> ====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function setup_timer_mapping at 0x7fc684e9b550>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:710
==================== <function populate_data_channel_coders at 0x7fc684e9b670>
====================
INFO apache_beam.runners.worker.statecache:statecache.py:234 Creating state
cache with size 104857600
INFO
apache_beam.runners.portability.fn_api_runner.worker_handlers:worker_handlers.py:903
Created Worker handler
<apache_beam.runners.portability.fn_api_runner.worker_handlers.EmbeddedWorkerHandler
object at 0x7fc640b25f70> for environment
ref_Environment_default_environment_1 (beam:env:embedded_python:v1, b'')
ERROR apache_beam.io.gcp.gcsio:gcsio.py:803 Error in _start_upload while
inserting file
gs://temp-storage-for-end-to-end-tests/temp-it/4b9064e8-ce30-4016-a3d2-4e4917ee30c0/beam-temp-result-chars-93a0c94377ef11ed82ae91f5bbf19933/1d9b9bb4-23be-4ca6-a7a6-83bdb2baee41.result-chars:
Traceback (most recent call last):
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/gcsio.py";,>
line 799, in _start_upload
self._client.objects.Insert(self._insert_request, upload=self._upload)
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py";,>
line 1152, in Insert
return self._RunMethod(
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/build/gradleenv/1398941893/lib/python3.9/site-packages/apitools/base/py/base_api.py";,>
line 731, in _RunMethod
return self.ProcessHttpResponse(method_config, http_response, request)
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/build/gradleenv/1398941893/lib/python3.9/site-packages/apitools/base/py/base_api.py";,>
line 737, in ProcessHttpResponse
self.__ProcessHttpResponse(method_config, http_response, request))
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/build/gradleenv/1398941893/lib/python3.9/site-packages/apitools/base/py/base_api.py";,>
line 603, in __ProcessHttpResponse
raise exceptions.HttpError.FromResponse(
apitools.base.py.exceptions.HttpError: HttpError accessing
<https://www.googleapis.com/resumable/upload/storage/v1/b/temp-storage-for-end-to-end-tests/o?alt=json&name=temp-it%2F4b9064e8-ce30-4016-a3d2-4e4917ee30c0%2Fbeam-temp-result-chars-93a0c94377ef11ed82ae91f5bbf19933%2F1d9b9bb4-23be-4ca6-a7a6-83bdb2baee41.result-chars&uploadType=resumable&upload_id=ADPycds8E9_5FmbS_XiACNAWInC5mWJPImc1LSMYdDRJYlWXyRcWE1QdFnCVVIFlxPd7PqZaUQ7eaTt7GnsRXf21eE7CPA>:
response: <{'content-type': 'text/plain; charset=utf-8',
'x-guploader-uploadid':
'ADPycds8E9_5FmbS_XiACNAWInC5mWJPImc1LSMYdDRJYlWXyRcWE1QdFnCVVIFlxPd7PqZaUQ7eaTt7GnsRXf21eE7CPA',
'content-length': '0', 'date': 'Fri, 09 Dec 2022 18:30:47 GMT', 'server':
'UploadServer', 'status': '503'}>, content <>
=============================== warnings summary ===============================
../../build/gradleenv/1398941893/lib/python3.9/site-packages/hdfs/config.py:15
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/build/gradleenv/1398941893/lib/python3.9/site-packages/hdfs/config.py>:15:
DeprecationWarning: the imp module is deprecated in favour of importlib; see
the module's documentation for alternative uses
from imp import load_source
../../build/gradleenv/1398941893/lib/python3.9/site-packages/tenacity/_asyncio.py:42
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/build/gradleenv/1398941893/lib/python3.9/site-packages/tenacity/_asyncio.py>:42:
DeprecationWarning: "@coroutine" decorator is deprecated since Python 3.8, use
"async def" instead
def call(self, fn, *args, **kwargs):
apache_beam/typehints/pandas_type_compatibility_test.py:67
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:67:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
}).set_index(pd.Int64Index(range(123, 223), name='an_index')),
apache_beam/typehints/pandas_type_compatibility_test.py:90
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:90:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(123, 223), name='an_index'),
apache_beam/typehints/pandas_type_compatibility_test.py:91
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/typehints/pandas_type_compatibility_test.py>:91:
FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas
in a future version. Use pandas.Index with the appropriate dtype instead.
pd.Int64Index(range(475, 575), name='another_index'),
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:63:
PendingDeprecationWarning: Client.dataset is deprecated and will be removed in
a future version. Use a string like 'my_project.my_dataset' or a
cloud.google.bigquery.DatasetReference object, instead.
dataset_ref = client.dataset(unique_dataset_name, project=project)
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1992:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
is_streaming_pipeline = p.options.view_as(StandardOptions).streaming
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1998:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
experiments = p.options.view_as(DebugOptions).experiments or []
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:1149:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = p.options.view_as(GoogleCloudOptions).temp_location
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:1151:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
p.options.view_as(GoogleCloudOptions).job_name or 'AUTOMATIC_JOB_NAME')
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2485:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = pcoll.pipeline.options.view_as(
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2487:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
job_name = pcoll.pipeline.options.view_as(GoogleCloudOptions).job_name
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2511:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
pipeline_options=pcoll.pipeline.options,
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:1988:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
self.table_reference.projectId = pcoll.pipeline.options.view_as(
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:1142:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
self.project = self.project or p.options.view_as(GoogleCloudOptions).project
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:100:
PendingDeprecationWarning: Client.dataset is deprecated and will be removed in
a future version. Use a string like 'my_project.my_dataset' or a
cloud.google.bigquery.DatasetReference object, instead.
table_ref = client.dataset(dataset_id).table(table_id)
apache_beam/examples/dataframe/flight_delays_it_test.py: 46 warnings
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/examples/dataframe/flight_delays.py>:47:
FutureWarning: The default value of numeric_only in DataFrame.mean is
deprecated. In a future version, it will default to False. In addition,
specifying 'numeric_only=None' is deprecated. Select only valid columns or
specify the value of numeric_only to silence this warning.
return airline_df[at_top_airports].mean()
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
apache_beam/examples/dataframe/wordcount_test.py::WordCountTest::test_basics
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/dataframe/io.py>:659:
FutureWarning: WriteToFiles is experimental.
return pcoll | fileio.WriteToFiles(
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
apache_beam/examples/dataframe/wordcount_test.py::WordCountTest::test_basics
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/fileio.py>:591:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
p.options.view_as(GoogleCloudOptions).temp_location or
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/pytest_postCommitExamples-direct-py39.xml>
-
=========================== short test summary info ============================
FAILED
apache_beam/examples/cookbook/multiple_output_pardo_test.py::MultipleOutputParDo::test_multiple_output_pardo
- AttributeError: 'HttpError' object has no attribute 'message' [while running
'write chars/Write/WriteImpl/WriteBundles']
= 1 failed, 21 passed, 6 skipped, 6760 deselected, 89 warnings in 445.83s
(0:07:25) =
> Task :sdks:python:test-suites:direct:py39:examples FAILED
FAILURE: Build completed with 2 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'>
line: 92
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py310:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'>
line: 92
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py39:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
==============================================================================
* Get more help at https://help.gradle.org
BUILD FAILED in 8m 50s
24 actionable tasks: 18 executed, 4 from cache, 2 up-to-date
Publishing build scan...
https://gradle.com/s/avqmxnzh67te4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
