See
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/220/display/redirect?page=changes>
Changes:
[Heejong Lee] [BEAM-14133] Fix potential NPE in
BigQueryServicesImpl.getErrorInfo
[Heejong Lee] add test
[Kiley Sok] Revert "[BEAM-14190] Python sends dataflow schema field"
------------------------------------------
[...truncated 589.97 KB...]
apache_beam/runners/portability/fn_api_runner/fn_runner.py:1276: in
process_bundle
result_future = self._worker_handler.control_conn.push(process_bundle_req)
apache_beam/runners/portability/fn_api_runner/worker_handlers.py:378: in push
response = self.worker.do_instruction(request)
apache_beam/runners/worker/sdk_worker.py:581: in do_instruction
getattr(request, request_type), request.instruction_id)
apache_beam/runners/worker/sdk_worker.py:618: in process_bundle
bundle_processor.process_bundle(instruction_id))
apache_beam/runners/worker/bundle_processor.py:996: in process_bundle
element.data)
apache_beam/runners/worker/bundle_processor.py:221: in process_encoded
self.output(decoded_value)
apache_beam/runners/worker/operations.py:348: in output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:215: in receive
self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:708: in process
delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1200: in process
self._reraise_augmented(exn)
apache_beam/runners/common.py:1281: in _reraise_augmented
raise new_exn.with_traceback(tb)
apache_beam/runners/common.py:1198: in process
return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:719: in invoke_process
windowed_value, additional_args, additional_kwargs)
apache_beam/runners/common.py:844: in _invoke_process_per_window
self.threadsafe_watermark_estimator)
apache_beam/runners/common.py:1334: in process_outputs
for result in results:
apache_beam/io/iobase.py:1231: in <genexpr>
window.TimestampedValue(v, timestamp.MAX_TIMESTAMP) for v in outputs)
apache_beam/io/filebasedsink.py:279: in finalize_write
self._check_state_for_finalize_write(writer_results, num_shards))
apache_beam/io/filebasedsink.py:243: in _check_state_for_finalize_write
file_metadata.path for mr in FileSystems.match([dst_glob])
apache_beam/io/filesystems.py:204: in match
return filesystem.match(patterns, limits)
apache_beam/io/filesystem.py:744: in match
result.append(_match(pattern, limit))
apache_beam/io/filesystem.py:733: in _match
for file_metadata in self.match_files(file_metadatas, pattern):
apache_beam/io/filesystem.py:614: in match_files
for file_metadata in file_metas:
apache_beam/io/localfilesystem.py:122: in _list
for f in list_files(dir_or_prefix):
apache_beam/io/localfilesystem.py:117: in list_files
for dirpath, _, files in os.walk(root):
/usr/lib/python3.7/os.py:413: in walk
yield from walk(new_path, topdown, onerror, followlinks)
/usr/lib/python3.7/os.py:412: in walk
if followlinks or not islink(new_path):
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
path = '/tmp/tmp2i_385eg/go-testwordcount-32af7325-9275-4d5b-8b46-4379b3952819'
def islink(path):
"""Test whether a path is a symbolic link"""
try:
> st = os.lstat(path)
E Failed: Timeout >600.0s [while running
'Write/Write/WriteImpl/FinalizeWrite']
/usr/lib/python3.7/posixpath.py:171: Failed
----------------------------- Captured stderr call -----------------------------
+++++++++++++++++++++++++++++++++++ Timeout ++++++++++++++++++++++++++++++++++++
~~~~~~~~~~~~~~~~~~~~~ Stack of Thread-1 (139963395573504) ~~~~~~~~~~~~~~~~~~~~~~
File "/usr/lib/python3.7/threading.py", line 890, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File
"<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/runners/worker/data_plane.py";,>
line 206, in run
while not self._finished.wait(next_call - time.time()):
File "/usr/lib/python3.7/threading.py", line 552, in wait
signaled = self._cond.wait(timeout)
File "/usr/lib/python3.7/threading.py", line 300, in wait
gotit = waiter.acquire(True, timeout)
+++++++++++++++++++++++++++++++++++ Timeout ++++++++++++++++++++++++++++++++++++
------------------------------ Captured log call -------------------------------
INFO root:pipeline.py:188 Missing pipeline option (runner). Executing
pipeline using the default runner: DirectRunner.
WARNING root:environments.py:374 Make sure that locally built Python SDK
docker image has Python 3.7 interpreter.
INFO root:environments.py:380 Default Python SDK image for environment is
apache/beam_python3.7_sdk:2.39.0.dev
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function annotate_downstream_side_inputs at
0x7f4bb30d03b0> ====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function fix_side_input_pcoll_coders at 0x7f4bb30d04d0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function pack_combiners at 0x7f4bb30d09e0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function lift_combiners at 0x7f4bb30d0a70>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function expand_sdf at 0x7f4bb30d0c20>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function expand_gbk at 0x7f4bb30d0cb0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function sink_flattens at 0x7f4bb30d0dd0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function greedily_fuse at 0x7f4bb30d0e60>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function read_to_impulse at 0x7f4bb30d0ef0>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function impulse_to_input at 0x7f4bb30d0f80>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function sort_stages at 0x7f4bb30cf200>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function add_impulse_to_dangling_transforms at
0x7f4bb30cf320> ====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function setup_timer_mapping at 0x7f4bb30cf170>
====================
INFO
apache_beam.runners.portability.fn_api_runner.translations:translations.py:711
==================== <function populate_data_channel_coders at 0x7f4bb30cf290>
====================
INFO apache_beam.runners.worker.statecache:statecache.py:172 Creating state
cache with size 100
INFO
apache_beam.runners.portability.fn_api_runner.worker_handlers:worker_handlers.py:894
Created Worker handler
<apache_beam.runners.portability.fn_api_runner.worker_handlers.EmbeddedWorkerHandler
object at 0x7f4badc59650> for environment
ref_Environment_default_environment_1 (beam:env:embedded_python:v1, b'')
=============================== warnings summary ===============================
apache_beam/io/filesystems_test.py:54
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54:
DeprecationWarning: invalid escape sequence \c
self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint:
disable=anomalous-backslash-in-string
apache_beam/io/filesystems_test.py:62
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62:
DeprecationWarning: invalid escape sequence \d
self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), #
pylint: disable=anomalous-backslash-in-string
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:63:
PendingDeprecationWarning: Client.dataset is deprecated and will be removed in
a future version. Use a string like 'my_project.my_dataset' or a
cloud.google.bigquery.DatasetReference object, instead.
dataset_ref = client.dataset(unique_dataset_name, project=project)
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2143:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
is_streaming_pipeline = p.options.view_as(StandardOptions).streaming
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2149:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
experiments = p.options.view_as(DebugOptions).experiments or []
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:1128:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = p.options.view_as(GoogleCloudOptions).temp_location
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py>:1130:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
p.options.view_as(GoogleCloudOptions).job_name or 'AUTOMATIC_JOB_NAME')
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2443:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
temp_location = pcoll.pipeline.options.view_as(
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2445:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
job_name = pcoll.pipeline.options.view_as(GoogleCloudOptions).job_name
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2476:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
| _PassThroughThenCleanup(files_to_remove_pcoll))
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2139:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
self.table_reference.projectId = pcoll.pipeline.options.view_as(
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:100:
PendingDeprecationWarning: Client.dataset is deprecated and will be removed in
a future version. Use a string like 'my_project.my_dataset' or a
cloud.google.bigquery.DatasetReference object, instead.
table_ref = client.dataset(dataset_id).table(table_id)
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/examples/dataframe/flight_delays.py>:47:
FutureWarning: Dropping of nuisance columns in DataFrame reductions (with
'numeric_only=None') is deprecated; in a future version this will raise
TypeError. Select only valid columns before calling the reduction.
return airline_df[at_top_airports].mean()
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
apache_beam/examples/dataframe/wordcount_test.py::WordCountTest::test_basics
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/dataframe/io.py>:632:
FutureWarning: WriteToFiles is experimental.
sink=lambda _: _WriteToPandasFileSink(
apache_beam/examples/dataframe/flight_delays_it_test.py::FlightDelaysTest::test_flight_delays
apache_beam/examples/dataframe/wordcount_test.py::WordCountTest::test_basics
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/fileio.py>:550:
BeamDeprecationWarning: options is deprecated since First stable release.
References to <pipeline>.options will not be supported
p.options.view_as(GoogleCloudOptions).temp_location or
-- Docs: https://docs.pytest.org/en/latest/warnings.html
- generated xml file:
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/pytest_postCommitExamples-direct-py37.xml>
-
= 3 failed, 19 passed, 1 skipped, 5267 deselected, 40 warnings in 2042.60
seconds =
> Task :sdks:python:test-suites:direct:py37:examples FAILED
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'>
line: 92
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py38:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
==============================================================================
2: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'>
line: 92
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py39:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
==============================================================================
3: Task failed with an exception.
-----------
* Where:
Script
'<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'>
line: 92
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py37:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
==============================================================================
* Get more help at https://help.gradle.org
BUILD FAILED in 35m 24s
21 actionable tasks: 15 executed, 4 from cache, 2 up-to-date
Publishing build scan...
https://gradle.com/s/twldeaazcsm7i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
