[
https://issues.apache.org/jira/browse/BEAM-1251?focusedWorklogId=120675&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-120675
]
ASF GitHub Bot logged work on BEAM-1251:
----------------------------------------
Author: ASF GitHub Bot
Created on: 09/Jul/18 08:50
Start Date: 09/Jul/18 08:50
Worklog Time Spent: 10m
Work Description: charlesccychen edited a comment on issue #5887:
[BEAM-1251] Upgrade from buffer to memoryview (again)
URL: https://github.com/apache/beam/pull/5887#issuecomment-403407139
Heads up that this change caused issues in postcommits. Will investigate
more in the morning.
```
======================================================================
ERROR: test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/examples/wordcount_it_test.py",
line 68, in test_wordcount_it
wordcount.run(test_pipeline.get_full_options_as_args(**extra_opts))
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/examples/wordcount.py",
line 118, in run
result = p.run()
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/pipeline.py",
line 395, in run
self.to_runner_api(), self.runner, self._options).run(False)
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/pipeline.py",
line 408, in run
return self.runner.run_pipeline(self)
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py",
line 61, in run_pipeline
self.result.wait_until_finish(duration=wait_duration)
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",
line 1135, in wait_until_finish
(self.state, getattr(self._runner, 'last_error_msg', None)), self)
DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File
"/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line
642, in do_work
work_executor.execute()
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py",
line 156, in execute
op.start()
File "dataflow_worker/shuffle_operations.py", line 49, in
dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
def start(self):
File "dataflow_worker/shuffle_operations.py", line 50, in
dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.scoped_start_state:
File "dataflow_worker/shuffle_operations.py", line 65, in
dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.scoped_process_state:
File "dataflow_worker/shuffle_operations.py", line 66, in
dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.shuffle_source.reader() as reader:
File "dataflow_worker/shuffle_operations.py", line 70, in
dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
self.output(windowed_value)
File "apache_beam/runners/worker/operations.py", line 180, in
apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver,
self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 90, in
apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "dataflow_worker/shuffle_operations.py", line 229, in
dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
with self.scoped_process_state:
File "dataflow_worker/shuffle_operations.py", line 236, in
dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
self.output(wvalue.with_value((k, wvalue.value)))
File "apache_beam/runners/worker/operations.py", line 180, in
apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver,
self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 90, in
apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 404, in
apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 405, in
apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 577, in
apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 585, in
apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 610, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 583, in
apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 360, in
apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 681, in
apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 90, in
apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 404, in
apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 405, in
apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 577, in
apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 585, in
apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 610, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 583, in
apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 360, in
apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 681, in
apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 90, in
apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 404, in
apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 405, in
apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 577, in
apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 585, in
apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 626, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
raise_(type(new_exn), new_exn, original_traceback)
File "apache_beam/runners/common.py", line 583, in
apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 469, in
apache_beam.runners.common.PerWindowInvoker.invoke_process
self._invoke_per_window(
File "apache_beam/runners/common.py", line 492, in
apache_beam.runners.common.PerWindowInvoker._invoke_per_window
[si[global_window] for si in self.side_inputs]))
File
"/usr/local/lib/python2.7/dist-packages/apache_beam/transforms/sideinputs.py",
line 63, in __getitem__
_FilteringIterable(self._iterable, target_window), self._view_options)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pvalue.py", line
338, in _from_runtime_iterable
return options['data'].view_fn(it)
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/pvalue.py",
line 315, in <lambda>
File
"/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python_Verify/src/sdks/python/apache_beam/pvalue.py",
line 417, in _from_runtime_iterable
File
"/usr/local/lib/python2.7/dist-packages/apache_beam/transforms/sideinputs.py",
line 79, in __iter__
for wv in self._iterable:
File
"/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sideinputs.py",
line 180, in __iter__
raise self.reader_exceptions.get()
TypeError: argument 1 must be string or read-only buffer, not memoryview
[while running 'write/Write/WriteImpl/WriteBundles/WriteBundles']
```
https://builds.apache.org/job/beam_PostCommit_Python_Verify/5529/consoleFull
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 120675)
Time Spent: 14h 10m (was: 14h)
> Python 3 Support
> ----------------
>
> Key: BEAM-1251
> URL: https://issues.apache.org/jira/browse/BEAM-1251
> Project: Beam
> Issue Type: Improvement
> Components: sdk-py-core
> Reporter: Eyad Sibai
> Assignee: Robbe
> Priority: Trivial
> Time Spent: 14h 10m
> Remaining Estimate: 0h
>
> I have been trying to use google datalab with python3. As I see there are
> several packages that does not support python3 yet which google datalab
> depends on. This is one of them.
> https://github.com/GoogleCloudPlatform/DataflowPythonSDK/issues/6
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
