CannonFodder created BEAM-12592:
-----------------------------------
Summary: using flink runner docker environment raise NameError
name 'datetime' is not defined
Key: BEAM-12592
URL: https://issues.apache.org/jira/browse/BEAM-12592
Project: Beam
Issue Type: Bug
Components: cross-language
Affects Versions: 2.31.0, 2.30.0
Reporter: CannonFodder
I got from datetime import dateim, and can run this transform in direct runner
or flink runner loopback environment.
but when I try to run this transform in python flink runner docker environment,
it raised error.
Environment:
centos 7
anaconda
python 3.8.8
flink 1.13
beam 2.30.0 and beam 2.31.0 (tried on both, got same error)
python 3.8
java 1.8.0_292
official docker image from docker hub
apache/beam_python3.8_sdk 2.30.0 and 2.31.0
apache/beam_java8_sdk 2.30.0 and 2.31.0
{code:python}
| 'ConvertEnTime' >> beam.Map(lambda x: (x[1],datetime.strptime(x[3], '%Y-%m-%d
%H:%M:%S'))).with_output_types(Tuple[str, datetime])#fromisoformat
{code}
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 292, in _execute
response = task()
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 365, in <lambda>
lambda: self.create_worker().do_instruction(request), request)
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 609, in do_instruction
return getattr(self, request_type)(
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 647, in process_bundle
bundle_processor.process_bundle(instruction_id))
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 1000, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File
"/usr/local/lib/python3.8/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 229, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/worker/operations.py", line 356, in
apache_beam.runners.worker.operations.Operation.output
File "apache_beam/runners/worker/operations.py", line 358, in
apache_beam.runners.worker.operations.Operation.output
File "apache_beam/runners/worker/operations.py", line 220, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 717, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 718, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1235, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1300, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1233, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 581, in
apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1395, in
apache_beam.runners.common._OutputProcessor.process_outputs
File "apache_beam/runners/worker/operations.py", line 220, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 717, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 718, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1235, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1300, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1233, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 762, in
apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 885, in
apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1395, in
apache_beam.runners.common._OutputProcessor.process_outputs
File "apache_beam/runners/worker/operations.py", line 220, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 717, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 718, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1235, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1300, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1233, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 581, in
apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1395, in
apache_beam.runners.common._OutputProcessor.process_outputs
File "apache_beam/runners/worker/operations.py", line 220, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
File "apache_beam/runners/worker/operations.py", line 717, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/worker/operations.py", line 718, in
apache_beam.runners.worker.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1235, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1315, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
File "/usr/local/lib/python3.8/site-packages/future/utils/__init__.py", line
446, in raise_with_traceback
raise exc.with_traceback(traceback)
File "apache_beam/runners/common.py", line 1233, in
apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 582, in
apache_beam.runners.common.SimpleInvoker.invoke_process
File
"/root/anaconda3/lib/python3.8/site-packages/apache_beam/transforms/core.py",
line 1570, in <lambda>
wrapper = lambda x: [fn(x)]
File "/home/Workspace/PythonApplicationTraining/BeamKafkaRead.py", line 95,
in <lambda>
NameError: name 'datetime' is not defined [while running 'ConvertEnTime']
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
