[
https://issues.apache.org/jira/browse/BEAM-13388?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17462772#comment-17462772
]
Valentyn Tymofieiev commented on BEAM-13388:
--------------------------------------------
{noformat}
> self)
E
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow
pipeline failed. State: FAILED, Error:
E Traceback (most recent call last):
E File "apache_beam/runners/common.py", line 1198, in
apache_beam.runners.common.DoFnRunner.process
E File "apache_beam/runners/common.py", line 536, in
apache_beam.runners.common.SimpleInvoker.invoke_process
E File "apache_beam/runners/common.py", line 1361, in
apache_beam.runners.common._OutputProcessor.process_outputs
E File "apache_beam/runners/worker/operations.py", line 214, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
E File "apache_beam/runners/worker/operations.py", line 178, in
apache_beam.runners.worker.operations.ConsumerSet.update_counters_start
E File "apache_beam/runners/worker/opcounters.py", line 211, in
apache_beam.runners.worker.opcounters.OperationCounters.update_from
E File "apache_beam/runners/worker/opcounters.py", line 250, in
apache_beam.runners.worker.opcounters.OperationCounters.do_sample
E File "apache_beam/coders/coder_impl.py", line 1407, in
apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 1418, in
apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 1175, in
apache_beam.coders.coder_impl.SequenceCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 359, in
apache_beam.coders.coder_impl.FastPrimitivesCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 423, in
apache_beam.coders.coder_impl.FastPrimitivesCoderImpl.encode_to_stream
E File "apache_beam/coders/coder_impl.py", line 263, in
apache_beam.coders.coder_impl.CallbackCoderImpl.encode_to_stream
E File
"/usr/local/lib/python3.6/site-packages/apache_beam/coders/coders.py", line
800, in <lambda>
E lambda x: dumps(x, protocol), pickle.loads)
E _pickle.PicklingError: Can't pickle <class 'Finding'>: attribute
lookup Finding on __main__ failed
E
E During handling of the above exception, another exception occurred:
E
E Traceback (most recent call last):
E File
"/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line
644, in do_work
E work_executor.execute()
E File
"/usr/local/lib/python3.6/site-packages/dataflow_worker/executor.py", line 208,
in execute
E op.start()
E File "dataflow_worker/native_operations.py", line 38, in
dataflow_worker.native_operations.NativeReadOperation.start
E File "dataflow_worker/native_operations.py", line 39, in
dataflow_worker.native_operations.NativeReadOperation.start
E File "dataflow_worker/native_operations.py", line 44, in
dataflow_worker.native_operations.NativeReadOperation.start
E File "dataflow_worker/native_operations.py", line 54, in
dataflow_worker.native_operations.NativeReadOperation.start
E File "apache_beam/runners/worker/operations.py", line 348, in
apache_beam.runners.worker.operations.Operation.output
E File "apache_beam/runners/worker/operations.py", line 215, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
E File "apache_beam/runners/worker/operations.py", line 707, in
apache_beam.runners.worker.operations.DoOperation.process
E File "apache_beam/runners/worker/operations.py", line 708, in
apache_beam.runners.worker.operations.DoOperation.process
E File "apache_beam/runners/common.py", line 1200, in
apache_beam.runners.common.DoFnRunner.process
E File "apache_beam/runners/common.py", line 1281, in
apache_beam.runners.common.DoFnRunner._reraise_augmented
E File "apache_beam/runners/common.py", line 1198, in
apache_beam.runners.common.DoFnRunner.process
E File "apache_beam/runners/common.py", line 536, in
apache_beam.runners.common.SimpleInvoker.invoke_process
E File "apache_beam/runners/common.py", line 1361, in
apache_beam.runners.common._OutputProcessor.process_outputs
E File "apache_beam/runners/worker/operations.py", line 214, in
apache_beam.runners.worker.operations.SingletonConsumerSet.receive
E File "apache_beam/runners/worker/operations.py", line 178, in
apache_beam.runners.worker.operations.ConsumerSet.update_counters_start
E File "apache_beam/runners/worker/opcounters.py", line 211, in
apache_beam.runners.worker.opcounters.OperationCounters.update_from
E File "apache_beam/runners/worker/opcounters.py", line 250, in
apache_beam.runners.worker.opcounters.OperationCounters.do_sample
E File "apache_beam/coders/coder_impl.py", line 1407, in
apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 1418, in
apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 1175, in
apache_beam.coders.coder_impl.SequenceCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 359, in
apache_beam.coders.coder_impl.FastPrimitivesCoderImpl.get_estimated_size_and_observables
E File "apache_beam/coders/coder_impl.py", line 423, in
apache_beam.coders.coder_impl.FastPrimitivesCoderImpl.encode_to_stream
E File "apache_beam/coders/coder_impl.py", line 263, in
apache_beam.coders.coder_impl.CallbackCoderImpl.encode_to_stream
E File
"/usr/local/lib/python3.6/site-packages/apache_beam/coders/coders.py", line
800, in <lambda>
E lambda x: dumps(x, protocol), pickle.loads)
E _pickle.PicklingError: Can't pickle <class 'Finding'>: attribute
lookup Finding on __main__ failed [while running
'InspectForDetails/ParDo(_InspectFn)']
apache_beam/runners/dataflow/dataflow_runner.py:1636: DataflowRuntimeException
{noformat}
Such errors could happen when main session is not saved or when there is a
mismatch between the dependency versions at submission time and at execution
time (on DF workers). Looking further.
> [beam_PostCommit_Python38]
> [apache_beam.ml.gcp.cloud_dlp_it_test.CloudDLPIT.test_deidentification]
> Failure summary
> ------------------------------------------------------------------------------------------------------------------
>
> Key: BEAM-13388
> URL: https://issues.apache.org/jira/browse/BEAM-13388
> Project: Beam
> Issue Type: Bug
> Components: test-failures
> Reporter: Yichi Zhang
> Assignee: Valentyn Tymofieiev
> Priority: P1
> Labels: currently-failing
> Fix For: 2.35.0
>
> Time Spent: 11h 10m
> Remaining Estimate: 0h
>
> _Use this form to file an issue for test failure:_
> * [Jenkins
> Job|https://ci-beam.apache.org/job/beam_PostCommit_Python38/2012/testReport/apache_beam.ml.gcp.cloud_dlp_it_test/CloudDLPIT/test_deidentification/]
> * [Gradle Build Scan|TODO]
> * [Test source code|TODO]
> Initial investigation:
> {code:java}
> E
> apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException:
> Dataflow pipeline failed. State: FAILED, Error:
> E Traceback (most recent call last):
> E File "apache_beam/runners/common.py", line 1239, in
> apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
> E File "apache_beam/runners/common.py", line 465, in
> apache_beam.runners.common.DoFnInvoker.invoke_setup
> E File
> "/usr/local/lib/python3.8/site-packages/apache_beam/ml/gcp/cloud_dlp.py",
> line 193, in setup
> E 'parent': self.client.project_path(self.project)
> E AttributeError: 'DlpServiceClient' object has no attribute
> 'project_path'
> E
> E During handling of the above exception, another exception occurred:
> E
> E Traceback (most recent call last):
> E File
> "/usr/local/lib/python3.8/site-packages/dataflow_worker/batchworker.py", line
> 644, in do_work
> E work_executor.execute()
> E File
> "/usr/local/lib/python3.8/site-packages/dataflow_worker/executor.py", line
> 208, in execute
> E op.start()
> E File "apache_beam/runners/worker/operations.py", line 699, in
> apache_beam.runners.worker.operations.DoOperation.start
> E File "apache_beam/runners/worker/operations.py", line 701, in
> apache_beam.runners.worker.operations.DoOperation.start
> E File "apache_beam/runners/worker/operations.py", line 702, in
> apache_beam.runners.worker.operations.DoOperation.start
> E File "apache_beam/runners/worker/operations.py", line 300, in
> apache_beam.runners.worker.operations.Operation.start
> E File "apache_beam/runners/worker/operations.py", line 306, in
> apache_beam.runners.worker.operations.Operation.start
> E File "apache_beam/runners/worker/operations.py", line 648, in
> apache_beam.runners.worker.operations.DoOperation.setup
> E File "apache_beam/runners/worker/operations.py", line 697, in
> apache_beam.runners.worker.operations.DoOperation.setup
> E File "apache_beam/runners/common.py", line 1245, in
> apache_beam.runners.common.DoFnRunner.setup
> E File "apache_beam/runners/common.py", line 1241, in
> apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
> E File "apache_beam/runners/common.py", line 1281, in
> apache_beam.runners.common.DoFnRunner._reraise_augmented
> E File "apache_beam/runners/common.py", line 1239, in
> apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
> E File "apache_beam/runners/common.py", line 465, in
> apache_beam.runners.common.DoFnInvoker.invoke_setup
> E File
> "/usr/local/lib/python3.8/site-packages/apache_beam/ml/gcp/cloud_dlp.py",
> line 193, in setup
> E 'parent': self.client.project_path(self.project)
> E AttributeError: 'DlpServiceClient' object has no attribute
> 'project_path' [while running 'MaskDetectedDetails/ParDo(_DeidentifyFn)']
> {code}
> Seems google-cloud-dlp package has backward uncompatible change.
>
> (Add any investigation notes so far)
> ----
> _After you've filled out the above details, please [assign the issue to an
> individual|https://beam.apache.org/contribute/postcommits-guides/index.html#find_specialist].
> Assignee should [treat test failures as
> high-priority|https://beam.apache.org/contribute/postcommits-policies/#assigned-failing-test],
> helping to fix the issue or find a more appropriate owner. See [Apache Beam
> Post-Commit
> Policies|https://beam.apache.org/contribute/postcommits-policies]._
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
