[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16355956#comment-16355956
]
James Meickle commented on AIRFLOW-351:
---------------------------------------
Ran into this via a very unhelpful error message in the web UI (attached log).
I got a better message on the command line: `can't pickle lru.LRU objects`
However, I didn't spot this anywhere in the documentation. It would be really
helpful to explain that this applies even if you are not using the task
pickling features. Even better, I'd love to see example code for good patterns
to support tasks that do require references to these objects.
[^error.log]
> Failed to clear downstream tasks
> --------------------------------
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
> Affects Versions: Airflow 1.7.1.3
> Reporter: Adinata
> Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> ____/ ( ( ) ) \___
> /( ( ( ) _ )) ) )\
> (( ( )( ) ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) (( ( ) .((_ ) . )_
> ( ( ) ( ( ) ) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ( (_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _ ) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )( ( _ ) _) _(_ ( (_ )
> (_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
> ((__) \\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> ( ( ( ( | | ) ) )\ )
> ( /(| / ( )) ) ) )) )
> ( ( ((((_(|)_))))) )
> ( ||\(|(|)|/|| )
> ( |(||(||)|||| )
> ( //|/l|||)|\\ \ )
> (/ / // /|//||||\\ \ \ \ _)
> -------------------------------------------------------------------------------
> Node: 9889a7c79e9b
> -------------------------------------------------------------------------------
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 230, in _deepcopy_list
> y.append(deepcopy(a, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 329, in _reconstruct
> y = callable(*args)
> File "/usr/lib/python2.7/copy_reg.py", line 93, in __newobj__
> return cls.__new__(cls, *args)
> TypeError: object.__new__(thread.lock) is not safe, use thread.lock.__new__()
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
