[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=17069117#comment-17069117
]
Daniel Imberman commented on AIRFLOW-351:
-
This issue has been moved to https://github.com/apache/airflow/issues/7929
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, operators, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Labels: subdag
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y =
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16903067#comment-16903067
]
Oleg Khavronin commented on AIRFLOW-351:
[~jackjack10]j, our Airflow instances run inside docker container (either
local, or in GKE). So, OS is Linux.
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, operators, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Labels: subdag
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16900772#comment-16900772
]
jack commented on AIRFLOW-351:
--
What operating system do you run Airflow on?
I suspect it has something to do with
[http://databasefaq.com/index.php/answer/114012/python-windows-multiprocessing-cherrypy-python-multiprocessing-multiprocessing-works-in-ubuntu-doesnt-in-windows]
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, operators, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Labels: subdag
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16820933#comment-16820933
]
Vianney FOUCAULT commented on AIRFLOW-351:
--
on a slack api post operator or a PythonOperator or on a PythonSensor
{code}
---
Node: airflow-webserver
---
Traceback (most recent call last):
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/app.py", line 1982,
in wsgi_app
response = self.full_dispatch_request()
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/app.py", line 1614,
in full_dispatch_request
rv = self.handle_user_exception(e)
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/app.py", line 1517,
in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/_compat.py", line
33, in reraise
raise value
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/app.py", line 1612,
in full_dispatch_request
rv = self.dispatch_request()
File "/data/venvs/airflow/lib/python3.5/site-packages/flask/app.py", line 1598,
in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/data/venvs/airflow/lib/python3.5/site-packages/flask_admin/base.py",
line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/data/venvs/airflow/lib/python3.5/site-packages/flask_admin/base.py",
line 368, in _run_view
return fn(self, *args, **kwargs)
File "/data/venvs/airflow/lib/python3.5/site-packages/flask_login/utils.py",
line 261, in decorated_view
return func(*args, **kwargs)
File "/data/venvs/airflow/lib/python3.5/site-packages/airflow/www/utils.py",
line 275, in wrapper
return f(*args, **kwargs)
File "/data/venvs/airflow/lib/python3.5/site-packages/airflow/www/utils.py",
line 322, in wrapper
return f(*args, **kwargs)
File "/data/venvs/airflow/lib/python3.5/site-packages/airflow/www/views.py",
line 1202, in clear
include_upstream=upstream)
File
"/data/venvs/airflow/lib/python3.5/site-packages/airflow/models/__init__.py",
line 3830, in sub_dag
dag = copy.deepcopy(self)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 166, in deepcopy
y = copier(memo)
File
"/data/venvs/airflow/lib/python3.5/site-packages/airflow/models/__init__.py",
line 3815, in __deepcopy__
setattr(result, k, copy.deepcopy(v, memo))
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 243, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 250, in _deepcopy_method
return type(x)(x.__func__, deepcopy(x.__self__, memo))
File "/data/venvs/airflow/lib/python3.5/copy.py", line 182, in deepcopy
y = _reconstruct(x, rv, 1, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 297, in _reconstruct
state = deepcopy(state, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 243, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 182, in deepcopy
y = _reconstruct(x, rv, 1, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 297, in _reconstruct
state = deepcopy(state, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 243, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 182, in deepcopy
y = _reconstruct(x, rv, 1, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 297, in _reconstruct
state = deepcopy(state, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 243, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 218, in _deepcopy_list
y.append(deepcopy(a, memo))
File "/data/venvs/airflow/lib/python3.5/copy.py", line 182, in deepcopy
y = _reconstruct(x, rv, 1, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 297, in _reconstruct
state = deepcopy(state, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 155, in deepcopy
y = copier(x, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 243, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
File "/data/venvs/airflow/lib/python3.5/copy.py", line 174, in deepcopy
rv = reductor(4)
TypeError: cannot serialize
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16820172#comment-16820172
]
Ash Berlin-Taylor commented on AIRFLOW-351:
---
[~vfoucault] Could you provide an updated stack trace and details about what
kind of operator(s) are failing to clear?
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16819896#comment-16819896
]
Vianney FOUCAULT commented on AIRFLOW-351:
--
I do have the issue in 1.10.3
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16807537#comment-16807537
]
jack commented on AIRFLOW-351:
--
[~vijaytdh] can you please check if the issue still present on the 1.10.3 test
version ?
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16807531#comment-16807531
]
Vijay Thakorlal commented on AIRFLOW-351:
-
Hi [~jackjack10], thanks for looking into this. We were running 1.7.1.3 but
moved to 1.8.1.1 and the issues seems to still persist in this version.
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16802561#comment-16802561
]
jack commented on AIRFLOW-351:
--
[~vijaytdh] are you running 1.9? According to
https://issues.apache.org/jira/browse/AIRFLOW-1972 your error might be caused
by retry_delay is set to a number. Check [~mjerdely] comment.
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16768750#comment-16768750
]
Tao Feng commented on AIRFLOW-351:
--
we do this in python_operator:
https://github.com/apache/airflow/blob/master/airflow/operators/python_operator.py#L77
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y =
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16768721#comment-16768721
]
Vijay Thakorlal commented on AIRFLOW-351:
-
In my case it's the RemoteSparkSubmit operator but from the stack trace it's
not clear what the attribute is - the stack trace is the same as the one in the
original issues except the last line says:
{code:java}
TypeError: can't pickle thread.lock objects
{code}
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16768684#comment-16768684
]
Tao Feng commented on AIRFLOW-351:
--
[~eronarn] , which operator do you encounter the issue? And which attribute
fails the deepcopy? Just want to let you know that you could put those
attribute in
[https://github.com/apache/airflow/blob/master/airflow/models/__init__.py#L2035]
to avoid deepcopy.
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16768492#comment-16768492
]
James Meickle commented on AIRFLOW-351:
---
Surprisingly enough, this issue still exists...
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line
[jira] [Commented] (AIRFLOW-351) Failed to clear downstream tasks
[
https://issues.apache.org/jira/browse/AIRFLOW-351?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16754410#comment-16754410
]
Vijay Thakorlal commented on AIRFLOW-351:
-
Just wondering if this issue is going to be addressed in a future release and
if there is a workaround that can be implemented? The PRs seemed to have gone
stale.
> Failed to clear downstream tasks
>
>
> Key: AIRFLOW-351
> URL: https://issues.apache.org/jira/browse/AIRFLOW-351
> Project: Apache Airflow
> Issue Type: Bug
> Components: models, subdag, webserver
>Affects Versions: 1.7.1.3
>Reporter: Adinata
>Priority: Major
> Attachments: dag_error.py, error.log, error_on_clear_dag.txt,
> ubuntu-14-packages.log, ubuntu-16-oops.log, ubuntu-16-packages.log
>
>
> {code}
> / ( () ) \___
> /( ( ( ) _)) ) )\
>(( ( )() ) ( ) )
> ((/ ( _( ) ( _) ) ( () ) )
> ( ( ( (_) ((( ) .((_ ) . )_
>( ( )( ( )) ) . ) ( )
> ( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
> ( ( ( ) ( ) ( )) ) _)( ) ) )
> ( ( ( \ ) ((_ ( ) ( ) ) ) ) )) ( )
> ( ( ( ( (_ ( ) ( _) ) ( ) ) )
> ( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
> (( ( )(( _) _) _(_ ( (_ )
>(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
>((__)\\||lll|l||/// \_))
> ( /(/ ( ) ) )\ )
> (( ( ( | | ) ) )\ )
>( /(| / ( )) ) ) )) )
> ( ( _(|)_) )
> ( ||\(|(|)|/|| )
> (|(||(||))
> ( //|/l|||)|\\ \ )
> (/ / // /|//\\ \ \ \ _)
> ---
> Node: 9889a7c79e9b
> ---
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in
> wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in
> full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1381, in
> handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1475, in
> full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1461, in
> dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 68,
> in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line
> 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in
> decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line
> 167, in wrapper
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line
> 1017, in clear
> include_upstream=upstream)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2870,
> in sub_dag
> dag = copy.deepcopy(self)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 2856,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
> y = copier(x, memo)
> File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
> y[deepcopy(key, memo)] = deepcopy(value, memo)
> File "/usr/lib/python2.7/copy.py", line 174, in deepcopy
> y = copier(memo)
> File "/usr/local/lib/python2.7/dist-packages/airflow/models.py", line 1974,
> in __deepcopy__
> setattr(result, k, copy.deepcopy(v, memo))
> File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
> y = _reconstruct(x, rv, 1, memo)
> File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
> state = deepcopy(state, memo)
> File
