Hi All,
We are running airflow version 1.9 in LocalExecutor mode. We are observing that
scheduler is crashed after few hours with below stack logs(Seems to be an issue
with Mysql Connection. Is there any fix or workaround for this)
Traceback (most recent call last):
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line
371, in helper
pickle_dags)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py",
line 50, in wrapper
result = func(*args, **kwargs)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line
1762, in process_file
dag.sync_to_db()
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py",
line 50, in wrapper
result = func(*args, **kwargs)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/models.py",
line 3806, in sync_to_db
DagModel).filter(DagModel.dag_id == self.dag_id).first()
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2825, in first
ret = list(self[0:1])
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2617, in __getitem__
return list(res)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2925, in __iter__
return self._execute_and_instances(context)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2946, in _execute_and_instances
close_with_result=True)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2955, in _get_bind_args
**kw
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line
2937, in _connection_from_session
conn = self.session.connection(**kw)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 1035, in connection
execution_options=execution_options)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 1040, in _connection_for_bind
engine, execution_options)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 409, in _connection_for_bind
conn = bind.contextual_connect()
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 2123, in contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 2158, in _wrap_pool_connect
return fn()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 403, in connect
return _ConnectionFairy._checkout(self)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 788, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 532, in checkout
rec = pool._do_get()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 1193, in _do_get
self._dec_overflow()
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py",
line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 1190, in _do_get
return self._create_connection()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 350, in _create_connection
return _ConnectionRecord(self)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 477, in __init__
self.__connect(first_connect_check=True)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py",
line 671, in __connect
connection = pool._invoke_creator(self)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py",
line 106, in connect
return dialect.connect(*cargs, **cparams)
File
"/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py",
line 410, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/__init__.py",
line 86, in Connect
return Connection(*args, **kwargs)
File
"/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line
209, in __init__
self._server_version = tuple([ numeric_part(n) for n in
self.get_server_info().split('.')[:2] ])
File
"/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line
69, in numeric_part
m = re_numeric_part.match(s)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py",
line 73, in sigint_handler
sys.exit(0)
SystemExit: 0
