tooptoop4 opened a new issue #11715:
URL: https://github.com/apache/airflow/issues/11715
airflow 1.10.12, mysql 5.7 metastore, rbac, no serialized dags, intermittent
error found 3 times in the logs:
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 589, in execute
uow,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 236, in save_obj
update,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 1028, in _emit_update_statements
% (table.description, len(records), rows)
sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'job' expected
to update 1 row(s); 0 were matched.
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 589, in execute
uow,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 236, in save_obj
update,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 1028, in _emit_update_statements
% (table.description, len(records), rows)
sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'job' expected
to update 1 row(s); 0 were matched.
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 589, in execute
uow,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 236, in save_obj
update,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 1028, in _emit_update_statements
% (table.description, len(records), rows)
sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'job' expected
to update 1 row(s); 0 were matched.
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 589, in execute
uow,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 236, in save_obj
update,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 1028, in _emit_update_statements
% (table.description, len(records), rows)
sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'job' expected
to update 1 row(s); 0 were matched.
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 589, in execute
uow,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 236, in save_obj
update,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/persistence.py",
line 1028, in _emit_update_statements
% (table.description, len(records), rows)
sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'job' expected
to update 1 row(s); 0 were matched.
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 37, in <module>
args.func(args)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/cli.py", line
76, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 579, in run
_run(args, dag, ti)
File "/home/ec2-user/venv/lib/python3.7/site-packages/airflow/bin/cli.py",
line 475, in _run
run_job.run()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 218, in run
self._execute()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/local_task_job.py",
line 105, in _execute
self.heartbeat()
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/jobs/base_job.py",
line 173, in heartbeat
previous_heartbeat = self.latest_heartbeat
File "/usr/lib64/python3.7/contextlib.py", line 119, in __exit__
next(self.gen)
File
"/home/ec2-user/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 45,
in create_session
session.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1042, in commit
self.transaction.commit()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 504, in commit
self._prepare_impl()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in _prepare_impl
self.session.flush()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2523, in flush
self._flush(objects)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2664, in _flush
transaction.rollback(_capture_exception=True)
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 69, in __exit__
exc_value, with_traceback=exc_tb,
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/util/compat.py",
line 182, in raise_
raise exception
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2624, in _flush
flush_context.execute()
File
"/home/ec2-user/venv/lib64/python3.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 422, in execute
rec.execute(self)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
