Mottimo commented on issue #24247:
URL: https://github.com/apache/airflow/issues/24247#issuecomment-1147342618
First test: initialization successfully done on MySQL 5.7.36 (Macports )
using Airflow 2.3.1, starting from an empty schema.
Second test: dump of the current schema used by Airflow 2.2.5, import on
MySQL without errors then upgrade. The result is a failure with the following
error (already seen upgrading from 2.2.5 to 2.3.0):
`(airflow) [airflow@rabbit airflow]$ airflow db upgrade
DB: mysql+mysqldb://airflow:***@localdb:3306/airflow
Performing upgrade with database
mysql+mysqldb://airflow:***@localdb:3306/airflow
[2022-06-06 13:05:00,689] {db.py:1445} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl MySQLImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade f9da662e7089 ->
e655c0453f75, Add ``map_index`` column to TaskInstance to identify task-mapping,
and a ``task_map`` table to track mapping values from XCom.
Traceback (most recent call last):
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1705, in _execute_context
self.dialect.do_execute(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/default.py",
line 716, in do_execute
cursor.execute(statement, parameters)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/cursors.py",
line 206, in execute
res = self._query(query)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/cursors.py",
line 319, in _query
db.query(q)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/connections.py",
line 254, in query
_mysql.connection.query(self, query)
MySQLdb._exceptions.OperationalError: (1091, "Can't DROP
'task_reschedule_ti_fkey'; check that column/key exists")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/app/x01/airflow/environments/airflow/bin/airflow", line 8, in
<module>
sys.exit(main())
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/__main__.py",
line 38, in main
args.func(args)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/cli/cli_parser.py",
line 51, in command
return func(*args, **kwargs)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/utils/cli.py",
line 99, in wrapper
return f(*args, **kwargs)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/cli/commands/db_command.py",
line 82, in upgradedb
db.upgradedb(to_revision=to_revision, from_revision=from_revision,
show_sql_only=args.show_sql_only)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/utils/session.py",
line 77, in wrapper
return func(*args, session=session, **kwargs)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/utils/db.py",
line 1446, in upgradedb
command.upgrade(config, revision=to_revision or 'heads')
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/command.py",
line 320, in upgrade
script.run_env()
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/script/base.py",
line 563, in run_env
util.load_python_file(self.dir, "env.py")
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/util/pyfiles.py",
line 92, in load_python_file
module = load_module_py(module_id, path)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/util/pyfiles.py",
line 108, in load_module_py
spec.loader.exec_module(module) # type: ignore
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in
_call_with_frames_removed
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/migrations/env.py",
line 107, in <module>
run_migrations_online()
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/migrations/env.py",
line 101, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/runtime/environment.py",
line 851, in run_migrations
self.get_context().run_migrations(**kw)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/runtime/migration.py",
line 620, in run_migrations
step.migration_fn(**kw)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/airflow/migrations/versions/0100_2_3_0_add_taskmap_and_map_id_on_taskinstance.py",
line 49, in upgrade
batch_op.drop_index("idx_task_reschedule_dag_task_run")
File "/opt/python-bit/lib/python3.8/contextlib.py", line 120, in __exit__
next(self.gen)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/operations/base.py",
line 374, in batch_alter_table
impl.flush()
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/operations/batch.py",
line 108, in flush
fn(*arg, **kw)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/ddl/mysql.py",
line 153, in drop_constraint
super(MySQLImpl, self).drop_constraint(const)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/ddl/impl.py",
line 336, in drop_constraint
self._exec(schema.DropConstraint(const))
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/alembic/ddl/impl.py",
line 193, in _exec
return conn.execute(construct, multiparams)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1200, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py",
line 77, in _execute_on_connection
return connection._execute_ddl(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1290, in _execute_ddl
ret = self._execute_context(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1748, in _execute_context
self._handle_dbapi_exception(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1929, in _handle_dbapi_exception
util.raise_(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/util/compat.py",
line 211, in raise_
raise exception
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/base.py",
line 1705, in _execute_context
self.dialect.do_execute(
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/sqlalchemy/engine/default.py",
line 716, in do_execute
cursor.execute(statement, parameters)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/cursors.py",
line 206, in execute
res = self._query(query)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/cursors.py",
line 319, in _query
db.query(q)
File
"/app/x01/airflow/environments/airflow/lib/python3.8/site-packages/MySQLdb/connections.py",
line 254, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError)
(1091, "Can't DROP 'task_reschedule_ti_fkey'; check that column/key exists")
[SQL: ALTER TABLE task_reschedule DROP FOREIGN KEY task_reschedule_ti_fkey]
(Background on this error at: http://sqlalche.me/e/14/e3q8)
`
In my view it's not a matter of support of MariaDB, given that the schema is
exactly the same, and not a matter of DDL but a matter of the code used for the
upgrade. The migration scripts seem to not completely cover all possible
scenarios, previously the missing table and now the missing index. In facts,
this second time the code is doing:
_Excerpt from 0100_2_3_0_add_taskmap_and_map_id_on_taskinstance.py)_
`def upgrade():
42 """
43 Add ``map_index`` column to TaskInstance to identify task-mapping,
44 and a ``task_map`` table to track mapping values from XCom.
45 """
46 # We need to first remove constraints on task_reschedule since they
depend on task_instance.
47 with op.batch_alter_table("task_reschedule") as batch_op:
48 batch_op.drop_constraint("task_reschedule_ti_fkey", "foreignkey")
49 batch_op.drop_index("idx_task_reschedule_dag_task_run")`
Then, it tries to drop a foreign key without checking if that key is in
place or not. Fixing that foreign key another similar error is fired.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
