Lee-W commented on code in PR #64972: URL: https://github.com/apache/airflow/pull/64972#discussion_r3062739338
########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker + return settings.engine.dialect.name + + +class TestDisableSqliteFkeys: + """Tests for :func:`disable_sqlite_fkeys`.""" + + def test_yields_op(self): + """The context manager yields the *op* object on every backend.""" + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + def test_sqlite_turns_off_fkeys(self): + """On SQLite, ``PRAGMA foreign_keys`` is OFF inside the context.""" + if _dialect() != "sqlite": + pytest.skip("SQLite-specific test") Review Comment: we can use `pytest.mark.skipif` ########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker Review Comment: ```suggestion if TYPE_CHECKING: assert settings.engine is not None # guaranteed by db_test marker ``` I think we usually do it for `assert` like this ########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker + return settings.engine.dialect.name + + +class TestDisableSqliteFkeys: + """Tests for :func:`disable_sqlite_fkeys`.""" + + def test_yields_op(self): + """The context manager yields the *op* object on every backend.""" + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + def test_sqlite_turns_off_fkeys(self): + """On SQLite, ``PRAGMA foreign_keys`` is OFF inside the context.""" + if _dialect() != "sqlite": + pytest.skip("SQLite-specific test") + + with settings.engine.connect() as conn: + # Ensure foreign_keys is ON before entering the context manager. + conn.execute(text("PRAGMA foreign_keys=ON")) + conn.commit() + + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + + with disable_sqlite_fkeys(op): + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 0, "foreign_keys should be OFF inside disable_sqlite_fkeys" + + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 1, "foreign_keys should be restored to ON after disable_sqlite_fkeys" + + def test_non_sqlite_is_noop(self): + """On non-SQLite backends, the context manager simply yields.""" + if _dialect() == "sqlite": + pytest.skip("Non-SQLite backends only") + + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + +class TestMysqlDropForeignkeyIfExists: + """Tests for :func:`mysql_drop_foreignkey_if_exists`.""" + + def test_drops_existing_fk(self): + """On MySQL, an existing foreign key is dropped.""" + if _dialect() != "mysql": Review Comment: same ########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker + return settings.engine.dialect.name + + +class TestDisableSqliteFkeys: + """Tests for :func:`disable_sqlite_fkeys`.""" + + def test_yields_op(self): + """The context manager yields the *op* object on every backend.""" + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + def test_sqlite_turns_off_fkeys(self): + """On SQLite, ``PRAGMA foreign_keys`` is OFF inside the context.""" + if _dialect() != "sqlite": + pytest.skip("SQLite-specific test") + + with settings.engine.connect() as conn: + # Ensure foreign_keys is ON before entering the context manager. + conn.execute(text("PRAGMA foreign_keys=ON")) + conn.commit() + + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + + with disable_sqlite_fkeys(op): + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 0, "foreign_keys should be OFF inside disable_sqlite_fkeys" + + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 1, "foreign_keys should be restored to ON after disable_sqlite_fkeys" + + def test_non_sqlite_is_noop(self): + """On non-SQLite backends, the context manager simply yields.""" + if _dialect() == "sqlite": + pytest.skip("Non-SQLite backends only") + + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + +class TestMysqlDropForeignkeyIfExists: + """Tests for :func:`mysql_drop_foreignkey_if_exists`.""" + + def test_drops_existing_fk(self): + """On MySQL, an existing foreign key is dropped.""" + if _dialect() != "mysql": + pytest.skip("MySQL-specific stored procedure test") + + with settings.engine.connect() as conn: + conn.execute(text("CREATE TABLE IF NOT EXISTS _test_mig_parent ( id INT PRIMARY KEY)")) + conn.execute( + text( + "CREATE TABLE IF NOT EXISTS _test_mig_child (" + " id INT PRIMARY KEY," + " parent_id INT," + " CONSTRAINT _test_mig_fk FOREIGN KEY (parent_id)" + " REFERENCES _test_mig_parent(id)" + ")" Review Comment: ```suggestion """ CREATE TABLE IF NOT EXISTS _test_mig_child ( id INT PRIMARY KEY, parent_id INT, CONSTRAINT _test_mig_fk FOREIGN KEY (parent_id) REFERENCES _test_mig_parent(id) )""" ``` ########## scripts/ci/prek/check_migration_patterns.py: ########## @@ -0,0 +1,384 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Static analysis checks for Alembic migration anti-patterns. + +These checks use AST analysis to detect common migration mistakes that can cause +silent failures, particularly on SQLite where ``PRAGMA foreign_keys`` has specific +requirements about transaction state. + +TODO: These checks are candidates for future Ruff rules (AIR category). + +Rules +===== + +MIG001 -- DML before ``disable_sqlite_fkeys`` +---------------------------------------------- + +**What it does:** +Detects ``op.execute()`` calls containing DML keywords (UPDATE, INSERT, DELETE) that +appear before a ``with disable_sqlite_fkeys(op):`` block in the same function. + +**Why is this bad:** +On SQLite, any DML statement triggers *autobegin*, starting a transaction. Once a +transaction is active, ``PRAGMA foreign_keys=off`` (issued by ``disable_sqlite_fkeys``) +becomes a no-op. This means foreign key checks remain enabled, and +``batch_alter_table`` (which recreates tables) may fail with foreign key constraint +violations. + +**Example (bad):** + +.. code-block:: python + + def upgrade(): + op.execute("UPDATE dag SET col = '' WHERE col IS NULL") # triggers autobegin + with disable_sqlite_fkeys(op): # PRAGMA is now a no-op! + with op.batch_alter_table("dag") as batch_op: + batch_op.alter_column("col", nullable=False) + +**Use instead:** + +.. code-block:: python + + def upgrade(): + with disable_sqlite_fkeys(op): + op.execute("UPDATE dag SET col = '' WHERE col IS NULL") + with op.batch_alter_table("dag") as batch_op: + batch_op.alter_column("col", nullable=False) + + +MIG002 -- DDL before ``disable_sqlite_fkeys`` +---------------------------------------------- + +**What it does:** +Detects any Alembic ``op.*`` call (DDL operations like ``op.add_column``, +``op.drop_column``, ``op.create_table``, ``op.drop_table``, ``op.create_index``, +``op.batch_alter_table``, etc.) that appears before a +``with disable_sqlite_fkeys(op):`` block in the same function. Excludes DML calls +covered by MIG001. + +**Why is this bad:** +DDL operations also trigger *autobegin* on SQLite, making the subsequent +``PRAGMA foreign_keys=off`` a no-op, exactly like DML. + +**Example (bad):** + +.. code-block:: python + + def upgrade(): + op.add_column("dag_run", sa.Column("created_at", ...)) # triggers autobegin + with disable_sqlite_fkeys(op): # PRAGMA is now a no-op! + with op.batch_alter_table("backfill") as batch_op: + batch_op.alter_column("col", nullable=True) + +**Use instead:** + +.. code-block:: python + + def upgrade(): + with disable_sqlite_fkeys(op): + op.add_column("dag_run", sa.Column("created_at", ...)) + with op.batch_alter_table("backfill") as batch_op: + batch_op.alter_column("col", nullable=True) + + +MIG003 -- DML without offline-mode guard Review Comment: I have never encountered this one, so I can't really tell 🤔 I don't disagree, just want to make sure. ########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker + return settings.engine.dialect.name + + +class TestDisableSqliteFkeys: + """Tests for :func:`disable_sqlite_fkeys`.""" + + def test_yields_op(self): + """The context manager yields the *op* object on every backend.""" + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + def test_sqlite_turns_off_fkeys(self): + """On SQLite, ``PRAGMA foreign_keys`` is OFF inside the context.""" + if _dialect() != "sqlite": + pytest.skip("SQLite-specific test") + + with settings.engine.connect() as conn: + # Ensure foreign_keys is ON before entering the context manager. + conn.execute(text("PRAGMA foreign_keys=ON")) + conn.commit() + + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + + with disable_sqlite_fkeys(op): + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 0, "foreign_keys should be OFF inside disable_sqlite_fkeys" + + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 1, "foreign_keys should be restored to ON after disable_sqlite_fkeys" + + def test_non_sqlite_is_noop(self): + """On non-SQLite backends, the context manager simply yields.""" + if _dialect() == "sqlite": + pytest.skip("Non-SQLite backends only") + Review Comment: same here. then we probably don't need to make _dialect a fixture. `_get_dialect` might be a better name? ########## airflow-core/tests/unit/migrations/test_migration_utils.py: ########## @@ -0,0 +1,183 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for ``airflow.migrations.utils``.""" + +from __future__ import annotations + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from airflow import settings +from airflow.migrations.utils import ( + disable_sqlite_fkeys, + ignore_sqlite_value_error, + mysql_drop_foreignkey_if_exists, +) + +pytestmark = pytest.mark.db_test + + +def _dialect() -> str: + assert settings.engine is not None # guaranteed by db_test marker + return settings.engine.dialect.name + + +class TestDisableSqliteFkeys: + """Tests for :func:`disable_sqlite_fkeys`.""" + + def test_yields_op(self): + """The context manager yields the *op* object on every backend.""" + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + def test_sqlite_turns_off_fkeys(self): + """On SQLite, ``PRAGMA foreign_keys`` is OFF inside the context.""" + if _dialect() != "sqlite": + pytest.skip("SQLite-specific test") + + with settings.engine.connect() as conn: + # Ensure foreign_keys is ON before entering the context manager. + conn.execute(text("PRAGMA foreign_keys=ON")) + conn.commit() + + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + + with disable_sqlite_fkeys(op): + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 0, "foreign_keys should be OFF inside disable_sqlite_fkeys" + + result = conn.execute(text("PRAGMA foreign_keys")).scalar() + assert result == 1, "foreign_keys should be restored to ON after disable_sqlite_fkeys" + + def test_non_sqlite_is_noop(self): + """On non-SQLite backends, the context manager simply yields.""" + if _dialect() == "sqlite": + pytest.skip("Non-SQLite backends only") + + with settings.engine.connect() as conn: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + with disable_sqlite_fkeys(op) as yielded: + assert yielded is op + + +class TestMysqlDropForeignkeyIfExists: + """Tests for :func:`mysql_drop_foreignkey_if_exists`.""" + + def test_drops_existing_fk(self): + """On MySQL, an existing foreign key is dropped.""" + if _dialect() != "mysql": + pytest.skip("MySQL-specific stored procedure test") + + with settings.engine.connect() as conn: + conn.execute(text("CREATE TABLE IF NOT EXISTS _test_mig_parent ( id INT PRIMARY KEY)")) + conn.execute( + text( + "CREATE TABLE IF NOT EXISTS _test_mig_child (" + " id INT PRIMARY KEY," + " parent_id INT," + " CONSTRAINT _test_mig_fk FOREIGN KEY (parent_id)" + " REFERENCES _test_mig_parent(id)" + ")" + ) + ) + conn.commit() + + try: + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + mysql_drop_foreignkey_if_exists("_test_mig_fk", "_test_mig_child", op) + conn.commit() + + count = conn.execute( + text( + "SELECT COUNT(*) FROM information_schema.TABLE_CONSTRAINTS " Review Comment: same format as the above one would be easier to read -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
