Barry Warsaw pushed to branch master at mailman / Mailman
Commits:
16903aae by Aurélien Bompard at 2015-09-15T01:33:36Z
Test the schema migrations with Alembic
- - - - -
d8e96d95 by Aurélien Bompard at 2015-09-15T01:33:37Z
Fixes for the review comments
- - - - -
2b091495 by Aurélien Bompard at 2015-09-15T01:33:37Z
Fix it with PostgreSQL
- - - - -
7 changed files:
- src/mailman/database/alembic/script.py.mako
- src/mailman/database/alembic/versions/16c2b25c7b_list_subscription_policy.py
- src/mailman/database/alembic/versions/33e1f5f6fa8_.py
-
src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
- src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
- + src/mailman/database/helpers.py
- + src/mailman/database/tests/test_migrations.py
Changes:
=====================================
src/mailman/database/alembic/script.py.mako
=====================================
--- a/src/mailman/database/alembic/script.py.mako
+++ b/src/mailman/database/alembic/script.py.mako
@@ -10,8 +10,9 @@ Create Date: ${create_date}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
-from alembic import op
import sqlalchemy as sa
+from alembic import op
+from mailman.database.helpers import is_sqlite, exists_in_db
${imports if imports else ""}
def upgrade():
=====================================
src/mailman/database/alembic/versions/16c2b25c7b_list_subscription_policy.py
=====================================
---
a/src/mailman/database/alembic/versions/16c2b25c7b_list_subscription_policy.py
+++
b/src/mailman/database/alembic/versions/16c2b25c7b_list_subscription_policy.py
@@ -14,14 +14,17 @@ from alembic import op
import sqlalchemy as sa
from mailman.database.types import Enum
+from mailman.database.helpers import is_sqlite, exists_in_db
from mailman.interfaces.mailinglist import SubscriptionPolicy
def upgrade():
### Update the schema
- op.add_column('mailinglist', sa.Column(
- 'subscription_policy', Enum(SubscriptionPolicy), nullable=True))
+ if not exists_in_db(op.get_bind(), 'mailinglist', 'subscription_policy'):
+ # SQLite may not have removed it when downgrading.
+ op.add_column('mailinglist', sa.Column(
+ 'subscription_policy', Enum(SubscriptionPolicy), nullable=True))
### Now migrate the data
# don't import the table definition from the models, it may break this
@@ -36,6 +39,6 @@ def upgrade():
def downgrade():
- if op.get_bind().dialect.name != 'sqlite':
+ if not is_sqlite(op.get_bind()):
# SQLite does not support dropping columns.
op.drop_column('mailinglist', 'subscription_policy')
=====================================
src/mailman/database/alembic/versions/33e1f5f6fa8_.py
=====================================
--- a/src/mailman/database/alembic/versions/33e1f5f6fa8_.py
+++ b/src/mailman/database/alembic/versions/33e1f5f6fa8_.py
@@ -33,6 +33,7 @@ __all__ = [
from alembic import op
import sqlalchemy as sa
+from mailman.database.helpers import is_sqlite
# revision identifiers, used by Alembic.
@@ -40,21 +41,33 @@ revision = '33e1f5f6fa8'
down_revision = '51b7f92bd06c'
+COLUMNS_TO_CHANGE = (
+ ('message', 'message_id_hash'),
+ ('message', 'path'),
+ ('pended', 'token'),
+ ('_request', 'data_hash'),
+ ('user', 'password'),
+ )
+
+
def upgrade():
- if op.get_bind().dialect.name != 'sqlite':
+ if is_sqlite(op.get_bind()):
# SQLite does not support altering columns.
- op.alter_column('message', 'message_id_hash', type_=sa.Unicode)
- op.alter_column('message', 'path', type_=sa.Unicode)
- op.alter_column('pended', 'token', type_=sa.Unicode)
- op.alter_column('_request', 'data_hash', type_=sa.Unicode)
- op.alter_column('user', 'password', type_=sa.Unicode)
+ return
+ for table, column in COLUMNS_TO_CHANGE:
+ op.alter_column(table, column, type_=sa.Unicode)
def downgrade():
- if op.get_bind().dialect.name != 'sqlite':
+ if is_sqlite(op.get_bind()):
# SQLite does not support altering columns.
- op.alter_column('message', 'message_id_hash', type_=sa.LargeBinary)
- op.alter_column('message', 'path', type_=sa.LargeBinary)
- op.alter_column('pended', 'token', type_=sa.LargeBinary)
- op.alter_column('_request', 'data_hash', type_=sa.LargeBinary)
- op.alter_column('user', 'password', type_=sa.LargeBinary)
+ return
+ for table, column in COLUMNS_TO_CHANGE:
+ if op.get_bind().dialect.name == 'postgresql':
+ # PostgreSQL needs the USING clause that Alembic does not support
+ # yet.
+ op.execute(('ALTER TABLE "{table}" ALTER COLUMN "{column}" '
+ 'TYPE BYTEA USING decode("{column}",
\'UTF8\')').format(
+ table=table, column=column))
+ else:
+ op.alter_column(table, column, type_=sa.LargeBinary)
=====================================
src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
=====================================
---
a/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
+++
b/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
@@ -29,6 +29,7 @@ down_revision = '33e1f5f6fa8'
from alembic import op
import sqlalchemy as sa
+from mailman.database.helpers import is_sqlite, exists_in_db
def upgrade():
@@ -40,16 +41,20 @@ def upgrade():
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('user_id', 'domain_id')
)
- op.add_column(
- 'user',
- sa.Column('is_server_owner', sa.Boolean(), nullable=True))
- if op.get_bind().dialect.name != 'sqlite':
+ if not exists_in_db(op.get_bind(), 'user', 'is_server_owner'):
+ # SQLite may not have removed it when downgrading.
+ op.add_column(
+ 'user',
+ sa.Column('is_server_owner', sa.Boolean(), nullable=True))
+ if not is_sqlite(op.get_bind()):
op.drop_column('domain', 'contact_address')
def downgrade():
- if op.get_bind().dialect.name != 'sqlite':
+ if not is_sqlite(op.get_bind()):
op.drop_column('user', 'is_server_owner')
+ if not exists_in_db(op.get_bind(), 'domain', 'contact_address'):
+ # SQLite may not have removed it.
op.add_column(
'domain',
sa.Column('contact_address', sa.VARCHAR(), nullable=True))
=====================================
src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
=====================================
--- a/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
+++ b/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
@@ -35,8 +35,9 @@ __all__ = [
]
-from alembic import op
import sqlalchemy as sa
+from alembic import op
+from mailman.database.helpers import is_sqlite, exists_in_db
# Revision identifiers, used by Alembic.
@@ -46,7 +47,7 @@ down_revision = None
def upgrade():
op.drop_table('version')
- if op.get_bind().dialect.name != 'sqlite':
+ if not is_sqlite(op.get_bind()):
# SQLite does not support dropping columns.
op.drop_column('mailinglist', 'acceptable_aliases_id')
op.create_index(op.f('ix_user__user_id'), 'user',
@@ -58,6 +59,7 @@ def downgrade():
op.create_table('version')
op.create_index('ix_user_user_id', 'user', ['_user_id'], unique=False)
op.drop_index(op.f('ix_user__user_id'), table_name='user')
- op.add_column(
- 'mailinglist',
- sa.Column('acceptable_aliases_id', sa.INTEGER(), nullable=True))
+ if not exists_in_db(op.get_bind(), 'mailinglist', 'acceptable_aliases_id'):
+ op.add_column(
+ 'mailinglist',
+ sa.Column('acceptable_aliases_id', sa.INTEGER(), nullable=True))
=====================================
src/mailman/database/helpers.py
=====================================
--- /dev/null
+++ b/src/mailman/database/helpers.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2015 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""Common database helpers."""
+
+__all__ = [
+ 'exists_in_db',
+ 'is_sqlite',
+ ]
+
+import sqlalchemy as sa
+
+
+def is_sqlite(bind):
+ return bind.dialect.name == 'sqlite'
+
+
+def exists_in_db(bind, tablename, columnname=None):
+ md = sa.MetaData()
+ md.reflect(bind=bind)
+ if columnname is None:
+ return tablename in md.tables
+ else:
+ return (
+ tablename in md.tables and
+ columnname in [c.name for c in md.tables[tablename].columns]
+ )
=====================================
src/mailman/database/tests/test_migrations.py
=====================================
--- /dev/null
+++ b/src/mailman/database/tests/test_migrations.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2015 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""Test database schema migrations with Alembic"""
+
+__all__ = [
+ 'TestMigrations',
+ ]
+
+
+import alembic.command
+import sqlalchemy as sa
+import unittest
+
+from mailman.config import config
+from mailman.database.alembic import alembic_cfg
+from mailman.database.model import Model
+from mailman.testing.layers import ConfigLayer
+
+
+
+class TestMigrations(unittest.TestCase):
+
+ layer = ConfigLayer
+
+ def setUp(self):
+ alembic.command.stamp(alembic_cfg, 'head')
+
+ def tearDown(self):
+ # Drop and restore a virgin database.
+ md = sa.MetaData(bind=config.db.engine)
+ md.reflect()
+ md.drop_all()
+ Model.metadata.create_all(config.db.engine)
+
+ def test_all_migrations(self):
+ script_dir = alembic.script.ScriptDirectory.from_config(alembic_cfg)
+ revisions = [sc.revision for sc in
+ script_dir.walk_revisions('base', 'heads')]
+ for revision in revisions:
+ alembic.command.downgrade(alembic_cfg, revision)
+ revisions.reverse()
+ for revision in revisions:
+ alembic.command.upgrade(alembic_cfg, revision)
View it on GitLab:
https://gitlab.com/mailman/mailman/compare/2f6a250ca38a8846c1d417c380d6311de35c3ee7...2b09149598144e49cf06e6293b3b9de5364bba2a
_______________________________________________
Mailman-checkins mailing list
Mailman-checkins@python.org
Unsubscribe:
https://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
