#23979: Multi-db test fails to run because of fixture containing M2M field
(dumped
with --natural-foreign)
-------------------------------------+-------------------------------------
Reporter: ej | Owner: nobody
Type: Bug | Status: new
Component: Core | Version: 1.7
(Serialization) | Keywords: multi-db, m2m, fixture,
Severity: Normal | natural key
Triage Stage: Unreviewed | Has patch: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------
I noticed this issue in Django 1.5.10 but I've managed to replicate it in
Django 1.7.1 as well.
I have a multi-db configuration but when I run a test case that uses a
fixture containing many-to-many field (dumped using --natural-foreign
option), it throws an error because it's trying to query a table in the
secondary DB that only exists in the default DB when installing the
fixture.
Looking at this code, it looks like Django tries install the fixture for
all configured DB:
Line 899 (django/test/testcases.py):
{{{
for db_name in self._databases_names(include_mirrors=False):
if self.fixtures:
try:
call_command('loaddata', *self.fixtures,
**{
'verbosity': 0,
'commit': False,
'database': db_name,
'skip_checks': True,
})
except Exception:
self._fixture_teardown()
raise
}}}
...which is fine, but then here during serialization, it's making a query
using the requested db without checking with "allow_migrate" first:
Line 112: django/core/serializers/python.py
{{{
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__') and not isinstance(value,
six.text_type):
return
field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_text(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v:
smart_text(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
}}}
The line
`field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk`
doesn't check if the related model actually exists (or allowed) in `db`,
causing a DeserializationError when fixture is being installed during unit
test. In my example, I have a model that has a Many-to-Many reference to
"Permission" table. I serialize this model with "dumpdata" command using
"--natural-foreign" option.
However, when I run the test, the error I got was:
{{{
(django1.7)$ ./manage.py test
Creating test database for alias 'default'...
Creating test database for alias 'misc'...
E
======================================================================
ERROR: test_something (book.tests.MyTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/test/testcases.py", line 182, in __call__
self._pre_setup()
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/test/testcases.py", line 754, in _pre_setup
self._fixture_setup()
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/test/testcases.py", line 907, in _fixture_setup
'skip_checks': True,
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/management/__init__.py", line 115, in call_command
return klass.execute(*args, **defaults)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/management/base.py", line 338, in execute
output = self.handle(*args, **options)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/management/commands/loaddata.py", line 61, in handle
self.loaddata(fixture_labels)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/management/commands/loaddata.py", line 91, in
loaddata
self.load_label(fixture_label)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/management/commands/loaddata.py", line 142, in
load_label
for obj in objects:
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/serializers/json.py", line 81, in Deserializer
six.reraise(DeserializationError, DeserializationError(e),
sys.exc_info()[2])
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/serializers/json.py", line 75, in Deserializer
for obj in PythonDeserializer(objects, **options):
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/serializers/python.py", line 122, in Deserializer
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/core/serializers/python.py", line 117, in m2m_convert
return
field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/contrib/auth/models.py", line 35, in get_by_natural_key
model),
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/manager.py", line 92, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/query.py", line 351, in get
num = len(clone)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/query.py", line 122, in __len__
self._fetch_all()
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/query.py", line 966, in _fetch_all
self._result_cache = list(self.iterator())
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/query.py", line 265, in iterator
for row in compiler.results_iter():
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/sql/compiler.py", line 700, in results_iter
for rows in self.execute_sql(MULTI):
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/models/sql/compiler.py", line 786, in execute_sql
cursor.execute(sql, params)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/home/ejaury/venvs/django1.7/local/lib/python2.7/site-
packages/django/db/backends/sqlite3/base.py", line 485, in execute
return Database.Cursor.execute(self, query, params)
DeserializationError: Problem installing fixture
'/scratch/django/1.7/mybook/book/fixtures/permset_test.json': no such
table: auth_permission
}}}
Here's my setup (note that I have simplified this example from my actual
code, but it's still reproducible with this simplified code):
router.py
{{{
class MyRouter(object):
def db_for_read(self, model, **hints):
return None
def db_for_write(self, model, **hints):
return None
def allow_relation(self, obj1, obj2, **hints):
return True
def allow_migrate(self, db, model):
return (db == 'default')
}}}
database settings
{{{
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
},
'misc': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db_misc.sqlite3'),
},
}
DATABASE_ROUTERS = ['router.MyRouter']
}}}
models.py
{{{
from django.db import models
from django.contrib.auth.models import Permission
class PermSet(models.Model):
name = models.CharField(max_length=100)
permissions = models.ManyToManyField(Permission)
class AdminProfile(models.Model):
name = models.CharField(max_length=100)
perm_sets = models.ManyToManyField(PermSet)
}}}
tests.py
{{{
from django.test import TestCase
from book.models import PermSet
class MyTest(TestCase):
fixtures = ['permset_test.json']
multi_db = True
def test_something(self):
pass
}}}
This is my fixture:
permset_test.json
{{{
[
{
"fields": {
"name": "test",
"permissions": [
[
"change_logentry",
"admin",
"logentry"
],
[
"delete_logentry",
"admin",
"logentry"
]
]
},
"model": "book.permset",
"pk": 1
}
]
...which I dumped using this command:
{{{
(django1.7)$ ./manage.py dumpdata book.PermSet --indent=4 --natural-
foreign > permset_test.json
}}}
--
Ticket URL: <https://code.djangoproject.com/ticket/23979>
Django <https://code.djangoproject.com/>
The Web framework for perfectionists with deadlines.
--
You received this message because you are subscribed to the Google Groups
"Django updates" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/django-updates/045.6b0e3fd45968507cb8a22ec2bbc291b0%40djangoproject.com.
For more options, visit https://groups.google.com/d/optout.
