#30541: Django MultiDB tests not loading fixtures as expected
---------------------------------------------+------------------------
               Reporter:  Vackar Afzal       |          Owner:  nobody
                   Type:  Bug                |         Status:  new
              Component:  Testing framework  |        Version:  2.2
               Severity:  Normal             |       Keywords:
           Triage Stage:  Unreviewed         |      Has patch:  0
    Needs documentation:  0                  |    Needs tests:  0
Patch needs improvement:  0                  |  Easy pickings:  0
                  UI/UX:  0                  |
---------------------------------------------+------------------------
 I've recently upgraded from Django 2.0 to Django 2.2 and have found the
 fixture loading logic appears to have changed. The core issue seems to be
 related to the introduction of `databases`

 Given the following test case:

 {{{
 class BaseTestCase(TestCase, TestUtilsMixin):

     databases = '__all__'
     fixtures = [
         'data_x1.default.yaml',
         'data_x2.default.yaml',
         'data_x3.default.yaml',
         'data_x4.default.yaml',
         'data_x5.default.yaml',
         'data_x6.default.yaml'
     ]

 }}}

 I would expect data_xx fixtures to only to be loaded into the 'default'
 alias, but it appears to be loading into all connections defined in
 `DATABASES`, resulting in the following error

 {{{
 Error
 Traceback (most recent call last):
   File "django/db/backends/utils.py", line 84, in _execute
     return self.cursor.execute(sql, params)
   File "django/db/backends/oracle/base.py", line 510, in execute
     return self.cursor.execute(query, self._param_generator(params))
 cx_Oracle.DatabaseError: ORA-00942: table or view does not exist

 The above exception was the direct cause of the following exception:

 Traceback (most recent call last):
   File "django/core/serializers/pyyaml.py", line 73, in Deserializer
     yield from PythonDeserializer(yaml.load(stream, Loader=SafeLoader),
 **options)
   File "django/core/serializers/python.py", line 147, in Deserializer
     obj = base.build_instance(Model, data, using)
   File "django/core/serializers/base.py", line 266, in build_instance
     default_manager.db_manager(db).get_by_natural_key(*natural_key).pk
   File "managers.py", line 15, in get_by_natural_key
     return self.get(name=name)
   File "django/db/models/manager.py", line 82, in manager_method
     return getattr(self.get_queryset(), name)(*args, **kwargs)
   File "django/db/models/query.py", line 402, in get
     num = len(clone)
   File "django/db/models/query.py", line 256, in __len__
     self._fetch_all()
   File "django/db/models/query.py", line 1242, in _fetch_all
     self._result_cache = list(self._iterable_class(self))
   File "django/db/models/query.py", line 55, in __iter__
     results = compiler.execute_sql(chunked_fetch=self.chunked_fetch,
 chunk_size=self.chunk_size)
   File "django/db/models/sql/compiler.py", line 1100, in execute_sql
     cursor.execute(sql, params)
   File "raven/contrib/django/client.py", line 127, in execute
     return real_execute(self, sql, params)
   File "django/db/backends/utils.py", line 67, in execute
     return self._execute_with_wrappers(sql, params, many=False,
 executor=self._execute)
   File "django/db/backends/utils.py", line 76, in _execute_with_wrappers
     return executor(sql, params, many, context)
   File "django/db/backends/utils.py", line 84, in _execute
     return self.cursor.execute(sql, params)
   File "django/db/utils.py", line 89, in __exit__
     raise dj_exc_value.with_traceback(traceback) from exc_value
   File "django/db/backends/utils.py", line 84, in _execute
     return self.cursor.execute(sql, params)
   File "django/db/backends/oracle/base.py", line 510, in execute
     return self.cursor.execute(query, self._param_generator(params))
 django.db.utils.DatabaseError: ORA-00942: table or view does not exist

 The above exception was the direct cause of the following exception:

 Traceback (most recent call last):
   File "django/test/testcases.py", line 1131, in setUpClass
     call_command('loaddata', *cls.fixtures, **{'verbosity': 0, 'database':
 db_name})
   File "django/core/management/__init__.py", line 148, in call_command
     return command.execute(*args, **defaults)
   File "django/core/management/base.py", line 364, in execute
     output = self.handle(*args, **options)
   File "django/core/management/commands/loaddata.py", line 72, in handle
     self.loaddata(fixture_labels)
   File "django/core/management/commands/loaddata.py", line 114, in
 loaddata
     self.load_label(fixture_label)
   File "django/core/management/commands/loaddata.py", line 172, in
 load_label
     for obj in objects:
   File "django/core/serializers/pyyaml.py", line 77, in Deserializer
     raise DeserializationError() from exc
 }}}



 I've hacked together a workaround by overriding setUpClass as follows:

 {{{
     @classmethod
     def setUpClass(cls):
         if not cls._databases_support_transactions():
             return
         cls.cls_atomics = cls._enter_atomics()

         if cls.fixtures:
             for db_name in cls._databases_names(include_mirrors=False):
                 for fixture in cls.fixtures:
                     load_data = True
                     fixture_sections = fixture.split('.')

                     # Very naive hack to see if a connection alias is in
 the fixture
                     if len(fixture_sections) == 3 and fixture_sections[1]
 != db_name:
                         load_data = False

                     if load_data:
                         try:
                             call_command('loaddata', fixture,
 **{'verbosity': 0, 'database': db_name})
                         except Exception:
                             cls._rollback_atomics(cls.cls_atomics)
                             cls._remove_databases_failures()
                             raise
         try:
             cls.setUpTestData()
         except Exception:
             cls._rollback_atomics(cls.cls_atomics)
             cls._remove_databases_failures()
             raise
 }}}


 But this has it's own issues. If I use `databases = '__all__'` this error
 is thrown
 {{{
 Error
 Traceback (most recent call last):
   File "/Users/vafzal/anaconda3/envs/centaur/lib/python3.7/site-
 packages/django/db/utils.py", line 166, in ensure_defaults
     conn = self.databases[alias]
 KeyError: '_'
 }}}

 If I use `databases = {'__all__'}` this error is thrown
 {{{
 Error
 Traceback (most recent call last):
   File "/Users/vafzal/anaconda3/envs/centaur/lib/python3.7/site-
 packages/django/db/utils.py", line 166, in ensure_defaults
     conn = self.databases[alias]
 KeyError: '__all__'
 }}}

 Instead, I have to use:
 {{{
   databases = {'default', 'other_conn', ...}
 }}}

 But unless I list all connections in DATABASES I get this error:
 {{{
 Error
 Traceback (most recent call last):
   File "tests/base.py", line 253, in tearDownClass
     cls._remove_databases_failures()
   File "django/test/testcases.py", line 240, in _remove_databases_failures
     setattr(connection, name, method.wrapped)
 AttributeError: 'function' object has no attribute 'wrapped'
 }}}

 The least hacky solution I've found to this problem is to do this:

 {{{
   class BaseTestCase(TestCase):
     databases = '__all__'

     default_fixtures = [
         'data_x1.default.yaml',
         'data_x2.default.yaml',
         'data_x3.default.yaml',
         'data_x4.default.yaml',
         'data_x5.default.yaml',
         'data_x6.default.yaml'
     ]

     @classmethod
     def setUpClass(cls):
         super().setUpClass()
         call_command('loaddata', *cls.default_fixtures, **{'verbosity': 0,
 'database': 'default'})
 }}}

 Is this a bug, or am I simply not initialising the tests correctly?

-- 
Ticket URL: <https://code.djangoproject.com/ticket/30541>
Django <https://code.djangoproject.com/>
The Web framework for perfectionists with deadlines.

-- 
You received this message because you are subscribed to the Google Groups 
"Django updates" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to django-updates+unsubscr...@googlegroups.com.
To post to this group, send email to django-updates@googlegroups.com.
To view this discussion on the web visit 
https://groups.google.com/d/msgid/django-updates/054.9763a046d99314c5e03fe4f5995619f7%40djangoproject.com.
For more options, visit https://groups.google.com/d/optout.

Reply via email to