Hey, as discussed in https://code.djangoproject.com/ticket/31096, I have implemented a module regarding manipulating caching on Model instances loaded from QuerySets, and hot-replaced ManyToManyField.save_form_data() to cache the instance relations from a validated ModelMultipleChoiceField, keeping the newly modified ManyToMany relations cached on the validated instance when a form gets posted.
The argument for this in my case is, I have extended a ModelForm with mixins where after saving, I log the differences between the original and changed instance, which has the ManyToMany relations. One of the ManyToMany relation's __str__() contains a lookup that spans over two other related models. Hence, the logging of new/changed fields takes tons of DB queries when the instance coming from the ModelForm has uncached relations, much less their prefetched variants. As the ticket is closed there, I propose my module here for starting a debate on a better ManyToMany (and ForeignKey)relations handling, which my module is able to do. It was said in the ticket that the module itself is complicated, but in my humble opinion understanding the code of how the Form, ModelChoiceField and ManyToMany fields are wired together, is way more complicated. So all in all, it's just a question of viewpoint. I'll attach three modules, the one which I pasted into the ticket and two another that will/check return relations for traversing them in the server code. I hope to spark some debate and maybe provide something useful for the dev community, and maybe have these modules taken into the project. Mind you, this all here is voluntary. I'm already using these 3 modules in my production systems with huge speed improvements, and if nothing else, I'll opensource them and keep using them that way. Tests: since I have tests that are testing these modules wired up in relation with my system, unfortunately I can't post them here. Basically I test models and relations in my own project by evaluating the results of these functions, expecting some wired-up results. Cheers, -- László Károlyi http://linkedin.com/in/karolyi -- You received this message because you are subscribed to the Google Groups "Django developers (Contributions to Django itself)" group. To unsubscribe from this group and stop receiving emails from it, send an email to django-developers+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/django-developers/4908cdd9-9d9a-b9e9-80cb-3ac260a5375f%40karolyi.hu.
from django.db.models.base import Model
from .query_cache import is_query_prefetched
from .related_field import get_reverse_fk_relations
def remote_fks_exist(obj: Model, ignores: frozenset = frozenset()) -> bool:
"""
Return `True` if the passed Model instance `obj` is referred in
its remote `ForeignKey` relations.
"""
for rel_name in get_reverse_fk_relations(model=obj._meta.model):
if rel_name in ignores:
continue
relation = getattr(obj, rel_name)
if is_query_prefetched(relation=relation):
if relation.all():
return True
elif relation.exists():
return True
return False
from typing import Iterable, Optional
from django import VERSION
from django.db.models.base import Model
from django.db.models.fields.related import ManyToManyField
from django.db.models.fields.reverse_related import ManyToOneRel
from django.db.models.manager import Manager
from django.db.models.query import QuerySet
def invalidate_onetomany(objs: Iterable[Model], prefetch_keys: Iterable[str]):
"""
Invalidate one-to-many caches. These are remote `ForeignKey` and
`ManyToManyField` fields fetched with `prefetch_related()`.
"""
if VERSION[0] == 1 or VERSION[0] == 2:
for obj in objs:
if not hasattr(obj, '_prefetched_objects_cache'):
continue
for key in prefetch_keys:
if key not in obj._prefetched_objects_cache:
continue
del obj._prefetched_objects_cache[key]
def invalidate_manytoone(objs: Iterable[Model], field_names: Iterable[str]):
"""
Invalidate many-to-one caches. These are `ForeignKey` and
`OneToOneField` fields fetched with `select_related()` or
`prefetch_related()`.
"""
if VERSION[0] == 1:
for obj in objs:
for field_name in field_names:
if not is_fk_cached(obj=obj, field_name=field_name):
continue
del obj.__dict__[f'_{field_name}_cache']
elif VERSION[0] == 2:
for obj in objs:
for field_name in field_names:
if not is_fk_cached(obj=obj, field_name=field_name):
continue
del obj._state.fields_cache[field_name]
def get_prefetch_cache_key(relation: Manager) -> str:
'Return a key used in the prefetched cache for a relation.'
try:
# Works on ManyToMany
return relation.prefetch_cache_name
except AttributeError:
# Is a ForeignKey (OneToMany)
rel_field = relation.field.remote_field # type: ManyToOneRel
if rel_field.related_name:
return rel_field.related_name
if VERSION[0] == 1:
return rel_field.name
elif VERSION[0] == 2:
return f'{rel_field.name}_set'
def init_prefetch_cache(obj: Model):
'Init a prefetch cache on the model.'
if VERSION[0] == 1 or VERSION[0] == 2:
if hasattr(obj, '_prefetched_objects_cache'):
return
obj._prefetched_objects_cache = {}
def is_query_prefetched(relation: Manager) -> bool:
'Return `True` if the relation is prefetched.'
if VERSION[0] == 1 or VERSION[0] == 2:
obj = relation.instance
if not hasattr(obj, '_prefetched_objects_cache'):
return False
prefetch_cache_key = get_prefetch_cache_key(relation=relation)
return prefetch_cache_key in obj._prefetched_objects_cache
return False
def set_prefetch_cache(
relation: Manager, queryset: QuerySet, override: bool = True):
'Set prefetch cache on a `Model` for a relation.'
if is_query_prefetched(relation=relation) and not override:
return
obj = relation.instance
init_prefetch_cache(obj=obj)
if VERSION[0] == 1 or VERSION[0] == 2:
key = get_prefetch_cache_key(relation=relation)
obj._prefetched_objects_cache[key] = queryset
def is_queryresult_loaded(qs: QuerySet) -> bool:
'Return `True` if the query is loaded, `False` otherwise.'
if VERSION[0] == 1 or VERSION[0] == 2:
return qs._result_cache is not None
return False
def set_queryresult(qs: QuerySet, result: list, override: bool = True):
'Set result on a previously setup query.'
if VERSION[0] == 1 or VERSION[0] == 2:
if override or not is_queryresult_loaded(qs=qs):
qs._result_cache = result
def get_queryresult(qs: QuerySet) -> Optional[list]:
'Return the cached query result of the passed `QuerySet`.'
if VERSION[0] == 1 or VERSION[0] == 2:
return qs._result_cache
def is_fk_cached(obj: Model, field_name: str) -> bool:
'Return `True` if the `ForeignKey` field on the object is cached.'
if VERSION[0] == 1:
return hasattr(obj, f'_{field_name}_cache')
elif VERSION[0] == 2:
if getattr(obj, '_state', None) is None or \
getattr(obj._state, 'fields_cache', None) is None:
return False
return field_name in obj._state.fields_cache
return False
def set_fk_cache(
obj: Model, field_name: str, value: Model, override: bool = True):
"""
Set a cache on the `obj` for a `ForeignKey` field, override when
requested.
"""
if is_fk_cached(obj=obj, field_name=field_name) and not override:
return
if VERSION[0] == 1:
setattr(obj, f'_{field_name}_cache', value)
elif VERSION[0] == 2:
if getattr(obj, '_state', None) is None:
obj._state = dict()
if getattr(obj._state, 'fields_cache', None) is None:
obj._state.fields_cache = dict()
obj._state.fields_cache[field_name] = value
def del_fk_cache(obj: Model, field_name: str):
'Delete a cached `ForeignKey` on the `Model`.'
if not is_fk_cached(obj=obj, field_name=field_name):
return
if VERSION[0] == 1:
delattr(obj, f'_{field_name}_cache')
elif VERSION[0] == 2:
del obj._state.fields_cache
_old_m2m_savedata = ManyToManyField.save_form_data
def _save_m2m_form_data(
self: ManyToManyField, instance: Model, data: QuerySet):
_old_m2m_savedata(self=self, instance=instance, data=data)
set_prefetch_cache(
relation=getattr(instance, self.name), queryset=data, override=True)
ManyToManyField.save_form_data = _save_m2m_form_data
from dataclasses import dataclass
from functools import lru_cache
from typing import Dict, Tuple
from django import VERSION
from django.db.models.base import Model
from django.db.models.fields.related import ForeignKey, ManyToManyField
from django.db.models.fields.reverse_related import ManyToManyRel, ManyToOneRel
from .query_cache import get_prefetch_cache_key
@dataclass(frozen=True)
class RemoteForeignKeyResult(object):
"""
Result of a `ForeignKey` reverse relations lookup.
The name used for annotation/aggregation expressions is
`prefetch_cache_key`.
"""
# The model on the other side
model: Model
# The field on the original model
field: ForeignKey
# The cache key
prefetch_cache_key: str
# Name used for annotation
annotation_key: str
@dataclass(frozen=True)
class RemoteManyToManyResult(object):
"""
Result of a `ManyToMany` reverse relations lookup.
The name used for annotation/aggregation expressions is
`prefetch_cache_key`.
"""
# The field on the model
m2m_field: ManyToManyField
# The cache key
prefetch_cache_key: str
# The intermediate model
through: Model
# The model's key in the intermediate
through_fk_mine: ForeignKey
# Other model's key in the intermediate
through_fk_other: ForeignKey
@lru_cache(maxsize=None)
def get_reverse_fk_relations(
model: Model) -> Dict[str, RemoteForeignKeyResult]:
"""
Return a `dict` of `str-RemoteForeignKeyResult` relations.
That is, one-to-many relations where the passed model has
`ForeignKey`s pointing to it from remote models.
"""
result = dict()
related_fields = [
x for x in model._meta.get_fields()
if x.is_relation and x.auto_created and not x.concrete]
fk_fields = [x for x in related_fields if x.one_to_many or x.one_to_one]
for rel_field in fk_fields: # type: ManyToOneRel
related_name = rel_field.related_name or f'{rel_field.name}_set'
prefetch_cache_key = get_prefetch_cache_key(
relation=getattr(model, related_name))
result.update({related_name: RemoteForeignKeyResult(
model=rel_field.related_model, field=rel_field.field,
prefetch_cache_key=prefetch_cache_key,
annotation_key=rel_field.related_name or rel_field.name)})
return result
@lru_cache(maxsize=None)
def get_reverse_m2m_relations(
model: Model) -> Dict[str, RemoteManyToManyResult]:
"""
Return a `dict` of `str-RemoteManyToManyResult` relations.
That is, many-to-many relations where the passed model has
`ManyToManyField`s pointing to it from remote models.
"""
result = dict()
m2m_fields = [
x for x in model._meta.get_fields()
if x.is_relation and x.many_to_many]
for field in m2m_fields:
rel_field, m2m_field = (field, field.remote_field) \
if type(field) is ManyToManyRel else \
(field.remote_field, field) # type: ManyToManyRel, ManyToManyField
related_name = m2m_field.name
prefetch_cache_key = m2m_field.name
if type(field) is ManyToManyRel:
related_name = rel_field.related_name or f'{rel_field.name}_set'
prefetch_cache_key = rel_field.related_name or rel_field.name
fk_mine, fk_other = get_m2mthrough_fk_fields(
from_model=model, m2m_field=m2m_field)
result.update({related_name: RemoteManyToManyResult(
m2m_field=m2m_field, prefetch_cache_key=prefetch_cache_key,
through=rel_field.through,
through_fk_mine=fk_mine, through_fk_other=fk_other)})
return result
def get_m2mthrough_fk_fields(
m2m_field: ManyToManyField, from_model: Model
) -> Tuple[ForeignKey, ForeignKey]:
"""
Return the two `ForeignKey` sides from a through model of a
many-to-many relationship.
"""
from_name, to_name = \
(m2m_field.m2m_field_name(), m2m_field.m2m_reverse_field_name()) \
if from_model == m2m_field.model else \
(m2m_field.m2m_reverse_field_name(), m2m_field.m2m_field_name())
if VERSION[0] == 1:
m2m_model = m2m_field.rel.through
elif VERSION[0] == 2:
m2m_model = m2m_field.remote_field.through
fk_from = m2m_model._meta.get_field(from_name)
fk_to = m2m_model._meta.get_field(to_name)
return fk_from, fk_to
signature.asc
Description: OpenPGP digital signature
