#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Owner: (none)
Type: Bug | Status: new
Component: Database layer | Version: 5.2
(models, ORM) |
Severity: Normal | Resolution:
Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------
Description changed by Marc DEBUREAUX:
Old description:
> Have this simple model:
>
> {{{
> class Link(models.Model):
> item = models.ForeignKey("Item", on_delete=models.CASCADE)
> extra = models.JSONField(blank=True, null=True)
> }}}
>
> And have this simple query:
>
> {{{
> Link.objects.all().values("extra__key",
> "item_id").annotate(count=Count("id"))
> }}}
>
> This throws the following stacktrace error since 5.2 release:
>
> {{{
> ---------------------------------------------------------------------------
> IndexError Traceback (most recent call
> last)
> File ~/python3.12/site-packages/IPython/core/formatters.py:770, in
> PlainTextFormatter.__call__(self, obj)
> 763 stream = StringIO()
> 764 printer = pretty.RepresentationPrinter(stream, self.verbose,
> 765 self.max_width, self.newline,
> 766 max_seq_length=self.max_seq_length,
> 767 singleton_pprinters=self.singleton_printers,
> 768 type_pprinters=self.type_printers,
> 769 deferred_pprinters=self.deferred_printers)
> --> 770 printer.pretty(obj)
> 771 printer.flush()
> 772 return stream.getvalue()
>
> File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in
> RepresentationPrinter.pretty(self, obj)
> 400 return meth(obj, self, cycle)
> 401 if (
> 402 cls is not object
> 403 # check if cls defines __repr__
> (...) 409 and callable(_safe_getattr(cls,
> "__repr__", None))
> 410 ):
> --> 411 return _repr_pprint(obj, self, cycle)
> 413 return _default_pprint(obj, self, cycle)
> 414 finally:
>
> File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in
> _repr_pprint(obj, p, cycle)
> 784 """A pprint that just redirects to the normal repr function."""
> 785 # Find newlines and replace them with p.break_()
> --> 786 output = repr(obj)
> 787 lines = output.splitlines()
> 788 with p.group():
>
> File ~/python3.12/site-packages/django/db/models/query.py:360, in
> QuerySet.__repr__(self)
> 359 def __repr__(self):
> --> 360 data = list(self[: REPR_OUTPUT_SIZE + 1])
> 361 if len(data) > REPR_OUTPUT_SIZE:
> 362 data[-1] = "...(remaining elements truncated)..."
>
> File ~/python3.12/site-packages/django/db/models/query.py:384, in
> QuerySet.__iter__(self)
> 369 def __iter__(self):
> 370 """
> 371 The queryset iterator protocol uses three nested iterators in
> the
> 372 default case:
> (...) 382 - Responsible for turning the rows into model
> objects.
> 383 """
> --> 384 self._fetch_all()
> 385 return iter(self._result_cache)
>
> File ~/python3.12/site-packages/django/db/models/query.py:1935, in
> QuerySet._fetch_all(self)
> 1933 def _fetch_all(self):
> 1934 if self._result_cache is None:
> -> 1935 self._result_cache = list(self._iterable_class(self))
> 1936 if self._prefetch_related_lookups and not
> self._prefetch_done:
> 1937 self._prefetch_related_objects()
>
> File ~/python3.12/site-packages/django/db/models/query.py:216, in
> ValuesIterable.__iter__(self)
> 210 names = [
> 211 *query.extra_select,
> 212 *query.values_select,
> 213 *query.annotation_select,
> 214 ]
> 215 indexes = range(len(names))
> --> 216 for row in compiler.results_iter(
> 217 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
> 218 ):
> 219 yield {names[i]: row[i] for i in indexes}
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in
> SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch,
> chunk_size)
> 1569 """Return an iterator over the results from executing this
> query."""
> 1570 if results is None:
> -> 1571 results = self.execute_sql(
> 1572 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
> 1573 )
> 1574 fields = [s[0] for s in self.select[0 : self.col_count]]
> 1575 converters = self.get_converters(fields)
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in
> SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
> 1607 result_type = result_type or NO_RESULTS
> 1608 try:
> -> 1609 sql, params = self.as_sql()
> 1610 if not sql:
> 1611 raise EmptyResultSet
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in
> SQLCompiler.as_sql(self, with_limits, with_col_aliases)
> 763 try:
> 764 combinator = self.query.combinator
> --> 765 extra_select, order_by, group_by = self.pre_sql_setup(
> 766 with_col_aliases=with_col_aliases or bool(combinator),
> 767 )
> 768 for_update_part = None
> 769 # Is a LIMIT/OFFSET clause needed?
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in
> SQLCompiler.pre_sql_setup(self, with_col_aliases)
> 79 def pre_sql_setup(self, with_col_aliases=False):
> 80 """
> 81 Do any necessary class setup immediately prior to producing
> SQL. This
> 82 is for things that can't necessarily be done in __init__
> because we
> 83 might not have all the pieces in place at that time.
> 84 """
> ---> 85 self.setup_query(with_col_aliases=with_col_aliases)
> 86 order_by = self.get_order_by()
> 87 self.where, self.having, self.qualify =
> self.query.where.split_having_qualify(
> 88 must_group_by=self.query.group_by is not None
> 89 )
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in
> SQLCompiler.setup_query(self, with_col_aliases)
> 72 if all(self.query.alias_refcount[a] == 0 for a in
> self.query.alias_map):
> 73 self.query.get_initial_alias()
> ---> 74 self.select, self.klass_info, self.annotation_col_map =
> self.get_select(
> 75 with_col_aliases=with_col_aliases,
> 76 )
> 77 self.col_count = len(self.select)
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in
> SQLCompiler.get_select(self, with_col_aliases)
> 284 # Reference to a column.
> 285 elif isinstance(expression, int):
> --> 286 expression = cols[expression]
> 287 # ColPairs cannot be aliased.
> 288 if isinstance(expression, ColPairs):
> }}}
New description:
**EDIT:** After further investigation, it seems the issue only happens if
the JSONField key is the first item of the group clause, it works fine if
it is alone or if another non-JSON field group clause is positioned first.
Have this simple model:
{{{
class Link(models.Model):
item = models.ForeignKey("Item", on_delete=models.CASCADE)
extra = models.JSONField(blank=True, null=True)
}}}
And have this simple query:
{{{
Link.objects.all().values("extra__key",
"item_id").annotate(count=Count("id"))
}}}
This throws the following stacktrace error since 5.2 release:
{{{
---------------------------------------------------------------------------
IndexError Traceback (most recent call
last)
File ~/python3.12/site-packages/IPython/core/formatters.py:770, in
PlainTextFormatter.__call__(self, obj)
763 stream = StringIO()
764 printer = pretty.RepresentationPrinter(stream, self.verbose,
765 self.max_width, self.newline,
766 max_seq_length=self.max_seq_length,
767 singleton_pprinters=self.singleton_printers,
768 type_pprinters=self.type_printers,
769 deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
771 printer.flush()
772 return stream.getvalue()
File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in
RepresentationPrinter.pretty(self, obj)
400 return meth(obj, self, cycle)
401 if (
402 cls is not object
403 # check if cls defines __repr__
(...) 409 and callable(_safe_getattr(cls,
"__repr__", None))
410 ):
--> 411 return _repr_pprint(obj, self, cycle)
413 return _default_pprint(obj, self, cycle)
414 finally:
File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in
_repr_pprint(obj, p, cycle)
784 """A pprint that just redirects to the normal repr function."""
785 # Find newlines and replace them with p.break_()
--> 786 output = repr(obj)
787 lines = output.splitlines()
788 with p.group():
File ~/python3.12/site-packages/django/db/models/query.py:360, in
QuerySet.__repr__(self)
359 def __repr__(self):
--> 360 data = list(self[: REPR_OUTPUT_SIZE + 1])
361 if len(data) > REPR_OUTPUT_SIZE:
362 data[-1] = "...(remaining elements truncated)..."
File ~/python3.12/site-packages/django/db/models/query.py:384, in
QuerySet.__iter__(self)
369 def __iter__(self):
370 """
371 The queryset iterator protocol uses three nested iterators in
the
372 default case:
(...) 382 - Responsible for turning the rows into model
objects.
383 """
--> 384 self._fetch_all()
385 return iter(self._result_cache)
File ~/python3.12/site-packages/django/db/models/query.py:1935, in
QuerySet._fetch_all(self)
1933 def _fetch_all(self):
1934 if self._result_cache is None:
-> 1935 self._result_cache = list(self._iterable_class(self))
1936 if self._prefetch_related_lookups and not self._prefetch_done:
1937 self._prefetch_related_objects()
File ~/python3.12/site-packages/django/db/models/query.py:216, in
ValuesIterable.__iter__(self)
210 names = [
211 *query.extra_select,
212 *query.values_select,
213 *query.annotation_select,
214 ]
215 indexes = range(len(names))
--> 216 for row in compiler.results_iter(
217 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
218 ):
219 yield {names[i]: row[i] for i in indexes}
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in
SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch,
chunk_size)
1569 """Return an iterator over the results from executing this
query."""
1570 if results is None:
-> 1571 results = self.execute_sql(
1572 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
1573 )
1574 fields = [s[0] for s in self.select[0 : self.col_count]]
1575 converters = self.get_converters(fields)
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in
SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
1607 result_type = result_type or NO_RESULTS
1608 try:
-> 1609 sql, params = self.as_sql()
1610 if not sql:
1611 raise EmptyResultSet
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in
SQLCompiler.as_sql(self, with_limits, with_col_aliases)
763 try:
764 combinator = self.query.combinator
--> 765 extra_select, order_by, group_by = self.pre_sql_setup(
766 with_col_aliases=with_col_aliases or bool(combinator),
767 )
768 for_update_part = None
769 # Is a LIMIT/OFFSET clause needed?
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in
SQLCompiler.pre_sql_setup(self, with_col_aliases)
79 def pre_sql_setup(self, with_col_aliases=False):
80 """
81 Do any necessary class setup immediately prior to producing
SQL. This
82 is for things that can't necessarily be done in __init__
because we
83 might not have all the pieces in place at that time.
84 """
---> 85 self.setup_query(with_col_aliases=with_col_aliases)
86 order_by = self.get_order_by()
87 self.where, self.having, self.qualify =
self.query.where.split_having_qualify(
88 must_group_by=self.query.group_by is not None
89 )
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in
SQLCompiler.setup_query(self, with_col_aliases)
72 if all(self.query.alias_refcount[a] == 0 for a in
self.query.alias_map):
73 self.query.get_initial_alias()
---> 74 self.select, self.klass_info, self.annotation_col_map =
self.get_select(
75 with_col_aliases=with_col_aliases,
76 )
77 self.col_count = len(self.select)
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in
SQLCompiler.get_select(self, with_col_aliases)
284 # Reference to a column.
285 elif isinstance(expression, int):
--> 286 expression = cols[expression]
287 # ColPairs cannot be aliased.
288 if isinstance(expression, ColPairs):
}}}
--
--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:3>
Django <https://code.djangoproject.com/>
The Web framework for perfectionists with deadlines.
--
You received this message because you are subscribed to the Google Groups
"Django updates" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to django-updates+unsubscr...@googlegroups.com.
To view this discussion visit
https://groups.google.com/d/msgid/django-updates/01070196502cc6b8-0ca456ed-111f-4cd1-9640-f57ec72657b5-000000%40eu-central-1.amazonses.com.
