leaves12138 commented on code in PR #9050:
URL: https://github.com/apache/paimon/pull/9050#discussion_r3722098984
##########
paimon-python/pypaimon/globalindex/data_evolution_global_index_scanner.py:
##########
@@ -103,24 +106,33 @@ def _create_evaluator(self, fields, file_io, index_path,
index_files):
options = self._options
def readers_function(field: DataField) ->
Collection[GlobalIndexReader]:
+ groups = []
group = index_metas.get(field.id)
if group is not None:
- return _create_readers(
- file_io, index_path, group.metas, field, executor, options)
+ groups.append(group)
extra_groups = extra_index_metas.get(field.id)
- if not extra_groups:
+ if extra_groups:
+ groups.extend(
+ extra_group
+ for extra_group in extra_groups
+ if extra_group not in groups
+ )
+ if not groups:
return []
+ if len(groups) == 1:
+ return _create_readers(
+ file_io, index_path, groups[0].metas, field, executor,
options)
union_coverage = Range.sort_and_merge_overlap(
[
range_key
- for group in extra_groups
+ for group in groups
for range_key in group.coverage_ranges
],
True,
)
readers = []
- for group in extra_groups:
+ for group in groups:
pad_ranges = _exclude_ranges(union_coverage,
group.coverage_ranges)
readers.extend(
_create_readers(
Review Comment:
Verified this on my side. Reproduced with a dedicated BTree index on `c`
over [0,4] plus an `es-index(a, c)` extra-field group over [5,9]: the base
branch returned the BTree result, while the original commits raised
`ValueError` from `_create_inner_readers` (silent full-scan fallback in
`FileScanner._eval_global_index`, hard failure in the vector raw pre-filter).
The capability-aware filtering in feb8c8a fixes it — the repro returns the
BTree result again, and since the unreadable files also drop out of
`_coverage`, ranges [5,9] correctly become unindexed fallback instead of being
claimed as indexed. The new regression test asserts the right contract.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
