JingsongLi commented on code in PR #10984:
URL: https://github.com/apache/paimon/pull/10984#discussion_r4478871303


##########
paimon-python/pypaimon/multimodal/primary_key_batch_lookup.py:
##########
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Share physical-position reads across primary-key vector query results."""
+
+from copy import copy
+
+import pyarrow as pa
+
+from pypaimon.execution.python.deletionvectors.deletion_vector import 
DeletionVector
+from pypaimon.execution.python.index.pk.primary_key_index_source_policy import 
should_read
+from pypaimon.globalindex.indexed_split import IndexedSplit
+from pypaimon.table.source.primary_key_scored_result import (
+    PrimaryKeyScoredResult, _partition_bytes, _ranges,
+)
+from pypaimon.utils.arrow_utils import zero_column_batch
+
+
+def _take(rows, indices):
+    if not rows.num_columns:
+        return pa.Table.from_batches([zero_column_batch(len(indices), 
rows.schema.metadata)])
+    return rows.take(pa.array(indices, type=pa.int64()))
+
+
+def read_primary_key_batch(query, results):
+    """Return shared lookup tables, or None when position mapping is 
unavailable."""
+    table = query._table
+    if (not table.is_primary_key_table or query._predicate is not None
+            or table.options.query_auth_enabled or 
table.options.native_read_enabled()
+            or not all(isinstance(result, PrimaryKeyScoredResult) for result 
in results)
+            or len({result.snapshot_id for result in results}) != 1):
+        return None
+    projection = query._effective_projection()
+    if projection and not query._configured_read_builder()._output_fields():
+        return None
+
+    groups = {}
+    selections = []
+    for result in results:
+        selected = []
+        for split in result.splits:
+            if not isinstance(split, IndexedSplit) or len(split.files) != 1:
+                return None
+            data = split.data_split()
+            file = data.files[0]
+            if not should_read(file):
+                return None
+            key = (_partition_bytes(data.partition), data.bucket, 
file.file_name)
+            positions = {position for r in split.row_ranges()
+                         for position in range(r.from_, r.to + 1)}
+            if key not in groups:
+                groups[key] = (data, set())
+            source, union = groups[key]
+            if source.files != data.files or source.data_deletion_files != 
data.data_deletion_files:
+                return None
+            union.update(positions)
+            selected.append((key, positions))
+        selections.append(selected)
+
+    lookup = copy(query)
+    # A shared file may contain more rows than any individual query's Top-K.
+    lookup._limit = None
+    reader = lookup._configured_read_builder().new_read()
+    shared = {}
+    for key, (data, positions) in groups.items():
+        ordered = sorted(positions)
+        deletion = data.data_deletion_files[0] if data.data_deletion_files 
else None
+        if deletion is not None:
+            dv = DeletionVector.read(table.file_io, deletion)
+            ordered = [position for position in ordered if not 
dv.is_deleted(position)]
+        rows = reader.to_arrow([IndexedSplit(data, _ranges(ordered))])

Review Comment:
   [P2] Preserve bounded multi-file read parallelism
   
   Each `to_arrow` call receives one split, which bypasses the existing 
`read.parallelism` path. With `read.parallelism=3`, I exercised two disjoint 
queries over two compacted Parquet files each. Independent lookups opened the 
same four files on reader workers, while the shared path opened all four 
serially on the caller thread. Disjoint Top-K results save no opens, so 
remote-file latency can regress. Please keep the existing path when no file is 
reused or use bounded parallelism across the shared file groups.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to