jerry-024 commented on code in PR #720:
URL: https://github.com/apache/paimon-rust/pull/720#discussion_r3794980659
##########
crates/paimon/src/vindex/range_reader.rs:
##########
@@ -319,17 +344,28 @@ impl VindexFileReader {
});
}
- for batch in merged.chunks(RANGE_READ_CONCURRENCY) {
- let ranges: Vec<_> = batch.iter().map(|merged|
merged.range.clone()).collect();
- let fetched = self.fetch_range_batch(&ranges)?;
- for (merged_range, data) in batch.iter().zip(fetched) {
- for &request_index in &merged_range.request_indices {
- let request = &mut requests[request_index];
- let start = (request.pos - merged_range.range.start) as
usize;
- request
- .buf
- .copy_from_slice(&data[start..start +
request.buf.len()]);
- }
+ if let Some(stats) = &self.stats {
+ stats
+ .read_many_merged_ranges
+ .fetch_add(merged.len() as u64, Ordering::Relaxed);
+ }
+ if let Some(stats) = &self.stats {
+ stats.read_many_chunks.fetch_add(1, Ordering::Relaxed);
+ stats
+ .read_many_chunk_sizes
+ .lock()
+ .unwrap()
+ .push(merged.len());
+ }
+ let ranges: Vec<_> = merged.iter().map(|merged|
merged.range.clone()).collect();
+ let fetched = self.fetch_range_batch(&ranges)?;
+ for (merged_range, data) in merged.iter().zip(fetched) {
+ for &request_index in &merged_range.request_indices {
+ let request = &mut requests[request_index];
Review Comment:
<!-- dlf-review -->
**[MAJOR]** The concurrency limit no longer bounds one `pread` memory
working set.
`read_many` now passes every merged range to `fetch_range_batch`, and
`try_join_all` retains every completed `Bytes` until the slowest read finishes.
The semaphore limits active I/O only, so a request with many disjoint ranges
can still retain O(total response bytes) in addition to the destination
buffers. Please use a bounded rolling queue (for example `FuturesUnordered`),
consume completed results promptly, and preserve ordering with request indices.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
