jayzhan211 commented on code in PR #27604:
URL: https://github.com/apache/datafusion/pull/27604#discussion_r4656142718
##########
datafusion/fuzz/tests/suites/memory.rs:
##########
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Checks incremental memory accounting against recounting the retained arrays
+//! after each operation, including arrays that share buffers.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use arrow::array::{
+ ArrayRef, BooleanArray, DictionaryArray, Int32Array, StringArray,
StringViewArray,
+};
+use arrow::datatypes::Int32Type;
+use datafusion_common::Result;
+use datafusion_common::utils::memory::{
+ RecordBatchMemoryCounter, RecordBatchMemoryTracker,
+};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use test_utils::fuzz::FuzzTest;
+
+pub(crate) struct RecordBatchMemoryFuzzer;
+
+impl FuzzTest for RecordBatchMemoryFuzzer {
+ type Dataset = Vec<ArrayRef>;
+
+ fn name(&self) -> &str {
+ "record_batch_memory_tracker"
+ }
+
+ fn prepare_dataset(&self, _seed: u64) -> impl Future<Output =
Result<Self::Dataset>> {
Review Comment:
Optional: the dataset seed is unused, and the fixed arrays never hold more
than 16 distinct buffers. That means the tracker never moves to its overflow
`HashMap`, and no test covers that path. I changed the overflow `decrement` so
it never removes an entry, which makes a re-counted buffer add 0 bytes. All
`utils::memory` unit tests and this fuzzer still passed. Letting the dataset
seed add a few arrays catches that change within 0.02 s:
```diff
- fn prepare_dataset(&self, _seed: u64) -> impl Future<Output =
Result<Self::Dataset>> {
+ fn prepare_dataset(&self, seed: u64) -> impl Future<Output =
Result<Self::Dataset>> {
@@
- std::future::ready(Ok(vec![
+ let mut arrays = vec![
@@
Arc::new(BooleanArray::from(vec![true, false])),
- ]))
+ ];
+ // The tracker moves from inline slots to a HashMap past 16 distinct
+ // buffers; the fixed arrays above hold fewer.
+ let extra = StdRng::seed_from_u64(seed).random_range(0..=16);
+ arrays.extend(
+ (0..extra).map(|_| Arc::new(Int32Array::from(vec![0; 4])) as
ArrayRef),
+ );
+ std::future::ready(Ok(arrays))
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]