This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11460-7d2cbbab813cb5d880c2f1559b7d16c57853d669 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 53e7e369fa3db5cb560bf2a92748840e19b7d726 Author: Kosta Tarasov <[email protected]> AuthorDate: Thu Oct 15 22:24:52 2026 +0000 bench(parquet): try mimalloc for Arrow writer benchmarks (#11460) ## Which issue does this PR close? None. This is a sibling experiment to #11449. ## Rationale for this change Arrow writer benchmarks can show substantial allocator-dependent drift. Use mimalloc, consistent with the allocator used by DataFusion's CLI and benchmark harness, to evaluate whether benchmark repeatability improves. The existing same-commit comparisons showed lower tail drift with mimalloc: median 0.85%, p95 3.87%, and maximum 6.1%, compared with 1.52%, 11.46%, and 55.8% for the system allocator. These are initial observations from the benchmark runner, not proof that execution-order effects are eliminated. See the [comparison and discussion](https://github.com/apache/arrow-rs/pull/11460#issuecomment-6083801014). ## What changes are included in this PR? Use mimalloc as the global allocator for the `arrow_writer` benchmark executable, without a Linux-only restriction. The dependency is development-only; library consumers and other executables retain their existing allocators. Use default mimalloc settings. Purging remains configurable externally with `MIMALLOC_PURGE_DELAY`, including `-1` to retain freed pages for reuse. ## Are these changes tested? - Formatting and compilation with the benchmark runner's features passed. - Linux `primitive/default` smoke testing passed, including allocator diagnostics with purging disabled. - Benchmark-bot comparisons completed with default mimalloc, purging disabled, and the same mimalloc commit on both sides. - The ungated source files exactly match those tested in temporary draft #11564. On both [macOS](https://github.com/apache/arrow-rs/actions/runs/38624980817/job/116114845763) and [Windows](https://github.com/apache/arrow-rs/actions/runs/38624980817/job/116114845768), the release benchmark build, `primitive/default` smoke case, and workspace tests passed. Logs confirm mimalloc initialized and the smoke case executed successfully. - Formatting, spelling, and diff checks passed again for the gate-removal update. ## Are there any significant user-facing changes? None. --- Cargo.lock | 19 +++++++++++++++++++ parquet/Cargo.toml | 1 + parquet/benches/arrow_writer.rs | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 90dbef2da9..a791b610f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2159,6 +2159,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index"; checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2247,6 +2256,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index"; +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -2583,6 +2601,7 @@ dependencies = [ "hashbrown 0.17.1", "insta", "lz4_flex", + "mimalloc", "num-bigint 0.5.1", "num-integer", "num-traits", diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index d71120d9d7..b895fcc485 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -75,6 +75,7 @@ simdutf8 = { workspace = true , optional = true } ring = { version = "0.17", default-features = false, features = ["std"], optional = true } [dev-dependencies] +mimalloc = { version = "0.1", default-features = false } base64 = { version = "0.23", default-features = false, features = ["std"] } criterion = { workspace = true, default-features = false, features = ["async_futures"] } snap = { version = "1.0", default-features = false } diff --git a/parquet/benches/arrow_writer.rs b/parquet/benches/arrow_writer.rs index d399c0a593..ef479205ee 100644 --- a/parquet/benches/arrow_writer.rs +++ b/parquet/benches/arrow_writer.rs @@ -18,6 +18,11 @@ #[macro_use] extern crate criterion; +// Use MIMALLOC_PURGE_DELAY=-1 to retain freed pages for reuse when comparing +// benchmark execution orders. Set it before starting the benchmark executable. +#[global_allocator] +static BENCH_ALLOCATOR: mimalloc::MiMalloc = mimalloc::MiMalloc; + use arrow_array::builder::StringDictionaryBuilder; use criterion::{Bencher, Criterion, Throughput}; use parquet::arrow::ArrowWriter;
