This is an automated email from the ASF dual-hosted git repository.
ytyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 853eee09b3 Add benchmark utility to profile peak memory usage (#16814)
853eee09b3 is described below
commit 853eee09b33ca6500fd6ca00f1a82d9fdfec68ff
Author: ding-young <lsyh...@snu.ac.kr>
AuthorDate: Thu Jul 31 13:35:43 2025 +0900
Add benchmark utility to profile peak memory usage (#16814)
* add benchmark utility to profile memory usage
* get memory stats from mimalloc, not procfs
* support more benchmarks
* update benchmarks/README and refactor
* fix sort-tpch output format & taplo format
* add e2e test & comments
* update description to major page fault
---------
Co-authored-by: Yongting You <2010you...@gmail.com>
---
Cargo.lock | 545 +++++++++++++++++++++-----------------
benchmarks/Cargo.toml | 3 +
benchmarks/README.md | 61 +++++
benchmarks/src/bin/mem_profile.rs | 360 +++++++++++++++++++++++++
benchmarks/src/clickbench.rs | 10 +-
benchmarks/src/h2o.rs | 19 +-
benchmarks/src/imdb/mod.rs | 3 +
benchmarks/src/imdb/run.rs | 15 +-
benchmarks/src/sort_tpch.rs | 16 +-
benchmarks/src/tpch/mod.rs | 3 +
benchmarks/src/tpch/run.rs | 15 +-
benchmarks/src/util/memory.rs | 57 ++++
benchmarks/src/util/mod.rs | 2 +
13 files changed, 844 insertions(+), 265 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 34501f8ac2..bab581eabf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -61,9 +61,9 @@ dependencies = [
[[package]]
name = "adler2"
-version = "2.0.0"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "adler32"
@@ -149,9 +149,9 @@ checksum =
"4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
-version = "0.6.18"
+version = "0.6.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -164,36 +164,36 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.10"
+version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
[[package]]
name = "anstyle-parse"
-version = "0.2.6"
+version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
-version = "1.1.2"
+version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.7"
+version = "3.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
+checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
dependencies = [
"anstyle",
- "once_cell",
+ "once_cell_polyfill",
"windows-sys 0.59.0",
]
@@ -295,7 +295,7 @@ dependencies = [
"chrono",
"chrono-tz",
"half",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
"num",
]
@@ -617,9 +617,9 @@ checksum =
"1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
-version = "1.4.0"
+version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-config"
@@ -665,9 +665,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.13.1"
+version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7"
+checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -675,9 +675,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.29.0"
+version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079"
+checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff"
dependencies = [
"bindgen",
"cc",
@@ -1075,7 +1075,7 @@ dependencies = [
"bitflags 2.9.1",
"cexpr",
"clang-sys",
- "itertools 0.10.5",
+ "itertools 0.12.1",
"lazy_static",
"lazycell",
"log",
@@ -1251,9 +1251,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.17.0"
+version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
+checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
[[package]]
name = "bytecheck"
@@ -1345,9 +1345,9 @@ checksum =
"37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.23"
+version = "1.2.30"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"
+checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
dependencies = [
"jobserver",
"libc",
@@ -1365,9 +1365,9 @@ dependencies = [
[[package]]
name = "cfg-if"
-version = "1.0.0"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
[[package]]
name = "cfg_aliases"
@@ -1435,7 +1435,7 @@ checksum =
"0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
- "libloading 0.8.7",
+ "libloading 0.8.8",
]
[[package]]
@@ -1451,9 +1451,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.41"
+version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
+checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1461,9 +1461,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.41"
+version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
+checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
dependencies = [
"anstream",
"anstyle",
@@ -1485,15 +1485,15 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.7.4"
+version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "clipboard-win"
-version = "5.4.0"
+version = "5.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892"
+checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4"
dependencies = [
"error-code",
]
@@ -1509,9 +1509,9 @@ dependencies = [
[[package]]
name = "colorchoice"
-version = "1.0.3"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "comfy-table"
@@ -1520,7 +1520,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a"
dependencies = [
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.1",
]
[[package]]
@@ -1532,7 +1532,7 @@ dependencies = [
"encode_unicode",
"libc",
"once_cell",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.1",
"windows-sys 0.59.0",
]
@@ -1568,9 +1568,9 @@ dependencies = [
[[package]]
name = "const_panic"
-version = "0.2.12"
+version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e"
+checksum = "b98d1483e98c9d67f341ab4b3915cfdc54740bd6f5cccc9226ee0535d86aa8fb"
[[package]]
name = "constant_time_eq"
@@ -1580,9 +1580,9 @@ checksum =
"7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation"
-version = "0.10.0"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
@@ -1629,9 +1629,9 @@ dependencies = [
[[package]]
name = "crc32fast"
-version = "1.4.2"
+version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
@@ -1645,7 +1645,7 @@ dependencies = [
"anes",
"cast",
"ciborium",
- "clap 4.5.41",
+ "clap 4.5.42",
"criterion-plot",
"futures",
"is-terminal",
@@ -1710,9 +1710,9 @@ checksum =
"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
-version = "0.2.3"
+version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
@@ -1761,6 +1761,12 @@ version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2"
+[[package]]
+name = "cty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
+
[[package]]
name = "darling"
version = "0.20.11"
@@ -1898,11 +1904,13 @@ dependencies = [
"datafusion-proto",
"env_logger",
"futures",
+ "libmimalloc-sys",
"log",
"mimalloc",
"object_store",
"parquet",
"rand 0.9.2",
+ "regex",
"serde",
"serde_json",
"snmalloc-rs",
@@ -1966,7 +1974,7 @@ dependencies = [
"async-trait",
"aws-config",
"aws-credential-types",
- "clap 4.5.41",
+ "clap 4.5.42",
"ctor",
"datafusion",
"dirs",
@@ -2653,7 +2661,7 @@ dependencies = [
"bigdecimal",
"bytes",
"chrono",
- "clap 4.5.41",
+ "clap 4.5.42",
"datafusion",
"datafusion-spark",
"datafusion-substrait",
@@ -2770,7 +2778,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -2824,9 +2832,9 @@ checksum =
"92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
-version = "1.0.19"
+version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
+checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "educe"
@@ -2909,12 +2917,12 @@ checksum =
"877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
-version = "0.3.12"
+version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
+checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -2959,7 +2967,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
dependencies = [
"cfg-if",
- "rustix 1.0.7",
+ "rustix 1.0.8",
"windows-sys 0.59.0",
]
@@ -3064,9 +3072,9 @@ dependencies = [
[[package]]
name = "fs-err"
-version = "3.1.0"
+version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa"
+checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683"
dependencies = [
"autocfg",
]
@@ -3222,7 +3230,7 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi 0.11.1+wasi-snapshot-preview1",
"wasm-bindgen",
]
@@ -3267,9 +3275,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.10"
+version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5"
+checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785"
dependencies = [
"atomic-waker",
"bytes",
@@ -3316,9 +3324,9 @@ dependencies = [
[[package]]
name = "hashbrown"
-version = "0.15.3"
+version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
+checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
dependencies = [
"allocator-api2",
"equivalent",
@@ -3342,9 +3350,9 @@ checksum =
"2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
-version = "0.5.1"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "hex"
@@ -3482,11 +3490,10 @@ dependencies = [
[[package]]
name = "hyper-rustls"
-version = "0.27.5"
+version = "0.27.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
+checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
dependencies = [
- "futures-util",
"http 1.3.1",
"hyper",
"hyper-util",
@@ -3513,19 +3520,23 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.12"
+version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cf9f1e950e0d9d1d3c47184416723cf29c0d1f93bd8cccf37e4beb6b44f31710"
+checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
dependencies = [
+ "base64 0.22.1",
"bytes",
"futures-channel",
+ "futures-core",
"futures-util",
"http 1.3.1",
"http-body 1.0.1",
"hyper",
+ "ipnet",
"libc",
+ "percent-encoding",
"pin-project-lite",
- "socket2 0.5.9",
+ "socket2 0.6.0",
"tokio",
"tower-service",
"tracing",
@@ -3701,7 +3712,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
dependencies = [
"equivalent",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
"serde",
]
@@ -3714,7 +3725,7 @@ dependencies = [
"console",
"number_prefix",
"portable-atomic",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.1",
"web-time",
]
@@ -3758,9 +3769,9 @@ checksum =
"8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "io-uring"
-version = "0.7.8"
+version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
+checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
dependencies = [
"bitflags 2.9.1",
"cfg-if",
@@ -3773,6 +3784,16 @@ version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+[[package]]
+name = "iri-string"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
[[package]]
name = "is-terminal"
version = "0.4.16"
@@ -3799,6 +3820,15 @@ dependencies = [
"either",
]
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itertools"
version = "0.13.0"
@@ -3825,9 +3855,9 @@ checksum =
"4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jiff"
-version = "0.2.14"
+version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"
+checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
dependencies = [
"jiff-static",
"log",
@@ -3838,9 +3868,9 @@ dependencies = [
[[package]]
name = "jiff-static"
-version = "0.2.14"
+version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"
+checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
dependencies = [
"proc-macro2",
"quote",
@@ -3991,12 +4021,12 @@ dependencies = [
[[package]]
name = "libloading"
-version = "0.8.7"
+version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c"
+checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
- "windows-targets 0.53.0",
+ "windows-targets 0.53.3",
]
[[package]]
@@ -4012,18 +4042,19 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d"
dependencies = [
"cc",
+ "cty",
"libc",
]
[[package]]
name = "libredox"
-version = "0.1.3"
+version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
+checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
dependencies = [
"bitflags 2.9.1",
"libc",
- "redox_syscall 0.5.12",
+ "redox_syscall 0.5.17",
]
[[package]]
@@ -4034,7 +4065,7 @@ checksum =
"5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33"
dependencies = [
"anstream",
"anstyle",
- "clap 4.5.41",
+ "clap 4.5.42",
"escape8259",
]
@@ -4067,9 +4098,9 @@ checksum =
"241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
[[package]]
name = "lock_api"
-version = "0.4.12"
+version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
@@ -4089,11 +4120,11 @@ checksum =
"112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
-version = "0.11.3"
+version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
+checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
dependencies = [
- "twox-hash 1.6.3",
+ "twox-hash",
]
[[package]]
@@ -4125,9 +4156,9 @@ dependencies = [
[[package]]
name = "memchr"
-version = "2.7.4"
+version = "2.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
[[package]]
name = "memoffset"
@@ -4171,22 +4202,22 @@ checksum =
"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
-version = "0.8.8"
+version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
]
[[package]]
name = "mio"
-version = "1.0.3"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
+checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.52.0",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -4409,6 +4440,12 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
+
[[package]]
name = "oorandom"
version = "11.1.5"
@@ -4450,15 +4487,15 @@ checksum =
"b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "owo-colors"
-version = "4.2.1"
+version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec"
+checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e"
[[package]]
name = "parking_lot"
-version = "0.12.3"
+version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13"
dependencies = [
"lock_api",
"parking_lot_core",
@@ -4466,13 +4503,13 @@ dependencies = [
[[package]]
name = "parking_lot_core"
-version = "0.9.10"
+version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.12",
+ "redox_syscall 0.5.17",
"smallvec",
"windows-targets 0.52.6",
]
@@ -4498,7 +4535,7 @@ dependencies = [
"flate2",
"futures",
"half",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
"lz4_flex",
"num",
"num-bigint",
@@ -4510,7 +4547,7 @@ dependencies = [
"snap",
"thrift",
"tokio",
- "twox-hash 2.1.0",
+ "twox-hash",
"zstd",
]
@@ -4605,7 +4642,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca"
dependencies = [
"fixedbitset",
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
"indexmap 2.10.0",
"serde",
]
@@ -4714,9 +4751,9 @@ dependencies = [
[[package]]
name = "portable-atomic"
-version = "1.11.0"
+version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
[[package]]
name = "portable-atomic-util"
@@ -4836,9 +4873,9 @@ dependencies = [
[[package]]
name = "prettyplease"
-version = "0.2.32"
+version = "0.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6"
+checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2"
dependencies = [
"proc-macro2",
"syn 2.0.104",
@@ -5068,7 +5105,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.1",
"rustls",
- "socket2 0.5.9",
+ "socket2 0.5.10",
"thiserror 2.0.12",
"tokio",
"tracing",
@@ -5098,14 +5135,14 @@ dependencies = [
[[package]]
name = "quinn-udp"
-version = "0.5.12"
+version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842"
+checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.5.9",
+ "socket2 0.5.10",
"tracing",
"windows-sys 0.59.0",
]
@@ -5121,9 +5158,9 @@ dependencies = [
[[package]]
name = "r-efi"
-version = "5.2.0"
+version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "radium"
@@ -5261,24 +5298,44 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.5.12"
+version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
dependencies = [
"bitflags 2.9.1",
]
[[package]]
name = "redox_users"
-version = "0.5.0"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b"
+checksum = "78eaea1f52c56d57821be178b2d47e09ff26481a6042e8e042fcb0ced068b470"
dependencies = [
"getrandom 0.2.16",
"libredox",
"thiserror 2.0.12",
]
+[[package]]
+name = "ref-cast"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf"
+dependencies = [
+ "ref-cast-impl",
+]
+
+[[package]]
+name = "ref-cast-impl"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
+]
+
[[package]]
name = "regex"
version = "1.11.1"
@@ -5316,11 +5373,11 @@ checksum =
"2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "regress"
-version = "0.10.3"
+version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366"
+checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010"
dependencies = [
- "hashbrown 0.15.3",
+ "hashbrown 0.15.4",
"memchr",
]
@@ -5350,9 +5407,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.12.15"
+version = "0.12.22"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
+checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -5365,17 +5422,13 @@ dependencies = [
"hyper",
"hyper-rustls",
"hyper-util",
- "ipnet",
"js-sys",
"log",
- "mime",
- "once_cell",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-native-certs",
- "rustls-pemfile",
"rustls-pki-types",
"serde",
"serde_json",
@@ -5385,13 +5438,13 @@ dependencies = [
"tokio-rustls",
"tokio-util",
"tower 0.5.2",
+ "tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
- "windows-registry",
]
[[package]]
@@ -5503,9 +5556,9 @@ dependencies = [
[[package]]
name = "rustc-demangle"
-version = "0.1.24"
+version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
[[package]]
name = "rustc-hash"
@@ -5543,22 +5596,22 @@ dependencies = [
[[package]]
name = "rustix"
-version = "1.0.7"
+version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
dependencies = [
"bitflags 2.9.1",
"errno",
"libc",
"linux-raw-sys 0.9.4",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
name = "rustls"
-version = "0.23.27"
+version = "0.23.31"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321"
+checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -5602,9 +5655,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.103.3"
+version = "0.103.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
+checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc"
dependencies = [
"aws-lc-rs",
"ring",
@@ -5614,9 +5667,9 @@ dependencies = [
[[package]]
name = "rustversion"
-version = "1.0.20"
+version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
+checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
[[package]]
name = "rustyline"
@@ -5635,7 +5688,7 @@ dependencies = [
"nix",
"radix_trie",
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.1",
"utf8parse",
"windows-sys 0.59.0",
]
@@ -5676,6 +5729,30 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "schemars"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f"
+dependencies = [
+ "dyn-clone",
+ "ref-cast",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
+dependencies = [
+ "dyn-clone",
+ "ref-cast",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "schemars_derive"
version = "0.8.22"
@@ -5827,15 +5904,17 @@ dependencies = [
[[package]]
name = "serde_with"
-version = "3.12.0"
+version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
+checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.10.0",
+ "schemars 0.9.0",
+ "schemars 1.0.4",
"serde",
"serde_derive",
"serde_json",
@@ -5845,9 +5924,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
-version = "3.12.0"
+version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
+checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f"
dependencies = [
"darling",
"proc-macro2",
@@ -5923,18 +6002,15 @@ checksum =
"56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "slab"
-version = "0.4.9"
+version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
+checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
[[package]]
name = "smallvec"
-version = "1.15.0"
+version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "snap"
@@ -5962,9 +6038,9 @@ dependencies = [
[[package]]
name = "socket2"
-version = "0.5.9"
+version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
@@ -6161,7 +6237,7 @@ dependencies = [
"prost-types",
"protobuf-src",
"regress",
- "schemars",
+ "schemars 0.8.22",
"semver",
"serde",
"serde_json",
@@ -6254,7 +6330,7 @@ dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
- "rustix 1.0.7",
+ "rustix 1.0.8",
"windows-sys 0.59.0",
]
@@ -6364,12 +6440,11 @@ dependencies = [
[[package]]
name = "thread_local"
-version = "1.1.8"
+version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
dependencies = [
"cfg-if",
- "once_cell",
]
[[package]]
@@ -6509,7 +6584,7 @@ dependencies = [
"postgres-protocol",
"postgres-types",
"rand 0.9.2",
- "socket2 0.5.9",
+ "socket2 0.5.10",
"tokio",
"tokio-util",
"whoami",
@@ -6566,15 +6641,15 @@ dependencies = [
[[package]]
name = "toml_datetime"
-version = "0.6.9"
+version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
[[package]]
name = "toml_edit"
-version = "0.22.26"
+version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.10.0",
"toml_datetime",
@@ -6602,7 +6677,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"prost",
- "socket2 0.5.9",
+ "socket2 0.5.10",
"tokio",
"tokio-stream",
"tower 0.4.13",
@@ -6646,6 +6721,24 @@ dependencies = [
"tower-service",
]
+[[package]]
+name = "tower-http"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
+dependencies = [
+ "bitflags 2.9.1",
+ "bytes",
+ "futures-util",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "iri-string",
+ "pin-project-lite",
+ "tower 0.5.2",
+ "tower-layer",
+ "tower-service",
+]
+
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -6671,9 +6764,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
-version = "0.1.28"
+version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
+checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
dependencies = [
"proc-macro2",
"quote",
@@ -6682,9 +6775,9 @@ dependencies = [
[[package]]
name = "tracing-core"
-version = "0.1.33"
+version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
+checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
dependencies = [
"once_cell",
"valuable",
@@ -6738,19 +6831,9 @@ checksum =
"e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a"
[[package]]
name = "twox-hash"
-version = "1.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
-dependencies = [
- "cfg-if",
- "static_assertions",
-]
-
-[[package]]
-name = "twox-hash"
-version = "2.1.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
+checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56"
[[package]]
name = "typed-arena"
@@ -6786,9 +6869,9 @@ checksum =
"1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
[[package]]
name = "typify"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fcc5bec3cdff70fd542e579aa2e52967833e543a25fae0d14579043d2e868a50"
+checksum = "6c6c647a34e851cf0260ccc14687f17cdcb8302ff1a8a687a24b97ca0f82406f"
dependencies = [
"typify-impl",
"typify-macro",
@@ -6796,16 +6879,16 @@ dependencies = [
[[package]]
name = "typify-impl"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b52a67305054e1da6f3d99ad94875dcd0c7c49adbd17b4b64f0eefb7ae5bf8ab"
+checksum = "741b7f1e2e1338c0bee5ad5a7d3a9bbd4e24c33765c08b7691810e68d879365d"
dependencies = [
"heck 0.5.0",
"log",
"proc-macro2",
"quote",
"regress",
- "schemars",
+ "schemars 0.8.22",
"semver",
"serde",
"serde_json",
@@ -6816,13 +6899,13 @@ dependencies = [
[[package]]
name = "typify-macro"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ff5799be156e4f635c348c6051d165e1c59997827155133351a8c4d333d9841"
+checksum = "7560adf816a1e8dad7c63d8845ef6e31e673e39eab310d225636779230cbedeb"
dependencies = [
"proc-macro2",
"quote",
- "schemars",
+ "schemars 0.8.22",
"semver",
"serde",
"serde_json",
@@ -6872,9 +6955,9 @@ checksum =
"7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "unicode-width"
-version = "0.2.0"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
[[package]]
name = "unindent"
@@ -6984,9 +7067,9 @@ dependencies = [
[[package]]
name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
+version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasi"
@@ -7149,7 +7232,7 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7"
dependencies = [
- "redox_syscall 0.5.12",
+ "redox_syscall 0.5.17",
"wasite",
"web-sys",
]
@@ -7187,9 +7270,9 @@ checksum =
"712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
-version = "0.61.1"
+version = "0.61.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419"
+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
dependencies = [
"windows-collections",
"windows-core",
@@ -7217,7 +7300,7 @@ dependencies = [
"windows-interface",
"windows-link",
"windows-result",
- "windows-strings 0.4.2",
+ "windows-strings",
]
[[package]]
@@ -7255,9 +7338,9 @@ dependencies = [
[[package]]
name = "windows-link"
-version = "0.1.1"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
+checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
[[package]]
name = "windows-numerics"
@@ -7269,17 +7352,6 @@ dependencies = [
"windows-link",
]
-[[package]]
-name = "windows-registry"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
-dependencies = [
- "windows-result",
- "windows-strings 0.3.1",
- "windows-targets 0.53.0",
-]
-
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -7289,15 +7361,6 @@ dependencies = [
"windows-link",
]
-[[package]]
-name = "windows-strings"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
-dependencies = [
- "windows-link",
-]
-
[[package]]
name = "windows-strings"
version = "0.4.2"
@@ -7325,6 +7388,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-sys"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
+dependencies = [
+ "windows-targets 0.53.3",
+]
+
[[package]]
name = "windows-targets"
version = "0.52.6"
@@ -7343,10 +7415,11 @@ dependencies = [
[[package]]
name = "windows-targets"
-version = "0.53.0"
+version = "0.53.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b"
+checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
dependencies = [
+ "windows-link",
"windows_aarch64_gnullvm 0.53.0",
"windows_aarch64_msvc 0.53.0",
"windows_i686_gnu 0.53.0",
@@ -7464,9 +7537,9 @@ checksum =
"271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "winnow"
-version = "0.7.10"
+version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
dependencies = [
"memchr",
]
@@ -7497,12 +7570,12 @@ dependencies = [
[[package]]
name = "xattr"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e"
+checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909"
dependencies = [
"libc",
- "rustix 1.0.7",
+ "rustix 1.0.8",
]
[[package]]
@@ -7552,18 +7625,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.25"
+version = "0.8.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
+checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.25"
+version = "0.8.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
+checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
dependencies = [
"proc-macro2",
"quote",
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index f9c198597b..0aa257952e 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -33,6 +33,7 @@ workspace = true
ci = []
default = ["mimalloc"]
snmalloc = ["snmalloc-rs"]
+mimalloc_extended = ["libmimalloc-sys/extended"]
[dependencies]
arrow = { workspace = true }
@@ -40,11 +41,13 @@ datafusion = { workspace = true, default-features = true }
datafusion-common = { workspace = true, default-features = true }
env_logger = { workspace = true }
futures = { workspace = true }
+libmimalloc-sys = { version = "0.1", optional = true }
log = { workspace = true }
mimalloc = { version = "0.1", optional = true, default-features = false }
object_store = { workspace = true }
parquet = { workspace = true, default-features = true }
rand = { workspace = true }
+regex.workspace = true
serde = { version = "1.0.219", features = ["derive"] }
serde_json = { workspace = true }
snmalloc-rs = { version = "0.3", optional = true }
diff --git a/benchmarks/README.md b/benchmarks/README.md
index d0f413b2e9..4e226376c0 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -283,6 +283,7 @@ This will produce output like:
└──────────────┴──────────────┴──────────────┴───────────────┘
```
+
# Benchmark Runner
The `dfbench` program contains subcommands to run the various
@@ -321,6 +322,66 @@ FLAGS:
...
```
+# Profiling Memory Stats for each benchmark query
+The `mem_profile` program wraps benchmark execution to measure memory usage
statistics, such as peak RSS. It runs each benchmark query in a separate
subprocess, capturing the child process’s stdout to print structured output.
+
+Subcommands supported by mem_profile are the subset of those in `dfbench`.
+Currently supported benchmarks include: Clickbench, H2o, Imdb, SortTpch, Tpch
+
+Before running benchmarks, `mem_profile` automatically compiles the benchmark
binary (`dfbench`) using `cargo build`. Note that the build profile used for
`dfbench` is not tied to the profile used for running `mem_profile` itself. We
can explicitly specify the desired build profile using the `--bench-profile`
option (e.g. release-nonlto). By prebuilding the binary and running each query
in a separate process, we can ensure accurate memory statistics.
+
+Currently, `mem_profile` only supports `mimalloc` as the memory allocator,
since it relies on `mimalloc`'s API to collect memory statistics.
+
+Because it runs the compiled binary directly from the target directory, make
sure your working directory is the top-level datafusion/ directory, where the
target/ is also located.
+
+The benchmark subcommand (e.g., `tpch`) and all following arguments are passed
directly to `dfbench`. Be sure to specify `--bench-profile` before the
benchmark subcommand.
+
+Example:
+```shell
+datafusion$ cargo run --profile release-nonlto --bin mem_profile --
--bench-profile release-nonlto tpch --path benchmarks/data/tpch_sf1
--partitions 4 --format parquet
+```
+Example Output:
+```
+Query Time (ms) Peak RSS Peak Commit Major Page Faults
+----------------------------------------------------------------
+1 503.42 283.4 MB 3.0 GB 0
+2 431.09 240.7 MB 3.0 GB 0
+3 594.28 350.1 MB 3.0 GB 0
+4 468.90 462.4 MB 3.0 GB 0
+5 653.58 385.4 MB 3.0 GB 0
+6 296.79 247.3 MB 2.0 GB 0
+7 662.32 652.4 MB 3.0 GB 0
+8 702.48 396.0 MB 3.0 GB 0
+9 774.21 611.5 MB 3.0 GB 0
+10 733.62 397.9 MB 3.0 GB 0
+11 271.71 209.6 MB 3.0 GB 0
+12 512.60 212.5 MB 2.0 GB 0
+13 507.83 381.5 MB 2.0 GB 0
+14 420.89 313.5 MB 3.0 GB 0
+15 539.97 288.0 MB 2.0 GB 0
+16 370.91 229.8 MB 3.0 GB 0
+17 758.33 467.0 MB 2.0 GB 0
+18 1112.32 638.9 MB 3.0 GB 0
+19 712.72 280.9 MB 2.0 GB 0
+20 620.64 402.9 MB 2.9 GB 0
+21 971.63 388.9 MB 2.9 GB 0
+22 404.50 164.8 MB 2.0 GB 0
+```
+
+## Reported Metrics
+When running benchmarks, `mem_profile` collects several memory-related
statistics using the mimalloc API:
+
+- Peak RSS (Resident Set Size):
+The maximum amount of physical memory used by the process.
+This is a process-level metric collected via OS-specific mechanisms and is not
mimalloc-specific.
+
+- Peak Commit:
+The peak amount of memory committed by the allocator (i.e., total virtual
memory reserved).
+This is mimalloc-specific. It gives a more allocator-aware view of memory
usage than RSS.
+
+- Major Page Faults:
+The number of major page faults triggered during execution.
+This metric is obtained from the operating system and is not mimalloc-specific.
# Writing a new benchmark
## Creating or downloading data outside of the benchmark
diff --git a/benchmarks/src/bin/mem_profile.rs
b/benchmarks/src/bin/mem_profile.rs
new file mode 100644
index 0000000000..16fc3871be
--- /dev/null
+++ b/benchmarks/src/bin/mem_profile.rs
@@ -0,0 +1,360 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! mem_profile binary entrypoint
+use datafusion::error::Result;
+use std::{
+ env,
+ io::{BufRead, BufReader},
+ path::Path,
+ process::{Command, Stdio},
+};
+use structopt::StructOpt;
+
+use datafusion_benchmarks::{
+ clickbench,
+ h2o::{self, AllQueries},
+ imdb, sort_tpch, tpch,
+};
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "Memory Profiling Utility")]
+struct MemProfileOpt {
+ /// Cargo profile to use in dfbench (e.g. release, release-nonlto)
+ #[structopt(long, default_value = "release")]
+ bench_profile: String,
+
+ #[structopt(subcommand)]
+ command: Options,
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(about = "Benchmark command")]
+enum Options {
+ Clickbench(clickbench::RunOpt),
+ H2o(h2o::RunOpt),
+ Imdb(imdb::RunOpt),
+ SortTpch(sort_tpch::RunOpt),
+ Tpch(tpch::RunOpt),
+}
+
+#[tokio::main]
+pub async fn main() -> Result<()> {
+ // 1. Parse args and check which benchmarks should be run
+ let mem_profile_opt = MemProfileOpt::from_args();
+ let profile = mem_profile_opt.bench_profile;
+ let query_range = match mem_profile_opt.command {
+ Options::Clickbench(opt) => {
+ let entries = std::fs::read_dir(&opt.queries_path)?
+ .filter_map(Result::ok)
+ .filter(|e| {
+ let path = e.path();
+ path.extension().map(|ext| ext == "sql").unwrap_or(false)
+ })
+ .collect::<Vec<_>>();
+
+ let max_query_id = entries.len().saturating_sub(1);
+ match opt.query {
+ Some(query_id) => query_id..=query_id,
+ None => 0..=max_query_id,
+ }
+ }
+ Options::H2o(opt) => {
+ let queries = AllQueries::try_new(&opt.queries_path)?;
+ match opt.query {
+ Some(query_id) => query_id..=query_id,
+ None => queries.min_query_id()..=queries.max_query_id(),
+ }
+ }
+ Options::Imdb(opt) => match opt.query {
+ Some(query_id) => query_id..=query_id,
+ None => imdb::IMDB_QUERY_START_ID..=imdb::IMDB_QUERY_END_ID,
+ },
+ Options::SortTpch(opt) => match opt.query {
+ Some(query_id) => query_id..=query_id,
+ None => {
+
sort_tpch::SORT_TPCH_QUERY_START_ID..=sort_tpch::SORT_TPCH_QUERY_END_ID
+ }
+ },
+ Options::Tpch(opt) => match opt.query {
+ Some(query_id) => query_id..=query_id,
+ None => tpch::TPCH_QUERY_START_ID..=tpch::TPCH_QUERY_END_ID,
+ },
+ };
+
+ // 2. Prebuild dfbench binary so that memory does not blow up due to build
process
+ println!("Pre-building benchmark binary...");
+ let status = Command::new("cargo")
+ .args([
+ "build",
+ "--profile",
+ &profile,
+ "--features",
+ "mimalloc_extended",
+ "--bin",
+ "dfbench",
+ ])
+ .status()
+ .expect("Failed to build dfbench");
+ assert!(status.success());
+ println!("Benchmark binary built successfully.");
+
+ // 3. Create a new process per each benchmark query and print summary
+ // Find position of subcommand to collect args for dfbench
+ let args: Vec<_> = env::args().collect();
+ let subcommands = ["tpch", "clickbench", "h2o", "imdb", "sort-tpch"];
+ let sub_pos = args
+ .iter()
+ .position(|s| subcommands.iter().any(|&cmd| s == cmd))
+ .expect("No benchmark subcommand found");
+
+ // Args starting from subcommand become dfbench args
+ let mut dfbench_args: Vec<String> =
+ args[sub_pos..].iter().map(|s| s.to_string()).collect();
+
+ run_benchmark_as_child_process(&profile, query_range, &mut dfbench_args)?;
+
+ Ok(())
+}
+
+fn run_benchmark_as_child_process(
+ profile: &str,
+ query_range: std::ops::RangeInclusive<usize>,
+ args: &mut Vec<String>,
+) -> Result<()> {
+ let mut query_strings: Vec<String> = Vec::new();
+ for i in query_range {
+ query_strings.push(i.to_string());
+ }
+
+ let target_dir =
+ env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| "target".to_string());
+ let command = format!("{target_dir}/{profile}/dfbench");
+ // Check whether benchmark binary exists
+ if !Path::new(&command).exists() {
+ panic!(
+ "Benchmark binary not found: `{command}`\nRun this command from
the top-level `datafusion/` directory so `target/{profile}/dfbench` can be
found.",
+ );
+ }
+ args.insert(0, command);
+ let mut results = vec![];
+
+ // Run Single Query (args already contain --query num)
+ if args.contains(&"--query".to_string()) {
+ let _ = run_query(args, &mut results);
+ print_summary_table(&results);
+ return Ok(());
+ }
+
+ // Run All Queries
+ args.push("--query".to_string());
+ for query_str in query_strings {
+ args.push(query_str);
+ let _ = run_query(args, &mut results);
+ args.pop();
+ }
+
+ print_summary_table(&results);
+ Ok(())
+}
+
+fn run_query(args: &[String], results: &mut Vec<QueryResult>) -> Result<()> {
+ let exec_path = &args[0];
+ let exec_args = &args[1..];
+
+ let mut child = Command::new(exec_path)
+ .args(exec_args)
+ .stdout(Stdio::piped())
+ .spawn()
+ .expect("Failed to start benchmark");
+
+ let stdout = child.stdout.take().unwrap();
+ let reader = BufReader::new(stdout);
+
+ // Buffer child's stdout
+ let lines: Result<Vec<String>, std::io::Error> =
+ reader.lines().collect::<Result<_, _>>();
+
+ child
+ .wait()
+ .expect("Benchmark process exited with an error");
+
+ // Parse after child process terminates
+ let lines = lines?;
+ let mut iter = lines.iter().peekable();
+
+ // Look for lines that contain execution time / memory stats
+ while let Some(line) = iter.next() {
+ if let Some((query, duration_ms)) = parse_query_time(line) {
+ if let Some(next_line) = iter.peek() {
+ if let Some((peak_rss, peak_commit, page_faults)) =
+ parse_vm_line(next_line)
+ {
+ results.push(QueryResult {
+ query,
+ duration_ms,
+ peak_rss,
+ peak_commit,
+ page_faults,
+ });
+ break;
+ }
+ }
+ }
+ }
+
+ Ok(())
+}
+
+#[derive(Debug)]
+struct QueryResult {
+ query: usize,
+ duration_ms: f64,
+ peak_rss: String,
+ peak_commit: String,
+ page_faults: String,
+}
+
+fn parse_query_time(line: &str) -> Option<(usize, f64)> {
+ let re = regex::Regex::new(r"Query (\d+) avg time: ([\d.]+) ms").unwrap();
+ if let Some(caps) = re.captures(line) {
+ let query_id = caps[1].parse::<usize>().ok()?;
+ let avg_time = caps[2].parse::<f64>().ok()?;
+ Some((query_id, avg_time))
+ } else {
+ None
+ }
+}
+
+fn parse_vm_line(line: &str) -> Option<(String, String, String)> {
+ let re = regex::Regex::new(
+ r"Peak RSS:\s*([\d.]+\s*[A-Z]+),\s*Peak
Commit:\s*([\d.]+\s*[A-Z]+),\s*Page Faults:\s*([\d.]+)"
+ ).ok()?;
+ let caps = re.captures(line)?;
+ let peak_rss = caps.get(1)?.as_str().to_string();
+ let peak_commit = caps.get(2)?.as_str().to_string();
+ let page_faults = caps.get(3)?.as_str().to_string();
+ Some((peak_rss, peak_commit, page_faults))
+}
+
+// Print as simple aligned table
+fn print_summary_table(results: &[QueryResult]) {
+ println!(
+ "\n{:<8} {:>10} {:>12} {:>12} {:>18}",
+ "Query", "Time (ms)", "Peak RSS", "Peak Commit", "Major Page Faults"
+ );
+ println!("{}", "-".repeat(64));
+
+ for r in results {
+ println!(
+ "{:<8} {:>10.2} {:>12} {:>12} {:>18}",
+ r.query, r.duration_ms, r.peak_rss, r.peak_commit, r.page_faults
+ );
+ }
+}
+
+#[cfg(test)]
+// Only run with "ci" mode when we have the data
+#[cfg(feature = "ci")]
+mod tests {
+ use datafusion::common::exec_err;
+ use datafusion::error::Result;
+ use std::path::{Path, PathBuf};
+ use std::process::Command;
+
+ fn get_tpch_data_path() -> Result<String> {
+ let path =
+ std::env::var("TPCH_DATA").unwrap_or_else(|_|
"benchmarks/data".to_string());
+ if !Path::new(&path).exists() {
+ return exec_err!(
+ "Benchmark data not found (set TPCH_DATA env var to override):
{}",
+ path
+ );
+ }
+ Ok(path)
+ }
+
+ // Try to find target/ dir upward
+ fn find_target_dir(start: &Path) -> Option<PathBuf> {
+ let mut dir = start;
+
+ while let Some(current) = Some(dir) {
+ if current.join("target").is_dir() {
+ return Some(current.join("target"));
+ }
+
+ dir = match current.parent() {
+ Some(parent) => parent,
+ None => break,
+ };
+ }
+
+ None
+ }
+
+ #[test]
+ // This test checks whether `mem_profile` runs successfully and produces
expected output
+ // using TPC-H query 6 (which runs quickly).
+ fn mem_profile_e2e_tpch_q6() -> Result<()> {
+ let profile = "ci";
+ let tpch_data = get_tpch_data_path()?;
+
+ // The current working directory may not be the top-level datafusion/
directory,
+ // so we manually walkdir upward, locate the target directory
+ // and set it explicitly via CARGO_TARGET_DIR for the mem_profile
command.
+ let target_dir = find_target_dir(&std::env::current_dir()?);
+ let output = Command::new("cargo")
+ .env("CARGO_TARGET_DIR", target_dir.unwrap())
+ .args([
+ "run",
+ "--profile",
+ profile,
+ "--bin",
+ "mem_profile",
+ "--",
+ "--bench-profile",
+ profile,
+ "tpch",
+ "--query",
+ "6",
+ "--path",
+ &tpch_data,
+ "--format",
+ "tbl",
+ ])
+ .output()
+ .expect("Failed to run mem_profile");
+
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ let stderr = String::from_utf8_lossy(&output.stderr);
+
+ if !output.status.success() {
+ panic!(
+ "mem_profile
failed\nstdout:\n{stdout}\nstderr:\n{stderr}---------------------",
+ );
+ }
+
+ assert!(
+ stdout.contains("Peak RSS")
+ && stdout.contains("Query")
+ && stdout.contains("Time"),
+ "Unexpected output:\n{stdout}",
+ );
+
+ Ok(())
+ }
+}
diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rs
index d5f6052d9e..a550503390 100644
--- a/benchmarks/src/clickbench.rs
+++ b/benchmarks/src/clickbench.rs
@@ -19,7 +19,7 @@ use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
-use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
use datafusion::logical_expr::{ExplainFormat, ExplainOption};
use datafusion::{
error::{DataFusionError, Result},
@@ -42,7 +42,7 @@ use structopt::StructOpt;
pub struct RunOpt {
/// Query number (between 0 and 42). If not specified, runs all queries
#[structopt(short, long)]
- query: Option<usize>,
+ pub query: Option<usize>,
/// If specified, enables Parquet Filter Pushdown.
///
@@ -73,7 +73,7 @@ pub struct RunOpt {
long = "queries-path",
default_value = "benchmarks/queries/clickbench/queries"
)]
- queries_path: PathBuf,
+ pub queries_path: PathBuf,
/// If present, write results json here
#[structopt(parse(from_os_str), short = "o", long = "output")]
@@ -206,6 +206,10 @@ impl RunOpt {
}
let avg = millis.iter().sum::<f64>() / millis.len() as f64;
println!("Query {query_id} avg time: {avg:.2} ms");
+
+ // Print memory usage stats using mimalloc (only when compiled with
--features mimalloc_extended)
+ print_memory_stats();
+
Ok(query_results)
}
diff --git a/benchmarks/src/h2o.rs b/benchmarks/src/h2o.rs
index 9d4deaf387..be74252031 100644
--- a/benchmarks/src/h2o.rs
+++ b/benchmarks/src/h2o.rs
@@ -20,7 +20,7 @@
//! - [H2O AI Benchmark](https://duckdb.org/2023/04/14/h2oai.html)
//! - [Extended window function
benchmark](https://duckdb.org/2024/06/26/benchmarks-over-time.html#window-functions-benchmark)
-use crate::util::{BenchmarkRun, CommonOpt};
+use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt};
use datafusion::logical_expr::{ExplainFormat, ExplainOption};
use datafusion::{error::Result, prelude::SessionContext};
use datafusion_common::{
@@ -34,7 +34,7 @@ use structopt::StructOpt;
#[structopt(verbatim_doc_comment)]
pub struct RunOpt {
#[structopt(short, long)]
- query: Option<usize>,
+ pub query: Option<usize>,
/// Common options
#[structopt(flatten)]
@@ -48,7 +48,7 @@ pub struct RunOpt {
long = "queries-path",
default_value = "benchmarks/queries/h2o/groupby.sql"
)]
- queries_path: PathBuf,
+ pub queries_path: PathBuf,
/// Path to data file (parquet or csv)
/// Default value is the G1_1e7_1e7_100_0.csv file in the h2o benchmark
@@ -132,6 +132,9 @@ impl RunOpt {
let avg = millis.iter().sum::<f64>() / millis.len() as f64;
println!("Query {query_id} avg time: {avg:.2} ms");
+ // Print memory usage stats using mimalloc (only when compiled
with --features mimalloc_extended)
+ print_memory_stats();
+
if self.common.debug {
ctx.sql(sql)
.await?
@@ -197,12 +200,12 @@ impl RunOpt {
}
}
-struct AllQueries {
+pub struct AllQueries {
queries: Vec<String>,
}
impl AllQueries {
- fn try_new(path: &Path) -> Result<Self> {
+ pub fn try_new(path: &Path) -> Result<Self> {
let all_queries = std::fs::read_to_string(path)
.map_err(|e| exec_datafusion_err!("Could not open {path:?}:
{e}"))?;
@@ -212,7 +215,7 @@ impl AllQueries {
}
/// Returns the text of query `query_id`
- fn get_query(&self, query_id: usize) -> Result<&str> {
+ pub fn get_query(&self, query_id: usize) -> Result<&str> {
self.queries
.get(query_id - 1)
.ok_or_else(|| {
@@ -225,11 +228,11 @@ impl AllQueries {
.map(|s| s.as_str())
}
- fn min_query_id(&self) -> usize {
+ pub fn min_query_id(&self) -> usize {
1
}
- fn max_query_id(&self) -> usize {
+ pub fn max_query_id(&self) -> usize {
self.queries.len()
}
}
diff --git a/benchmarks/src/imdb/mod.rs b/benchmarks/src/imdb/mod.rs
index 6a45242e6f..87462bc3e8 100644
--- a/benchmarks/src/imdb/mod.rs
+++ b/benchmarks/src/imdb/mod.rs
@@ -54,6 +54,9 @@ pub const IMDB_TABLES: &[&str] = &[
"person_info",
];
+pub const IMDB_QUERY_START_ID: usize = 1;
+pub const IMDB_QUERY_END_ID: usize = 113;
+
/// Get the schema for the IMDB dataset tables
/// see benchmarks/data/imdb/schematext.sql
pub fn get_imdb_table_schema(table: &str) -> Schema {
diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs
index 7c9d03a9d9..90e0947f64 100644
--- a/benchmarks/src/imdb/run.rs
+++ b/benchmarks/src/imdb/run.rs
@@ -18,8 +18,11 @@
use std::path::PathBuf;
use std::sync::Arc;
-use super::{get_imdb_table_schema, get_query_sql, IMDB_TABLES};
-use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
+use super::{
+ get_imdb_table_schema, get_query_sql, IMDB_QUERY_END_ID,
IMDB_QUERY_START_ID,
+ IMDB_TABLES,
+};
+use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty::{self, pretty_format_batches};
@@ -59,7 +62,7 @@ type BoolDefaultTrue = bool;
pub struct RunOpt {
/// Query number. If not specified, runs all queries
#[structopt(short, long)]
- query: Option<usize>,
+ pub query: Option<usize>,
/// Common options
#[structopt(flatten)]
@@ -91,9 +94,6 @@ pub struct RunOpt {
prefer_hash_join: BoolDefaultTrue,
}
-const IMDB_QUERY_START_ID: usize = 1;
-const IMDB_QUERY_END_ID: usize = 113;
-
fn map_query_id_to_str(query_id: usize) -> &'static str {
match query_id {
// 1
@@ -341,6 +341,9 @@ impl RunOpt {
let avg = millis.iter().sum::<f64>() / millis.len() as f64;
println!("Query {query_id} avg time: {avg:.2} ms");
+ // Print memory usage stats using mimalloc (only when compiled with
--features mimalloc_extended)
+ print_memory_stats();
+
Ok(query_results)
}
diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rs
index 21897f5bf2..09b5a676bb 100644
--- a/benchmarks/src/sort_tpch.rs
+++ b/benchmarks/src/sort_tpch.rs
@@ -40,7 +40,7 @@ use datafusion_common::instant::Instant;
use datafusion_common::utils::get_available_parallelism;
use datafusion_common::DEFAULT_PARQUET_EXTENSION;
-use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
#[derive(Debug, StructOpt)]
pub struct RunOpt {
@@ -50,7 +50,7 @@ pub struct RunOpt {
/// Sort query number. If not specified, runs all queries
#[structopt(short, long)]
- query: Option<usize>,
+ pub query: Option<usize>,
/// Path to data files (lineitem). Only parquet format is supported
#[structopt(parse(from_os_str), required = true, short = "p", long =
"path")]
@@ -74,6 +74,9 @@ pub struct RunOpt {
limit: Option<usize>,
}
+pub const SORT_TPCH_QUERY_START_ID: usize = 1;
+pub const SORT_TPCH_QUERY_END_ID: usize = 11;
+
impl RunOpt {
const SORT_TABLES: [&'static str; 1] = ["lineitem"];
@@ -178,7 +181,7 @@ impl RunOpt {
let query_range = match self.query {
Some(query_id) => query_id..=query_id,
- None => 1..=Self::SORT_QUERIES.len(),
+ None => SORT_TPCH_QUERY_START_ID..=SORT_TPCH_QUERY_END_ID,
};
for query_id in query_range {
@@ -238,13 +241,16 @@ impl RunOpt {
millis.push(ms);
println!(
- "Q{query_id} iteration {i} took {ms:.1} ms and returned
{row_count} rows"
+ "Query {query_id} iteration {i} took {ms:.1} ms and returned
{row_count} rows"
);
query_results.push(QueryResult { elapsed, row_count });
}
let avg = millis.iter().sum::<f64>() / millis.len() as f64;
- println!("Q{query_id} avg time: {avg:.2} ms");
+ println!("Query {query_id} avg time: {avg:.2} ms");
+
+ // Print memory usage stats using mimalloc (only when compiled with
--features mimalloc_extended)
+ print_memory_stats();
Ok(query_results)
}
diff --git a/benchmarks/src/tpch/mod.rs b/benchmarks/src/tpch/mod.rs
index 23d0681f56..233ea94a05 100644
--- a/benchmarks/src/tpch/mod.rs
+++ b/benchmarks/src/tpch/mod.rs
@@ -34,6 +34,9 @@ pub const TPCH_TABLES: &[&str] = &[
"part", "supplier", "partsupp", "customer", "orders", "lineitem",
"nation", "region",
];
+pub const TPCH_QUERY_START_ID: usize = 1;
+pub const TPCH_QUERY_END_ID: usize = 22;
+
/// The `.tbl` file contains a trailing column
pub fn get_tbl_tpch_table_schema(table: &str) -> Schema {
let mut schema = SchemaBuilder::from(get_tpch_table_schema(table).fields);
diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs
index 4f7e539a04..30ecb4d33b 100644
--- a/benchmarks/src/tpch/run.rs
+++ b/benchmarks/src/tpch/run.rs
@@ -19,9 +19,10 @@ use std::path::PathBuf;
use std::sync::Arc;
use super::{
- get_query_sql, get_tbl_tpch_table_schema, get_tpch_table_schema,
TPCH_TABLES,
+ get_query_sql, get_tbl_tpch_table_schema, get_tpch_table_schema,
TPCH_QUERY_END_ID,
+ TPCH_QUERY_START_ID, TPCH_TABLES,
};
-use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
use arrow::record_batch::RecordBatch;
use arrow::util::pretty::{self, pretty_format_batches};
@@ -60,7 +61,7 @@ type BoolDefaultTrue = bool;
pub struct RunOpt {
/// Query number. If not specified, runs all queries
#[structopt(short, long)]
- query: Option<usize>,
+ pub query: Option<usize>,
/// Common options
#[structopt(flatten)]
@@ -97,9 +98,6 @@ pub struct RunOpt {
sorted: bool,
}
-const TPCH_QUERY_START_ID: usize = 1;
-const TPCH_QUERY_END_ID: usize = 22;
-
impl RunOpt {
pub async fn run(self) -> Result<()> {
println!("Running benchmarks with the following options: {self:?}");
@@ -170,7 +168,7 @@ impl RunOpt {
}
}
- let elapsed = start.elapsed(); //.as_secs_f64() * 1000.0;
+ let elapsed = start.elapsed();
let ms = elapsed.as_secs_f64() * 1000.0;
millis.push(ms);
info!("output:\n\n{}\n\n", pretty_format_batches(&result)?);
@@ -184,6 +182,9 @@ impl RunOpt {
let avg = millis.iter().sum::<f64>() / millis.len() as f64;
println!("Query {query_id} avg time: {avg:.2} ms");
+ // Print memory stats using mimalloc (only when compiled with
--features mimalloc_extended)
+ print_memory_stats();
+
Ok(query_results)
}
diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rs
new file mode 100644
index 0000000000..944239df31
--- /dev/null
+++ b/benchmarks/src/util/memory.rs
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Print Peak RSS, Peak Commit, Page Faults based on mimalloc api
+pub fn print_memory_stats() {
+ #[cfg(all(feature = "mimalloc", feature = "mimalloc_extended"))]
+ {
+ use datafusion::execution::memory_pool::human_readable_size;
+ let mut peak_rss = 0;
+ let mut peak_commit = 0;
+ let mut page_faults = 0;
+ unsafe {
+ libmimalloc_sys::mi_process_info(
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ &mut peak_rss,
+ std::ptr::null_mut(),
+ &mut peak_commit,
+ &mut page_faults,
+ );
+ }
+
+ // When modifying this output format, make sure to update the
corresponding
+ // parsers in `mem_profile.rs`, specifically `parse_vm_line` and
`parse_query_time`,
+ // to keep the log output and parser logic in sync.
+ println!(
+ "Peak RSS: {}, Peak Commit: {}, Page Faults: {}",
+ if peak_rss == 0 {
+ "N/A".to_string()
+ } else {
+ human_readable_size(peak_rss)
+ },
+ if peak_commit == 0 {
+ "N/A".to_string()
+ } else {
+ human_readable_size(peak_commit)
+ },
+ page_faults
+ );
+ }
+}
diff --git a/benchmarks/src/util/mod.rs b/benchmarks/src/util/mod.rs
index eb9a0b8bc9..ab4579a566 100644
--- a/benchmarks/src/util/mod.rs
+++ b/benchmarks/src/util/mod.rs
@@ -16,8 +16,10 @@
// under the License.
//! Shared benchmark utilities
+mod memory;
mod options;
mod run;
+pub use memory::print_memory_stats;
pub use options::CommonOpt;
pub use run::{BenchQuery, BenchmarkRun, QueryResult};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
