This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new db4aabf8 perf(core): write hfile structured keys straight into the
block buffer (#912)
db4aabf8 is described below
commit db4aabf8f0ee71207da592ca8fb6dedb90bfb816
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Oct 29 22:05:14 2026 -0700
perf(core): write hfile structured keys straight into the block buffer
(#912)
`HFileWriter::write` now appends each structured key's parts (the `u16`
length, the content, the family and timestamp bytes, the type byte) straight
into the data block buffer instead of building a `Vec` per key and copying it
in. The bytes written are the same, and the 64 KiB key check still rejects the
same keys with the same error. On a 10M-key record index build this removes one
allocation and one copy per entry.
---
crates/core/src/hfile/writer.rs | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
diff --git a/crates/core/src/hfile/writer.rs b/crates/core/src/hfile/writer.rs
index 0136a78d..e83d8477 100644
--- a/crates/core/src/hfile/writer.rs
+++ b/crates/core/src/hfile/writer.rs
@@ -63,14 +63,13 @@ impl HFileWriter {
.sum(),
);
for (key, value) in &entries {
- let key = structured_key(key)?;
- let key_len = i32::try_from(key.len())
+ let key_len = i32::try_from(key.len() + STRUCTURED_KEY_OVERHEAD)
.map_err(|_| HFileError::InvalidFormat("HFile key is too
large".to_string()))?;
let value_len = i32::try_from(value.len())
.map_err(|_| HFileError::InvalidFormat("HFile value is too
large".to_string()))?;
data.extend_from_slice(&key_len.to_be_bytes());
data.extend_from_slice(&value_len.to_be_bytes());
- data.extend_from_slice(&key);
+ push_structured_key(&mut data, key)?;
data.extend_from_slice(value);
data.push(0);
}
@@ -154,7 +153,19 @@ impl HFileWriter {
}
}
+/// Bytes a structured key adds around its content: the u16 length, the
+/// family terminator, the i64 timestamp and the type byte.
+const STRUCTURED_KEY_OVERHEAD: usize = 2 + 1 + 8 + 1;
+
fn structured_key(content: &str) -> crate::hfile::error::Result<Vec<u8>> {
+ let mut key = Vec::with_capacity(content.len() + STRUCTURED_KEY_OVERHEAD);
+ push_structured_key(&mut key, content)?;
+ Ok(key)
+}
+
+/// Appends the structured key for `content` to `out`, so a data block is built
+/// without a buffer per entry.
+fn push_structured_key(out: &mut Vec<u8>, content: &str) ->
crate::hfile::error::Result<()> {
// The KeyValue row-length field is u16; truncating would corrupt framing.
let len = u16::try_from(content.len()).map_err(|_| {
crate::hfile::error::HFileError::InvalidFormat(format!(
@@ -162,13 +173,12 @@ fn structured_key(content: &str) ->
crate::hfile::error::Result<Vec<u8>> {
content.len()
))
})?;
- let mut key = Vec::with_capacity(content.len() + 12);
- key.extend_from_slice(&len.to_be_bytes());
- key.extend_from_slice(content.as_bytes());
- key.push(0);
- key.extend_from_slice(&i64::MAX.to_be_bytes());
- key.push(4);
- Ok(key)
+ out.extend_from_slice(&len.to_be_bytes());
+ out.extend_from_slice(content.as_bytes());
+ out.push(0);
+ out.extend_from_slice(&i64::MAX.to_be_bytes());
+ out.push(4);
+ Ok(())
}
fn write_block(block_type: HFileBlockType, data: &[u8], previous_offset: i64)
-> Result<Vec<u8>> {
