leekeiabstraction commented on code in PR #156: URL: https://github.com/apache/fluss-rust/pull/156#discussion_r2686766736
########## crates/fluss/src/record/kv/kv_record.rs: ########## @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-Value record implementation. +//! +//! This module provides the KvRecord struct which represents an immutable key-value record. +//! The record format is: +//! - Length => Int32 +//! - KeyLength => Unsigned VarInt +//! - Key => bytes +//! - Row => BinaryRow (optional, if null then this is a deletion record) + +use bytes::{BufMut, Bytes, BytesMut}; +use std::io::{self, Write}; + +use crate::util::varint::{ + read_unsigned_varint_bytes, size_of_unsigned_varint, write_unsigned_varint, + write_unsigned_varint_buf, +}; + +/// Length field size in bytes +pub const LENGTH_LENGTH: usize = 4; + +/// A key-value record. +/// +/// The schema is: +/// - Length => Int32 +/// - KeyLength => Unsigned VarInt +/// - Key => bytes +/// - ValueLength => Unsigned VarInt (only present if value exists, even for empty values) +/// - Value => bytes (if ValueLength > 0) +/// +/// When the value is None (deletion), no ValueLength or Value bytes are present. +/// When the value is Some(&[]) (empty value), ValueLength is present with value 0. +#[derive(Debug, Clone)] +pub struct KvRecord { + key: Bytes, + value: Option<Bytes>, +} + +impl KvRecord { + /// Create a new KvRecord with the given key and optional value. + pub fn new(key: Bytes, value: Option<Bytes>) -> Self { + Self { key, value } + } + + /// Get the key bytes. + pub fn key(&self) -> &Bytes { + &self.key + } + + /// Get the value bytes (None indicates a deletion). + pub fn value(&self) -> Option<&Bytes> { + self.value.as_ref() + } + + /// Calculate the total size of the record when serialized (including length prefix). + pub fn size_of(key: &[u8], value: Option<&[u8]>) -> usize { + Self::size_without_length(key, value) + LENGTH_LENGTH + } + + /// Calculate the size without the length prefix. + fn size_without_length(key: &[u8], value: Option<&[u8]>) -> usize { + let key_len = key.len(); + let key_len_size = size_of_unsigned_varint(key_len as u32); + + match value { + Some(v) => { + let value_len_size = size_of_unsigned_varint(v.len() as u32); + key_len_size + .saturating_add(key_len) + .saturating_add(value_len_size) Review Comment: Why do we have to add `value_len_size`? Java implementation only adds `BinaryRow.getSizeInBytes()` which presumably is the bytes length of the value. ########## crates/fluss/src/record/kv/kv_record.rs: ########## @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-Value record implementation. +//! +//! This module provides the KvRecord struct which represents an immutable key-value record. +//! The record format is: +//! - Length => Int32 +//! - KeyLength => Unsigned VarInt +//! - Key => bytes +//! - Row => BinaryRow (optional, if null then this is a deletion record) + +use bytes::{BufMut, Bytes, BytesMut}; +use std::io::{self, Write}; + +use crate::util::varint::{ + read_unsigned_varint_bytes, size_of_unsigned_varint, write_unsigned_varint, + write_unsigned_varint_buf, +}; + +/// Length field size in bytes +pub const LENGTH_LENGTH: usize = 4; + +/// A key-value record. +/// +/// The schema is: +/// - Length => Int32 +/// - KeyLength => Unsigned VarInt +/// - Key => bytes +/// - ValueLength => Unsigned VarInt (only present if value exists, even for empty values) +/// - Value => bytes (if ValueLength > 0) +/// +/// When the value is None (deletion), no ValueLength or Value bytes are present. +/// When the value is Some(&[]) (empty value), ValueLength is present with value 0. +#[derive(Debug, Clone)] +pub struct KvRecord { + key: Bytes, + value: Option<Bytes>, +} + +impl KvRecord { + /// Create a new KvRecord with the given key and optional value. + pub fn new(key: Bytes, value: Option<Bytes>) -> Self { + Self { key, value } + } + + /// Get the key bytes. + pub fn key(&self) -> &Bytes { + &self.key + } + + /// Get the value bytes (None indicates a deletion). + pub fn value(&self) -> Option<&Bytes> { + self.value.as_ref() + } + + /// Calculate the total size of the record when serialized (including length prefix). + pub fn size_of(key: &[u8], value: Option<&[u8]>) -> usize { + Self::size_without_length(key, value) + LENGTH_LENGTH + } + + /// Calculate the size without the length prefix. + fn size_without_length(key: &[u8], value: Option<&[u8]>) -> usize { + let key_len = key.len(); + let key_len_size = size_of_unsigned_varint(key_len as u32); + + match value { + Some(v) => { + let value_len_size = size_of_unsigned_varint(v.len() as u32); + key_len_size + .saturating_add(key_len) + .saturating_add(value_len_size) + .saturating_add(v.len()) + } + None => { + // Deletion: no value length varint or value bytes + key_len_size.saturating_add(key_len) + } + } + } + + /// Write a KV record to a writer. + /// + /// Returns the number of bytes written. + pub fn write_to<W: Write>( + writer: &mut W, + key: &[u8], + value: Option<&[u8]>, + ) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Record size {} exceeds i32::MAX", size_in_bytes), + ) + })?; + writer.write_all(&size_i32.to_le_bytes())?; + + let key_len = key.len() as u32; + write_unsigned_varint(key_len, writer)?; + + writer.write_all(key)?; + + if let Some(v) = value { + let value_len = v.len() as u32; + write_unsigned_varint(value_len, writer)?; + writer.write_all(v)?; + } + // For None (deletion), don't write value length or value bytes + + Ok(size_in_bytes + LENGTH_LENGTH) + } + + /// Write a KV record to a buffer. + /// + /// Returns the number of bytes written. + pub fn write_to_buf(buf: &mut BytesMut, key: &[u8], value: Option<&[u8]>) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { Review Comment: This conversion with error is done a couple of times within this file, lets put it in a function to adhere to DRY ########## crates/fluss/src/record/kv/kv_record.rs: ########## @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-Value record implementation. +//! +//! This module provides the KvRecord struct which represents an immutable key-value record. +//! The record format is: +//! - Length => Int32 +//! - KeyLength => Unsigned VarInt +//! - Key => bytes +//! - Row => BinaryRow (optional, if null then this is a deletion record) + +use bytes::{BufMut, Bytes, BytesMut}; +use std::io::{self, Write}; + +use crate::util::varint::{ + read_unsigned_varint_bytes, size_of_unsigned_varint, write_unsigned_varint, + write_unsigned_varint_buf, +}; + +/// Length field size in bytes +pub const LENGTH_LENGTH: usize = 4; + +/// A key-value record. +/// +/// The schema is: +/// - Length => Int32 +/// - KeyLength => Unsigned VarInt +/// - Key => bytes +/// - ValueLength => Unsigned VarInt (only present if value exists, even for empty values) +/// - Value => bytes (if ValueLength > 0) +/// +/// When the value is None (deletion), no ValueLength or Value bytes are present. +/// When the value is Some(&[]) (empty value), ValueLength is present with value 0. +#[derive(Debug, Clone)] +pub struct KvRecord { + key: Bytes, + value: Option<Bytes>, +} + +impl KvRecord { + /// Create a new KvRecord with the given key and optional value. + pub fn new(key: Bytes, value: Option<Bytes>) -> Self { + Self { key, value } + } + + /// Get the key bytes. + pub fn key(&self) -> &Bytes { + &self.key + } + + /// Get the value bytes (None indicates a deletion). + pub fn value(&self) -> Option<&Bytes> { + self.value.as_ref() + } + + /// Calculate the total size of the record when serialized (including length prefix). + pub fn size_of(key: &[u8], value: Option<&[u8]>) -> usize { + Self::size_without_length(key, value) + LENGTH_LENGTH + } + + /// Calculate the size without the length prefix. + fn size_without_length(key: &[u8], value: Option<&[u8]>) -> usize { + let key_len = key.len(); + let key_len_size = size_of_unsigned_varint(key_len as u32); + + match value { + Some(v) => { + let value_len_size = size_of_unsigned_varint(v.len() as u32); + key_len_size + .saturating_add(key_len) + .saturating_add(value_len_size) + .saturating_add(v.len()) + } + None => { + // Deletion: no value length varint or value bytes + key_len_size.saturating_add(key_len) + } + } + } + + /// Write a KV record to a writer. + /// + /// Returns the number of bytes written. + pub fn write_to<W: Write>( + writer: &mut W, + key: &[u8], + value: Option<&[u8]>, + ) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Record size {} exceeds i32::MAX", size_in_bytes), + ) + })?; + writer.write_all(&size_i32.to_le_bytes())?; + + let key_len = key.len() as u32; + write_unsigned_varint(key_len, writer)?; + + writer.write_all(key)?; + + if let Some(v) = value { + let value_len = v.len() as u32; + write_unsigned_varint(value_len, writer)?; + writer.write_all(v)?; + } + // For None (deletion), don't write value length or value bytes + + Ok(size_in_bytes + LENGTH_LENGTH) + } + + /// Write a KV record to a buffer. + /// + /// Returns the number of bytes written. + pub fn write_to_buf(buf: &mut BytesMut, key: &[u8], value: Option<&[u8]>) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Record size {} exceeds i32::MAX", size_in_bytes), + ) + })?; + buf.put_i32_le(size_i32); + + // Write key length as unsigned varint + let key_len = key.len() as u32; + write_unsigned_varint_buf(key_len, buf); + + buf.put_slice(key); + + // Write the value length and bytes if present (even for empty values) + if let Some(v) = value { + let value_len = v.len() as u32; + write_unsigned_varint_buf(value_len, buf); + buf.put_slice(v); + } + // For None (deletion), don't write value length or value bytes + + Ok(size_in_bytes + LENGTH_LENGTH) + } + + /// Read a KV record from bytes at the given position. + /// + /// Returns the KvRecord and the number of bytes consumed. + pub fn read_from(bytes: &Bytes, position: usize) -> io::Result<(Self, usize)> { Review Comment: This implementation seems to differ from Java side's DefaultKvRecord. Can you elaborate the approach taken here? ########## crates/fluss/src/record/kv/kv_record.rs: ########## @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-Value record implementation. +//! +//! This module provides the KvRecord struct which represents an immutable key-value record. +//! The record format is: +//! - Length => Int32 +//! - KeyLength => Unsigned VarInt +//! - Key => bytes +//! - Row => BinaryRow (optional, if null then this is a deletion record) + +use bytes::{BufMut, Bytes, BytesMut}; +use std::io::{self, Write}; + +use crate::util::varint::{ + read_unsigned_varint_bytes, size_of_unsigned_varint, write_unsigned_varint, + write_unsigned_varint_buf, +}; + +/// Length field size in bytes +pub const LENGTH_LENGTH: usize = 4; + +/// A key-value record. +/// +/// The schema is: +/// - Length => Int32 +/// - KeyLength => Unsigned VarInt +/// - Key => bytes +/// - ValueLength => Unsigned VarInt (only present if value exists, even for empty values) +/// - Value => bytes (if ValueLength > 0) +/// +/// When the value is None (deletion), no ValueLength or Value bytes are present. +/// When the value is Some(&[]) (empty value), ValueLength is present with value 0. +#[derive(Debug, Clone)] +pub struct KvRecord { + key: Bytes, + value: Option<Bytes>, +} + +impl KvRecord { + /// Create a new KvRecord with the given key and optional value. + pub fn new(key: Bytes, value: Option<Bytes>) -> Self { + Self { key, value } + } + + /// Get the key bytes. + pub fn key(&self) -> &Bytes { + &self.key + } + + /// Get the value bytes (None indicates a deletion). + pub fn value(&self) -> Option<&Bytes> { + self.value.as_ref() + } + + /// Calculate the total size of the record when serialized (including length prefix). + pub fn size_of(key: &[u8], value: Option<&[u8]>) -> usize { + Self::size_without_length(key, value) + LENGTH_LENGTH + } + + /// Calculate the size without the length prefix. + fn size_without_length(key: &[u8], value: Option<&[u8]>) -> usize { + let key_len = key.len(); + let key_len_size = size_of_unsigned_varint(key_len as u32); + + match value { + Some(v) => { + let value_len_size = size_of_unsigned_varint(v.len() as u32); + key_len_size + .saturating_add(key_len) + .saturating_add(value_len_size) + .saturating_add(v.len()) + } + None => { + // Deletion: no value length varint or value bytes + key_len_size.saturating_add(key_len) + } + } + } + + /// Write a KV record to a writer. + /// + /// Returns the number of bytes written. + pub fn write_to<W: Write>( + writer: &mut W, + key: &[u8], + value: Option<&[u8]>, + ) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Record size {} exceeds i32::MAX", size_in_bytes), + ) + })?; + writer.write_all(&size_i32.to_le_bytes())?; + + let key_len = key.len() as u32; + write_unsigned_varint(key_len, writer)?; + + writer.write_all(key)?; + + if let Some(v) = value { + let value_len = v.len() as u32; + write_unsigned_varint(value_len, writer)?; + writer.write_all(v)?; + } + // For None (deletion), don't write value length or value bytes + + Ok(size_in_bytes + LENGTH_LENGTH) + } + + /// Write a KV record to a buffer. + /// + /// Returns the number of bytes written. + pub fn write_to_buf(buf: &mut BytesMut, key: &[u8], value: Option<&[u8]>) -> io::Result<usize> { + let size_in_bytes = Self::size_without_length(key, value); + + let size_i32 = i32::try_from(size_in_bytes).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Record size {} exceeds i32::MAX", size_in_bytes), + ) + })?; + buf.put_i32_le(size_i32); + + // Write key length as unsigned varint + let key_len = key.len() as u32; + write_unsigned_varint_buf(key_len, buf); + + buf.put_slice(key); + + // Write the value length and bytes if present (even for empty values) + if let Some(v) = value { + let value_len = v.len() as u32; + write_unsigned_varint_buf(value_len, buf); + buf.put_slice(v); + } + // For None (deletion), don't write value length or value bytes + + Ok(size_in_bytes + LENGTH_LENGTH) + } + + /// Read a KV record from bytes at the given position. + /// + /// Returns the KvRecord and the number of bytes consumed. + pub fn read_from(bytes: &Bytes, position: usize) -> io::Result<(Self, usize)> { + if bytes.len() < position.saturating_add(LENGTH_LENGTH) { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "Not enough bytes to read record length", + )); + } + + let size_in_bytes_i32 = i32::from_le_bytes([ + bytes[position], + bytes[position + 1], + bytes[position + 2], + bytes[position + 3], + ]); + + if size_in_bytes_i32 < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid record length: {}", size_in_bytes_i32), + )); + } + + let size_in_bytes = size_in_bytes_i32 as usize; + + let total_size = size_in_bytes.checked_add(LENGTH_LENGTH).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Record size overflow: {} + {}", + size_in_bytes, LENGTH_LENGTH + ), + ) + })?; + + let available = bytes.len().saturating_sub(position); + if available < total_size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!( + "Not enough bytes to read record: expected {}, available {}", + total_size, available + ), + )); + } + + // Start reading from after the length field + let mut current_offset = position + LENGTH_LENGTH; + let record_end = position + total_size; + + // Read key length as unsigned varint (bounded by record end) + let (key_len, varint_size) = + read_unsigned_varint_bytes(&bytes[current_offset..record_end])?; + current_offset += varint_size; + + // Read key bytes + let key_end = current_offset + key_len as usize; + if key_end > position + total_size { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Key length exceeds record size", + )); + } + let key = bytes.slice(current_offset..key_end); + current_offset = key_end; + + let remaining_bytes = (position + total_size) - current_offset; + + let value = if remaining_bytes > 0 { + // Value is present: read value length varint + let (value_len, varint_size) = + read_unsigned_varint_bytes(&bytes[current_offset..record_end])?; + current_offset += varint_size; + + // Read value bytes + let value_end = current_offset + value_len as usize; + if value_end > record_end { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Value length exceeds record size", + )); + } + + // Use slice for both empty and non-empty values (zero-copy) + let value_bytes = bytes.slice(current_offset..value_end); + current_offset = value_end; + + // Verify no trailing bytes remain + if current_offset != record_end { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Record has trailing bytes after value", + )); + } + + Some(value_bytes) + } else { + // No remaining bytes: this is a deletion record + // Note: current_offset == record_end is guaranteed here since remaining_bytes == 0 + None + }; + + Ok((Self { key, value }, total_size)) + } + + /// Get the total size in bytes of this record. + pub fn get_size_in_bytes(&self) -> usize { + Self::size_of(&self.key, self.value.as_deref()) Review Comment: Can we store this instead of re-calculating on further function calls? ########## crates/fluss/src/record/kv/kv_record.rs: ########## @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-Value record implementation. +//! +//! This module provides the KvRecord struct which represents an immutable key-value record. +//! The record format is: +//! - Length => Int32 +//! - KeyLength => Unsigned VarInt +//! - Key => bytes +//! - Row => BinaryRow (optional, if null then this is a deletion record) + +use bytes::{BufMut, Bytes, BytesMut}; +use std::io::{self, Write}; + +use crate::util::varint::{ + read_unsigned_varint_bytes, size_of_unsigned_varint, write_unsigned_varint, + write_unsigned_varint_buf, +}; + +/// Length field size in bytes +pub const LENGTH_LENGTH: usize = 4; + +/// A key-value record. +/// +/// The schema is: +/// - Length => Int32 +/// - KeyLength => Unsigned VarInt +/// - Key => bytes +/// - ValueLength => Unsigned VarInt (only present if value exists, even for empty values) +/// - Value => bytes (if ValueLength > 0) +/// +/// When the value is None (deletion), no ValueLength or Value bytes are present. +/// When the value is Some(&[]) (empty value), ValueLength is present with value 0. +#[derive(Debug, Clone)] +pub struct KvRecord { + key: Bytes, + value: Option<Bytes>, +} + +impl KvRecord { + /// Create a new KvRecord with the given key and optional value. + pub fn new(key: Bytes, value: Option<Bytes>) -> Self { + Self { key, value } + } + + /// Get the key bytes. + pub fn key(&self) -> &Bytes { + &self.key + } + + /// Get the value bytes (None indicates a deletion). + pub fn value(&self) -> Option<&Bytes> { + self.value.as_ref() + } + + /// Calculate the total size of the record when serialized (including length prefix). + pub fn size_of(key: &[u8], value: Option<&[u8]>) -> usize { + Self::size_without_length(key, value) + LENGTH_LENGTH + } + + /// Calculate the size without the length prefix. + fn size_without_length(key: &[u8], value: Option<&[u8]>) -> usize { + let key_len = key.len(); + let key_len_size = size_of_unsigned_varint(key_len as u32); + + match value { + Some(v) => { + let value_len_size = size_of_unsigned_varint(v.len() as u32); + key_len_size + .saturating_add(key_len) + .saturating_add(value_len_size) + .saturating_add(v.len()) + } + None => { + // Deletion: no value length varint or value bytes + key_len_size.saturating_add(key_len) + } + } + } + + /// Write a KV record to a writer. + /// + /// Returns the number of bytes written. + pub fn write_to<W: Write>( Review Comment: Can you clarify why we need write_to and write_to_buf? Java side does not have 2 variants of writeTo and this function seems to be unused. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
