fresh-borzoni commented on code in PR #430:
URL: https://github.com/apache/fluss-rust/pull/430#discussion_r2900688553
##########
crates/fluss/src/record/arrow.rs:
##########
@@ -251,26 +253,47 @@ impl RowAppendRecordBatchBuilder {
})
}
- fn create_builder(data_type: &arrow_schema::DataType) -> Result<Box<dyn
ArrayBuilder>> {
+ fn create_builder(
+ data_type: &arrow_schema::DataType,
+ capacity: usize,
+ ) -> Result<Box<dyn ArrayBuilder>> {
match data_type {
- arrow_schema::DataType::Int8 => Ok(Box::new(Int8Builder::new())),
- arrow_schema::DataType::Int16 => Ok(Box::new(Int16Builder::new())),
- arrow_schema::DataType::Int32 => Ok(Box::new(Int32Builder::new())),
- arrow_schema::DataType::Int64 => Ok(Box::new(Int64Builder::new())),
- arrow_schema::DataType::UInt8 => Ok(Box::new(UInt8Builder::new())),
- arrow_schema::DataType::UInt16 =>
Ok(Box::new(UInt16Builder::new())),
- arrow_schema::DataType::UInt32 =>
Ok(Box::new(UInt32Builder::new())),
- arrow_schema::DataType::UInt64 =>
Ok(Box::new(UInt64Builder::new())),
- arrow_schema::DataType::Float32 =>
Ok(Box::new(Float32Builder::new())),
- arrow_schema::DataType::Float64 =>
Ok(Box::new(Float64Builder::new())),
- arrow_schema::DataType::Boolean =>
Ok(Box::new(BooleanBuilder::new())),
- arrow_schema::DataType::Utf8 => Ok(Box::new(StringBuilder::new())),
- arrow_schema::DataType::Binary =>
Ok(Box::new(BinaryBuilder::new())),
- arrow_schema::DataType::FixedSizeBinary(size) => {
- Ok(Box::new(FixedSizeBinaryBuilder::new(*size)))
+ arrow_schema::DataType::Int8 =>
Ok(Box::new(Int8Builder::with_capacity(capacity))),
+ arrow_schema::DataType::Int16 =>
Ok(Box::new(Int16Builder::with_capacity(capacity))),
+ arrow_schema::DataType::Int32 =>
Ok(Box::new(Int32Builder::with_capacity(capacity))),
+ arrow_schema::DataType::Int64 =>
Ok(Box::new(Int64Builder::with_capacity(capacity))),
+ arrow_schema::DataType::UInt8 =>
Ok(Box::new(UInt8Builder::with_capacity(capacity))),
+ arrow_schema::DataType::UInt16 =>
Ok(Box::new(UInt16Builder::with_capacity(capacity))),
+ arrow_schema::DataType::UInt32 =>
Ok(Box::new(UInt32Builder::with_capacity(capacity))),
+ arrow_schema::DataType::UInt64 =>
Ok(Box::new(UInt64Builder::with_capacity(capacity))),
+ arrow_schema::DataType::Float32 => {
+ Ok(Box::new(Float32Builder::with_capacity(capacity)))
}
+ arrow_schema::DataType::Float64 => {
+ Ok(Box::new(Float64Builder::with_capacity(capacity)))
+ }
+ arrow_schema::DataType::Boolean => {
+ Ok(Box::new(BooleanBuilder::with_capacity(capacity)))
+ }
+ arrow_schema::DataType::Utf8 => {
+ // Estimate 64 bytes average per string value
+ Ok(Box::new(StringBuilder::with_capacity(
+ capacity,
+ capacity * 64,
+ )))
+ }
+ arrow_schema::DataType::Binary => {
+ // Estimate 64 bytes average per binary value
+ Ok(Box::new(BinaryBuilder::with_capacity(
+ capacity,
+ capacity * 64,
Review Comment:
Will extract to a const 👍
On the multiplier: Java's `BaseVariableWidthVector` default of 8 is a
conservative estimate for general-purpose Arrow vectors. Here we're pre-sizing
for a known batch of 256 records where the goal is zero reallocations. 8 bytes
would trigger a grow on any typical string (names, IDs, etc.). 64 × 256 = 16 KB
per column is still small and a better fit.
Also it will still be changed after 2d follow, for now 64b is a good
approximation for base scenario.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
