luoyuxia commented on code in PR #124: URL: https://github.com/apache/fluss-rust/pull/124#discussion_r2666610507
########## crates/fluss/src/row/field_getter.rs: ########## @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::DataType; +use crate::row::{Datum, InternalRow}; + +pub trait FieldGetter { Review Comment: I'm wondering whether it will be better to use enum than trait? ``` impl FieldGetter { pub fn get_field<'a>(&self, row: &'a dyn InternalRow) -> Datum<'a> { match self { Self::NotNull(getter) => getter.get_field(row), Self::Nullable(getter) => { if row.is_null_at(getter.position()) { Datum::Null } else { getter.get_field(row) } } } } pub fn create(data_type: &DataType, pos: usize) -> Self { let inner_field_getter = match data_type { DataType::Char(t) => InnerFieldGetter::Char { pos, len: t.length() as usize, }, DataType::String(_) => InnerFieldGetter::String { pos }, DataType::Boolean(_) => InnerFieldGetter::Boolean { pos }, DataType::Binary(t) => InnerFieldGetter::Binary { pos, len: t.length(), }, DataType::Bytes(_) => InnerFieldGetter::Bytes { pos }, DataType::TinyInt(_) => InnerFieldGetter::TinyInt { pos }, DataType::SmallInt(_) => InnerFieldGetter::SmallInt { pos }, DataType::Int(_) => InnerFieldGetter::Int { pos }, DataType::BigInt(_) => InnerFieldGetter::BigInt { pos }, DataType::Float(_) => InnerFieldGetter::Float { pos }, DataType::Double(_) => InnerFieldGetter::Double { pos }, _ => unimplemented!("DataType {:?} getter not implemented", data_type), }; if data_type.is_nullable() { Self::Nullable(inner_field_getter) } else { Self::NotNull(inner_field_getter) } } } pub enum InnerFieldGetter { Char { pos: usize, len: usize }, String { pos: usize }, Boolean { pos: usize }, Binary { pos: usize, len: usize }, Bytes { pos: usize }, TinyInt { pos: usize }, SmallInt { pos: usize }, Int { pos: usize }, BigInt { pos: usize }, Float { pos: usize }, Double { pos: usize }, // TODO: Decimal, Date, Timestamp } impl InnerFieldGetter { pub fn get_field<'a>(&self, row: &'a dyn InternalRow) -> Datum<'a> { match self { Self::Char { pos, len } => Datum::from(row.get_char(*pos, *len)), Self::String { pos } => Datum::from(row.get_string(*pos)), Self::Boolean { pos } => Datum::from(row.get_boolean(*pos)), Self::Binary { pos, len } => Datum::from(row.get_binary(*pos, *len)), Self::Bytes { pos } => Datum::from(row.get_bytes(*pos)), Self::TinyInt { pos } => Datum::from(row.get_byte(*pos)), Self::SmallInt { pos } => Datum::from(row.get_short(*pos)), Self::Int { pos } => Datum::from(row.get_int(*pos)), Self::BigInt { pos } => Datum::from(row.get_long(*pos)), Self::Float { pos } => Datum::from(row.get_float(*pos)), Self::Double { pos } => Datum::from(row.get_double(*pos)), } } fn position<'a>(&self) -> usize { match self { Self::Char { pos, .. } | Self::String { pos } | Self::Int { pos } => *pos, // ... } } } ``` ########## crates/fluss/src/row/compacted/compacted_key_writer.rs: ########## @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::row::compacted::compacted_row_writer::CompactedRowWriter; +use bytes::Bytes; + +use crate::error::Error::IllegalArgument; +use crate::error::Result; +use crate::metadata::DataType; +use crate::row::Datum; +use crate::row::binary::{BinaryRowFormat, BinaryWriter, ValueWriter}; +use delegate::delegate; + +/// A wrapping of [`CompactedRowWriter`] used to encode key columns. +/// The encoding is the same as [`CompactedRowWriter`], but is without header of null bits to +/// represent whether the field value is null or not since the key columns must be not null. +pub struct CompactedKeyWriter { + delegate: CompactedRowWriter, +} + +impl CompactedKeyWriter { + pub fn new() -> CompactedKeyWriter { + CompactedKeyWriter { + // in compacted key encoder, we don't need to set null bits as the key columns must be not + // null, to use field count 0 to init to make the null bits 0 + delegate: CompactedRowWriter::new(0), + } + } + + pub fn create_value_writer(field_type: &DataType) -> Result<Box<dyn ValueWriter>> { + let inner = <dyn ValueWriter>::create_not_null_value_writer( + field_type, + Some(&BinaryRowFormat::Compacted), + )?; + Ok(RejectNullValueWriter::wrap(field_type.clone(), inner)) + } + + delegate! { + to self.delegate { + pub fn reset(&mut self); + + #[allow(dead_code)] + pub fn position(&self) -> usize; + + #[allow(dead_code)] + pub fn buffer(&self) -> &[u8]; + + pub fn to_bytes(&self) -> Bytes; + } + } +} + +pub struct RejectNullValueWriter { + field_type: DataType, + delegate: Box<dyn ValueWriter>, +} + +impl RejectNullValueWriter { + fn wrap(field_type: DataType, delegate: Box<dyn ValueWriter>) -> Box<dyn ValueWriter> { + Box::new(RejectNullValueWriter { + field_type, + delegate, + }) + } +} +impl ValueWriter for RejectNullValueWriter { + fn write_value(&self, writer: &mut dyn BinaryWriter, pos: usize, value: &Datum) -> Result<()> { + if let Datum::Null = value { + return Err(IllegalArgument { + message: format!( + "Null value is not allowed for compacted key encoder in position {} with type {}", + pos, self.field_type + ), + }); + } + self.delegate.write_value(writer, pos, value) + } +} + +impl BinaryWriter for CompactedKeyWriter { + delegate! { + to self.delegate { + fn reset(&mut self); + + fn set_null_at(&mut self, pos: usize); + + fn write_boolean(&mut self, value: bool); + + fn write_byte(&mut self, value: u8); + + fn write_binary(&mut self, bytes: &[u8], length: usize); + + fn write_bytes(&mut self, value: &[u8]); + + fn write_char(&mut self, value: &str, _length: usize); + + fn write_string(&mut self, value: &str); + + fn write_short(&mut self, value: i16); + + fn write_int(&mut self, value: i32); + + fn write_long(&mut self, value: i64); + + fn write_float(&mut self, value: f32); + + fn write_double(&mut self, value: f64); + + + } + } + + fn complete(&mut self) {} Review Comment: nit: add comment ``` do nothing ``` if we really don't need any thing in this method ########## crates/fluss/src/row/binary/binary_writer.rs: ########## @@ -0,0 +1,359 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::error::Error::{IllegalArgument, IoUnsupported}; +use crate::error::Result; +use crate::metadata::DataType; +use crate::row::Datum; +use crate::row::binary::BinaryRowFormat; + +/// Writer to write a composite data format, like row, array, +#[allow(dead_code)] +pub trait BinaryWriter { + /// Reset writer to prepare next write + fn reset(&mut self); + + /// Set null to this field + fn set_null_at(&mut self, pos: usize); + + fn write_boolean(&mut self, value: bool); + + fn write_byte(&mut self, value: u8); + + fn write_bytes(&mut self, value: &[u8]); + + fn write_char(&mut self, value: &str, length: usize); + + fn write_string(&mut self, value: &str); + + fn write_short(&mut self, value: i16); + + fn write_int(&mut self, value: i32); + + fn write_long(&mut self, value: i64); + + fn write_float(&mut self, value: f32); + + fn write_double(&mut self, value: f64); + + fn write_binary(&mut self, bytes: &[u8], length: usize); + + // TODO Decimal type + // fn write_decimal(&mut self, pos: i32, value: f64); + + // TODO Timestamp type + // fn write_timestamp_ntz(&mut self, pos: i32, value: i64); + + // TODO Timestamp type + // fn write_timestamp_ltz(&mut self, pos: i32, value: i64); + + // TODO InternalArray, ArraySerializer + // fn write_array(&mut self, pos: i32, value: i64); + + // TODO Row serializer + // fn write_row(&mut self, pos: i32, value: &InternalRow); + + /// Finally, complete write to set real size to binary. + fn complete(&mut self); +} + +/// Accessor for writing the fields/elements of a binary writer during runtime, the +/// fields/elements must be written in the order. +pub trait ValueWriter { Review Comment: Also, will it be better use enum than trait? ########## crates/fluss/src/row/compacted/compacted_key_writer.rs: ########## @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::row::compacted::compacted_row_writer::CompactedRowWriter; +use bytes::Bytes; + +use crate::error::Error::IllegalArgument; +use crate::error::Result; +use crate::metadata::DataType; +use crate::row::Datum; +use crate::row::binary::{BinaryRowFormat, BinaryWriter, ValueWriter}; +use delegate::delegate; + +/// A wrapping of [`CompactedRowWriter`] used to encode key columns. +/// The encoding is the same as [`CompactedRowWriter`], but is without header of null bits to +/// represent whether the field value is null or not since the key columns must be not null. +pub struct CompactedKeyWriter { + delegate: CompactedRowWriter, +} + +impl CompactedKeyWriter { + pub fn new() -> CompactedKeyWriter { + CompactedKeyWriter { + // in compacted key encoder, we don't need to set null bits as the key columns must be not + // null, to use field count 0 to init to make the null bits 0 + delegate: CompactedRowWriter::new(0), + } + } + + pub fn create_value_writer(field_type: &DataType) -> Result<Box<dyn ValueWriter>> { + let inner = <dyn ValueWriter>::create_not_null_value_writer( + field_type, + Some(&BinaryRowFormat::Compacted), + )?; + Ok(RejectNullValueWriter::wrap(field_type.clone(), inner)) + } + + delegate! { + to self.delegate { + pub fn reset(&mut self); + + #[allow(dead_code)] + pub fn position(&self) -> usize; + + #[allow(dead_code)] + pub fn buffer(&self) -> &[u8]; + + pub fn to_bytes(&self) -> Bytes; + } + } +} + +pub struct RejectNullValueWriter { + field_type: DataType, + delegate: Box<dyn ValueWriter>, +} + +impl RejectNullValueWriter { + fn wrap(field_type: DataType, delegate: Box<dyn ValueWriter>) -> Box<dyn ValueWriter> { + Box::new(RejectNullValueWriter { + field_type, + delegate, + }) + } +} +impl ValueWriter for RejectNullValueWriter { + fn write_value(&self, writer: &mut dyn BinaryWriter, pos: usize, value: &Datum) -> Result<()> { + if let Datum::Null = value { + return Err(IllegalArgument { + message: format!( + "Null value is not allowed for compacted key encoder in position {} with type {}", + pos, self.field_type + ), + }); + } + self.delegate.write_value(writer, pos, value) + } +} + +impl BinaryWriter for CompactedKeyWriter { + delegate! { + to self.delegate { + fn reset(&mut self); Review Comment: nit: add `#[inline]` to avoid another function call ########## crates/fluss/src/row/mod.rs: ########## @@ -78,6 +82,7 @@ pub struct GenericRow<'a> { pub values: Vec<Datum<'a>>, } +// TODO Decide if migrate to Result<?> Review Comment: what does the todo mean? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
