zanmato1984 commented on code in PR #43763:
URL: https://github.com/apache/arrow/pull/43763#discussion_r1737515455
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -250,6 +259,51 @@ struct ARROW_EXPORT NullKeyEncoder : KeyEncoder {
}
};
+/// RowEncoder encodes ExecSpan to a variable length byte sequence
+/// created by concatenating the encoded form of each column. The encoding
+/// for each column depends on its datatype.
Review Comment:
```suggestion
/// for each column depends on its data type.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
Review Comment:
```suggestion
// If Encoder is for null type, the length would add 0.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
Review Comment:
```suggestion
// Increment the values in the lengths array by the length of the encoded
key for the corresponding value in the given column.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -250,6 +259,51 @@ struct ARROW_EXPORT NullKeyEncoder : KeyEncoder {
}
};
+/// RowEncoder encodes ExecSpan to a variable length byte sequence
+/// created by concatenating the encoded form of each column. The encoding
+/// for each column depends on its datatype.
+///
+/// This is used to encode rows for grouping and joining operations.
+///
+/// Unlike DuckDB and arrow-rs, currently this row format can not help
+/// sortings because the row-format is uncomparable.
+///
+/// The row format is composed of the the KeyColumn encodings for each,
+/// and the column is encoded as follows:
+/// 1. A null byte for each column, indicating whether the column is null.
+/// "1" for null, "0" for non-null.
+/// 2. The "fixed width" encoding for the column, it would exists whether
+/// the column is null or not.
+/// 3. The "variable width" encoding for the column, it would exists only
+/// for non-null string/binary columns.
+///
+/// ## Null Type
+///
+/// Null Type is a special case, it doesn't occupy any space in the encoded
row.
+///
+/// ## Fixed Width Type
+///
+/// Fixed Width Type is encoded as a fixed-width byte sequence. For example:
+/// ```
+/// Int8: [5, null, 6]
+/// ```
+/// Would be encoded as [0 5 1 0 0 6].
+///
+/// ### Dictionary Type
+///
+/// Dictionary Type is encoded as a fixed-width byte sequence using dictionary
+/// indices, the dictionary should be same for all rows.
Review Comment:
```suggestion
/// indices, the dictionary should be identical for all rows.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.cc:
##########
@@ -145,41 +145,35 @@ void FixedWidthKeyEncoder::AddLengthNull(int32_t* length)
{
Status FixedWidthKeyEncoder::Encode(const ExecValue& data, int64_t
batch_length,
uint8_t** encoded_bytes) {
+ auto handle_next_valid_value = [&](std::string_view bytes) {
Review Comment:
Is it possible to use a `const std::string_view &` instead?
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -250,6 +259,51 @@ struct ARROW_EXPORT NullKeyEncoder : KeyEncoder {
}
};
+/// RowEncoder encodes ExecSpan to a variable length byte sequence
+/// created by concatenating the encoded form of each column. The encoding
+/// for each column depends on its datatype.
+///
+/// This is used to encode rows for grouping and joining operations.
Review Comment:
```suggestion
/// This is used to encode columns into row-major format, which will be
beneficial for grouping and joining operations.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
+ // It's a special case for AddLength like `AddLength(Null-Scalar, 1,
lengths)`.
virtual void AddLengthNull(int32_t* length) = 0;
+ // Encode the value into the encoded_bytes buffer.
Review Comment:
```suggestion
// Encode the column into the encoded_bytes, which is an array of pointers
to each row buffer.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
+ // It's a special case for AddLength like `AddLength(Null-Scalar, 1,
lengths)`.
virtual void AddLengthNull(int32_t* length) = 0;
+ // Encode the value into the encoded_bytes buffer.
+ //
+ // If value is an array, the array-size should be batch_length.
+ // If value is a scalar, the value would repeat batch_length times.
Review Comment:
```suggestion
// If value is a scalar, the value would repeat batch_length times.
// NB: The pointers in the encoded_bytes will be advanced as values being
encoded into.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
+ // It's a special case for AddLength like `AddLength(Null-Scalar, 1,
lengths)`.
virtual void AddLengthNull(int32_t* length) = 0;
+ // Encode the value into the encoded_bytes buffer.
+ //
+ // If value is an array, the array-size should be batch_length.
+ // If value is a scalar, the value would repeat batch_length times.
virtual Status Encode(const ExecValue&, int64_t batch_length,
uint8_t** encoded_bytes) = 0;
+ // Add a null byte to the encoded_bytes buffer.
Review Comment:
```suggestion
// Encode a null value into the encoded_bytes, which is an array of
pointers to each row buffer.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
Review Comment:
```suggestion
// Increment the length by the length of an encoded null value.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -142,41 +159,33 @@ struct ARROW_EXPORT VarLengthKeyEncoder : KeyEncoder {
Status Encode(const ExecValue& data, int64_t batch_length,
uint8_t** encoded_bytes) override {
+ auto handle_next_valid_value = [&encoded_bytes](std::string_view bytes) {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kValidByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
+ encoded_ptr += sizeof(Offset);
+ memcpy(encoded_ptr, bytes.data(), bytes.size());
+ encoded_ptr += bytes.size();
+ };
+ auto handle_next_null_value = [&encoded_bytes]() {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kNullByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(0));
+ encoded_ptr += sizeof(Offset);
+ };
if (data.is_array()) {
- VisitArraySpanInline<T>(
- data.array,
- [&](std::string_view bytes) {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kValidByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
- encoded_ptr += sizeof(Offset);
- memcpy(encoded_ptr, bytes.data(), bytes.size());
- encoded_ptr += bytes.size();
- },
- [&] {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kNullByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(0));
- encoded_ptr += sizeof(Offset);
- });
+ VisitArraySpanInline<T>(data.array, handle_next_valid_value,
Review Comment:
```suggestion
DCHECK_EQ(data.length(), batch_length);
VisitArraySpanInline<T>(data.array, handle_next_valid_value,
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -270,14 +325,17 @@ class ARROW_EXPORT RowEncoder {
}
int32_t num_rows() const {
- return offsets_.size() == 0 ? 0 : static_cast<int32_t>(offsets_.size() -
1);
+ return offsets_.empty() ? 0 : static_cast<int32_t>(offsets_.size() - 1);
}
private:
ExecContext* ctx_{nullptr};
std::vector<std::shared_ptr<KeyEncoder>> encoders_;
+ // The offsets of each row in the encoded bytes.
+ // The size would be num_rows + 1 if there are rows.
std::vector<int32_t> offsets_;
std::vector<uint8_t> bytes_;
+ // A fixed nulls buffer for all null rows(kRowIdForNulls).
Review Comment:
```suggestion
// A constant row with all its columns encoded as null, referred as
kRowIdForNulls.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -142,41 +159,33 @@ struct ARROW_EXPORT VarLengthKeyEncoder : KeyEncoder {
Status Encode(const ExecValue& data, int64_t batch_length,
uint8_t** encoded_bytes) override {
+ auto handle_next_valid_value = [&encoded_bytes](std::string_view bytes) {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kValidByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
+ encoded_ptr += sizeof(Offset);
+ memcpy(encoded_ptr, bytes.data(), bytes.size());
+ encoded_ptr += bytes.size();
+ };
+ auto handle_next_null_value = [&encoded_bytes]() {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kNullByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(0));
+ encoded_ptr += sizeof(Offset);
+ };
if (data.is_array()) {
- VisitArraySpanInline<T>(
- data.array,
- [&](std::string_view bytes) {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kValidByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
- encoded_ptr += sizeof(Offset);
- memcpy(encoded_ptr, bytes.data(), bytes.size());
- encoded_ptr += bytes.size();
- },
- [&] {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kNullByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(0));
- encoded_ptr += sizeof(Offset);
- });
+ VisitArraySpanInline<T>(data.array, handle_next_valid_value,
Review Comment:
There are other places of this pattern, e.g., in other encoder sub-classes,
and `AddLength` methods. Could you update them as well?
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -142,41 +159,33 @@ struct ARROW_EXPORT VarLengthKeyEncoder : KeyEncoder {
Status Encode(const ExecValue& data, int64_t batch_length,
uint8_t** encoded_bytes) override {
+ auto handle_next_valid_value = [&encoded_bytes](std::string_view bytes) {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kValidByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
+ encoded_ptr += sizeof(Offset);
+ memcpy(encoded_ptr, bytes.data(), bytes.size());
+ encoded_ptr += bytes.size();
+ };
+ auto handle_next_null_value = [&encoded_bytes]() {
+ auto& encoded_ptr = *encoded_bytes++;
+ *encoded_ptr++ = kNullByte;
+ util::SafeStore(encoded_ptr, static_cast<Offset>(0));
+ encoded_ptr += sizeof(Offset);
+ };
if (data.is_array()) {
- VisitArraySpanInline<T>(
- data.array,
- [&](std::string_view bytes) {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kValidByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(bytes.size()));
- encoded_ptr += sizeof(Offset);
- memcpy(encoded_ptr, bytes.data(), bytes.size());
- encoded_ptr += bytes.size();
- },
- [&] {
- auto& encoded_ptr = *encoded_bytes++;
- *encoded_ptr++ = kNullByte;
- util::SafeStore(encoded_ptr, static_cast<Offset>(0));
- encoded_ptr += sizeof(Offset);
- });
+ VisitArraySpanInline<T>(data.array, handle_next_valid_value,
Review Comment:
Just found in this branch `batch_length` is not actually used nor checked -
meaning we are assuming `data` to be of length `batch_length`. Better assert it.
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
+ // It's a special case for AddLength like `AddLength(Null-Scalar, 1,
lengths)`.
virtual void AddLengthNull(int32_t* length) = 0;
+ // Encode the value into the encoded_bytes buffer.
+ //
+ // If value is an array, the array-size should be batch_length.
+ // If value is a scalar, the value would repeat batch_length times.
virtual Status Encode(const ExecValue&, int64_t batch_length,
uint8_t** encoded_bytes) = 0;
+ // Add a null byte to the encoded_bytes buffer.
+ //
+ // It's a special case for Encode like `Encode(Null-Scalar, 1,
encoded_bytes)`.
virtual void EncodeNull(uint8_t** encoded_bytes) = 0;
+ // Decode the encoded key from `encoded_bytes` buffer to an ArrayData.
Review Comment:
```suggestion
// Decode the encoded key from the encoded_bytes, which is an array of
pointers to each row buffer, into an ArrayData.
//
// NB: The pointers in the encoded_bytes will be advanced as values being
decoded from.
```
##########
cpp/src/arrow/compute/row/row_encoder_internal.h:
##########
@@ -38,16 +38,33 @@ struct ARROW_EXPORT KeyEncoder {
virtual ~KeyEncoder() = default;
+ // Increment the length of the encoded key for the given value to the
lengths array.
+ //
+ // Generally if Encoder is for a fixed-width type, the length of the encoded
key
+ // would add ExtraByteForNull + byte_width.
+ // If Encoder is for a variable-width type, the length would add
ExtraByteForNull +
+ // sizeof(Offset) + buffer_size.
+ // If Encoder is null type, the length would add 0.
virtual void AddLength(const ExecValue& value, int64_t batch_length,
int32_t* lengths) = 0;
+ // Increment the length for a null value.
+ // It's a special case for AddLength like `AddLength(Null-Scalar, 1,
lengths)`.
virtual void AddLengthNull(int32_t* length) = 0;
+ // Encode the value into the encoded_bytes buffer.
+ //
+ // If value is an array, the array-size should be batch_length.
+ // If value is a scalar, the value would repeat batch_length times.
virtual Status Encode(const ExecValue&, int64_t batch_length,
uint8_t** encoded_bytes) = 0;
+ // Add a null byte to the encoded_bytes buffer.
+ //
+ // It's a special case for Encode like `Encode(Null-Scalar, 1,
encoded_bytes)`.
Review Comment:
```suggestion
// It's a special case for Encode like `Encode(Null-Scalar, 1,
encoded_bytes)`.
// NB: The pointers in the encoded_bytes will be advanced as values being
encoded into.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
