lxy-9602 commented on code in PR #269:
URL: https://github.com/apache/paimon-cpp/pull/269#discussion_r3915112778
##########
src/paimon/core/io/key_value_data_file_record_reader.cpp:
##########
@@ -107,26 +110,34 @@ Result<std::unique_ptr<KeyValueRecordReader::Iterator>>
KeyValueDataFileRecordRe
return Status::Invalid("cannot cast data batch to StructArray");
}
auto data_batch = checked_pointer_cast<arrow::StructArray>(arrow_array);
- if (data_batch->num_fields() <
SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT) {
- return Status::Invalid(
- fmt::format("data batch field count {} is less than required
special field count {}",
- data_batch->num_fields(),
SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT));
- }
- if (!data_batch->field(0) || data_batch->field(0)->type_id() !=
arrow::Type::INT64) {
+ std::shared_ptr<arrow::Array> sequence_number =
+ data_batch->GetFieldByName(SpecialFields::SequenceNumber().Name());
+ if (!sequence_number || sequence_number->type_id() != arrow::Type::INT64) {
return Status::Invalid("cannot cast SEQUENCE_NUMBER column to int64
arrow array");
}
sequence_number_array_ =
-
checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(data_batch->field(0));
- if (!data_batch->field(1) || data_batch->field(1)->type_id() !=
arrow::Type::INT8) {
+
checked_pointer_cast<arrow::NumericArray<arrow::Int64Type>>(sequence_number);
+ if (sequence_number_array_->null_count() != 0) {
+ return Status::Invalid("SEQUENCE_NUMBER column contains null");
+ }
+ std::shared_ptr<arrow::Array> row_kind =
+ data_batch->GetFieldByName(SpecialFields::ValueKind().Name());
+ if (!row_kind || row_kind->type_id() != arrow::Type::INT8) {
return Status::Invalid("cannot cast VALUE_KIND column to int8 arrow
array");
}
- row_kind_array_ =
-
checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(data_batch->field(1));
+ row_kind_array_ =
checked_pointer_cast<arrow::NumericArray<arrow::Int8Type>>(row_kind);
+ if (row_kind_array_->null_count() != 0) {
+ return Status::Invalid("VALUE_KIND column contains null");
+ }
arrow::ArrayVector key_fields;
key_fields.reserve(key_schema_->num_fields());
for (const auto& key_field : key_schema_->fields()) {
- // skip special fields
- key_fields.emplace_back(data_batch->GetFieldByName(key_field->name()));
+ std::shared_ptr<arrow::Array> field_array =
data_batch->GetFieldByName(key_field->name());
Review Comment:
The physical Arrow type returned by a file reader may differ from the
logical schema type. For example, a logical string field may be returned as a
dictionary-encoded array to avoid unnecessary decoding and copying (e.g., when
set `orc.read.enable-lazy-decoding` or
`parquet.read.enable-dictionary-passthrough`). The read path already handles
such representations explicitly.
Therefore, comparing the actual type directly with `key_schema_` or
`value_schema_` would reject valid reader output. For the same reason as the
discussion above, we will only verify that the required fields exist and will
not validate exact type equality here for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
