lxy-9602 commented on code in PR #278:
URL: https://github.com/apache/paimon-cpp/pull/278#discussion_r3931999753
##########
src/paimon/format/blob/blob_file_batch_reader.cpp:
##########
@@ -252,9 +368,219 @@ Result<std::shared_ptr<arrow::Array>>
BlobFileBatchReader::BuildContentArray(
return std::make_shared<arrow::StructArray>(struct_array_data);
}
+Result<std::shared_ptr<arrow::Array>> BlobFileBatchReader::BuildMapBlobArray(
+ int32_t rows_to_read) const {
+ const auto& struct_type = static_cast<const
arrow::StructType&>(*target_type_);
+ const std::shared_ptr<arrow::Field>& map_field = struct_type.field(0);
+ auto map_type = checked_pointer_cast<arrow::MapType>(map_field->type());
+ const std::shared_ptr<arrow::DataType>& key_type = map_type->key_type();
+ if (key_type->id() == arrow::Type::STRING) {
+ arrow::util::InitializeUTF8();
+ }
+ PAIMON_ASSIGN_OR_RAISE(int32_t fixed_key_length,
GetMapBlobFixedKeyLength(key_type));
+
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr<arrow::ArrayBuilder>
key_builder_unique,
+ arrow::MakeBuilder(key_type,
arrow_pool_.get()));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr<arrow::ArrayBuilder>
item_builder_unique,
+
arrow::MakeBuilder(map_type->item_type(), arrow_pool_.get()));
+ std::shared_ptr<arrow::ArrayBuilder>
key_builder(std::move(key_builder_unique));
+ std::shared_ptr<arrow::ArrayBuilder>
item_builder(std::move(item_builder_unique));
+ if (!item_builder || !item_builder->type() ||
+ item_builder->type()->id() != arrow::Type::LARGE_BINARY) {
+ return Status::Invalid("cast MAP<..., BLOB> item builder to large
binary builder failed");
+ }
+ auto* blob_builder =
checked_cast<arrow::LargeBinaryBuilder*>(item_builder.get());
+ arrow::MapBuilder map_builder(arrow_pool_.get(), key_builder,
item_builder, map_type);
+
+ for (int32_t k = 0; k < rows_to_read; ++k) {
+ const size_t row_index = current_pos_ + k;
+ if (IsTargetNull(row_index)) {
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull());
+ continue;
+ }
+ if (IsTargetPlaceholder(row_index)) {
+ // Duplicate map keys cannot occur in a valid Paimon map, so two
empty/default keys
+ // with null values form an unambiguous, Arrow-valid internal
sentinel.
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->AppendEmptyValues(2));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->AppendNulls(2));
+ continue;
Review Comment:
C++ additionally supports selected keys for map fields.
The MAP placeholder is encoded as two identical default keys with both
values set to null, and the fallback logic can recognize it only if the map
still contains those two records. However, in the multi-sequence-layer read
path, data first goes through FieldMappingReader. When a field has
paimon.map.selected-keys and the empty string is not selected, the two
default-key placeholder entries are filtered out first and become a valid empty
map. The fallback logic then treats it as a real value from the newer layer and
no longer falls back to the older layer, which silently loses data from the
previous layer.
As a short-term solution, we can explicitly mark map<..., blob> as not
supporting selected keys.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
