lszskye commented on code in PR #195:
URL: https://github.com/apache/paimon-cpp/pull/195#discussion_r3772093064
##########
src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp:
##########
@@ -34,14 +35,254 @@
#include "paimon/core/casting/casting_utils.h"
namespace paimon {
+namespace {
+
+Result<std::string_view> GetStringMapKey(const std::shared_ptr<arrow::Array>&
keys, int64_t index) {
+ if (keys->IsNull(index)) {
+ return Status::Invalid("selected-key MAP read found a null MAP key");
+ }
+ if (keys->type_id() == arrow::Type::STRING) {
+ return
arrow::internal::checked_pointer_cast<arrow::StringArray>(keys)->GetView(index);
+ } else if (keys->type_id() == arrow::Type::DICTIONARY) {
+ auto dictionary =
arrow::internal::checked_pointer_cast<arrow::DictionaryArray>(keys);
+ int64_t dictionary_index = dictionary->GetValueIndex(index);
+ const auto& values = dictionary->dictionary();
+ if (values->IsNull(dictionary_index)) {
+ return Status::Invalid("selected-key MAP read found a null
dictionary MAP key");
+ }
+ if (values->type_id() == arrow::Type::STRING) {
+ return
arrow::internal::checked_pointer_cast<arrow::StringArray>(values)->GetView(
+ dictionary_index);
+ }
+ }
+ return Status::Invalid(
+ fmt::format("selected-key MAP read only supports string or dictionary
key array"));
+}
+
+std::vector<std::pair<std::string, int32_t>> ResolveSelectedKeyIds(
+ const MapSharedShreddingFieldMeta& meta, const std::vector<std::string>&
selected_keys) {
+ std::vector<std::pair<std::string, int32_t>> selected_key_ids;
+ selected_key_ids.reserve(selected_keys.size());
+ for (const auto& selected_key : selected_keys) {
+ auto id_iter = meta.name_to_id.find(selected_key);
+ if (id_iter != meta.name_to_id.end()) {
+ selected_key_ids.emplace_back(selected_key, id_iter->second);
+ }
+ }
+ return selected_key_ids;
+}
+
+void CollectPhysicalColumns(
+ const std::shared_ptr<arrow::StructArray>& physical_struct_array,
+ std::map<std::string, std::shared_ptr<arrow::Array>>*
physical_column_name_to_array,
+ std::shared_ptr<arrow::MapArray>* overflow_array) {
+ const auto& struct_type = physical_struct_array->struct_type();
+ for (int32_t i = 0; i < struct_type->num_fields(); ++i) {
+ const auto& sub_field = struct_type->field(i);
+ if (sub_field->name() == MapSharedShreddingDefine::kFieldMapping) {
+ continue;
+ }
+ if (sub_field->name() == MapSharedShreddingDefine::kOverflow) {
+ *overflow_array =
arrow::internal::checked_pointer_cast<arrow::MapArray>(
+ physical_struct_array->field(i));
+ continue;
+ }
+ (*physical_column_name_to_array)[sub_field->name()] =
physical_struct_array->field(i);
+ }
+}
+
+class FullMapReadPlan : public MapFieldReadPlan {
+ public:
+ FullMapReadPlan(const std::shared_ptr<arrow::Field>& logical_field,
+ const std::shared_ptr<arrow::Field>& physical_read_field,
+ std::vector<std::pair<std::string, int32_t>>&&
selected_key_ids)
+ : MapFieldReadPlan(logical_field, physical_read_field),
+ selected_key_ids_(std::move(selected_key_ids)),
+ logical_map_type_(
+
arrow::internal::checked_pointer_cast<arrow::MapType>(logical_field->type())) {}
+
+ Result<std::shared_ptr<arrow::Array>> Materialize(
+ const std::shared_ptr<arrow::Array>& physical_array,
+ arrow::MemoryPool* arrow_pool) const override;
+
+ private:
+ std::vector<std::pair<std::string, int32_t>> selected_key_ids_;
+ std::shared_ptr<arrow::MapType> logical_map_type_;
+};
+
+class SharedSelectedKeysReadPlan : public MapFieldReadPlan {
+ public:
+ struct SelectedKey {
+ int32_t field_id = -1;
+ std::vector<int32_t> candidate_columns;
+ bool may_use_overflow = false;
+ };
+
+ SharedSelectedKeysReadPlan(const std::shared_ptr<arrow::Field>&
logical_field,
+ const std::shared_ptr<arrow::Field>&
physical_read_field,
+ std::vector<SelectedKey>&& selected_keys)
+ : MapFieldReadPlan(logical_field, physical_read_field),
+ selected_keys_(std::move(selected_keys)) {}
+
+ Result<std::shared_ptr<arrow::Array>> Materialize(
+ const std::shared_ptr<arrow::Array>& physical_array,
+ arrow::MemoryPool* arrow_pool) const override;
+
+ private:
+ std::vector<SelectedKey> selected_keys_;
+};
+
+class DefaultSelectedKeysReadPlan : public MapFieldReadPlan {
+ public:
+ DefaultSelectedKeysReadPlan(const std::shared_ptr<arrow::Field>&
logical_field,
+ const std::shared_ptr<arrow::Field>&
physical_read_field,
+ const std::vector<std::string>& selected_keys)
+ : MapFieldReadPlan(logical_field, physical_read_field),
selected_keys_(selected_keys) {}
+
+ Result<std::shared_ptr<arrow::Array>> Materialize(
+ const std::shared_ptr<arrow::Array>& physical_array,
+ arrow::MemoryPool* arrow_pool) const override;
+
+ private:
+ std::vector<std::string> selected_keys_;
+};
+
+} // namespace
+
+Result<std::unique_ptr<MapFieldReadPlan>>
MapFieldReadPlanFactory::CreateFullMapReadPlan(
+ const std::shared_ptr<arrow::Field>& logical_map_field, const
MapSharedShreddingFieldMeta& meta,
+ const std::vector<std::string>& selected_keys) {
+ if (logical_map_field->type()->id() != arrow::Type::MAP) {
Review Comment:
(1)cpp paimon will support selected_keys for default map.
(2)The function name `CreateFullMapReadPlan` will cause misunderstanding,
I'll fix it.
(3)More cases going through will be added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
