wangyong9999 commented on code in PR #253:
URL: https://github.com/apache/paimon-cpp/pull/253#discussion_r3885097028
##########
src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp:
##########
@@ -71,39 +91,86 @@ PkSortedBucketIndexState
PkSortedBucketIndexState::FromActiveDataFiles(
continue;
}
PrimaryKeyIndexSourceMeta source_meta =
std::move(source_meta_result).value();
- auto desired = sources_by_level.find(source_meta.DataLevel());
- if (desired == sources_by_level.end() || desired->second !=
source_meta.SourceFiles()) {
+ const std::vector<PrimaryKeyIndexSourceFile>& payload_sources =
source_meta.SourceFiles();
+ bool valid_candidate = !payload_sources.empty();
+ std::vector<PrimaryKeyIndexSourceFile> active_intersection;
+ for (size_t i = 0; valid_candidate && i < payload_sources.size(); i++)
{
+ const PrimaryKeyIndexSourceFile& source = payload_sources[i];
+ if (i > 0 && payload_sources[i - 1].file_name >= source.file_name)
{
+ valid_candidate = false;
+ break;
+ }
+ auto active_source = active_sources.find(source.file_name);
+ if (active_source == active_sources.end()) {
+ continue;
+ }
+ if (active_source->second != source.row_count ||
+ ambiguous_active_source_names.count(source.file_name) > 0) {
+ valid_candidate = false;
+ break;
+ }
+ active_intersection.push_back(source);
+ }
+ if (!valid_candidate || active_intersection.empty()) {
+ rejected.push_back(payload);
+ continue;
+ }
+ std::shared_ptr<PkSortedIndexGroup> group =
+ PkSortedIndexGroup::Create(field_id, index_type, payload_sources,
payload, source_meta);
+ if (group == nullptr) {
rejected.push_back(payload);
continue;
}
- payloads_by_level[source_meta.DataLevel()].push_back(payload);
-
payload_metas_by_level[source_meta.DataLevel()].push_back(std::move(source_meta));
+ candidates.push_back(
+ {payload, std::move(group), std::move(active_intersection),
/*conflicted=*/false});
+ }
+
+ std::map<std::pair<std::string, int64_t>, std::vector<size_t>>
candidates_by_source;
Review Comment:
Updated. The same-level conflict graph and its multi-group test have been
removed. The reader again accepts at most one payload group per level and only
broadens reuse to that group’s eligible active intersection.
##########
src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp:
##########
@@ -49,10 +59,20 @@ PkSortedBucketIndexState
PkSortedBucketIndexState::FromActiveDataFiles(
});
}
- // Match payloads against the expected level sources; anything that does
not decode or
- // does not exactly cover its level is rejected.
- std::map<int32_t, std::vector<std::shared_ptr<IndexFileMeta>>>
payloads_by_level;
- std::map<int32_t, std::vector<PrimaryKeyIndexSourceMeta>>
payload_metas_by_level;
+ std::map<std::string, int64_t> active_sources;
+ std::set<std::string> ambiguous_active_source_names;
+ for (const auto& level_sources : sources_by_level) {
+ for (const PrimaryKeyIndexSourceFile& source : level_sources.second) {
+ if (!active_sources.emplace(source.file_name,
source.row_count).second) {
+ ambiguous_active_source_names.insert(source.file_name);
Review Comment:
Not intentional. The current revision now resolves the active set from
`source_meta.DataLevel()` and rejects a payload whose declared level has no
eligible active intersection. `RejectsPayloadForDifferentActiveDataLevel`
covers the metadata-level mismatch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
