gripleaf commented on code in PR #323:
URL: https://github.com/apache/paimon-cpp/pull/323#discussion_r3979079992
##########
src/paimon/core/operation/file_store_scan.cpp:
##########
@@ -422,15 +422,27 @@ Status FileStoreScan::ReadAndMergeBucketFileEntries(
const std::vector<ManifestFileMeta>& manifest_metas, int32_t bucket,
std::vector<ManifestEntry>* merged_entries) const {
const bool inferred_bucket = !bucket_filter_ && bucket_selector_ !=
nullptr;
- // Explicit-bucket lazy decoding cannot retain entries with a different
layout.
- if (!inferred_bucket &&
core_options_.ScanManifestEntryLazyDecodeEnabled()) {
+ if (core_options_.ScanManifestEntryLazyDecodeEnabled()) {
std::vector<std::future<Result<std::vector<ManifestEntry>>>> futures;
futures.reserve(manifest_metas.size());
for (const auto& meta : manifest_metas) {
- auto read_meta_task = [this, meta, bucket]() ->
Result<std::vector<ManifestEntry>> {
+ auto read_meta_task = [this, meta, bucket,
+ inferred_bucket]() ->
Result<std::vector<ManifestEntry>> {
std::vector<ManifestEntry> bucket_entries;
- PAIMON_RETURN_NOT_OK(
- manifest_file_->ReadBucketEntries(meta.FileName(), bucket,
&bucket_entries));
+ if (inferred_bucket) {
+
PAIMON_RETURN_NOT_OK(manifest_file_->ReadInferredBucketEntries(
+ meta.FileName(), bucket, core_options_.GetBucket(),
table_schema_->Id(),
+ &bucket_entries));
+ } else if (meta.MinBucket() && meta.MaxBucket() &&
+ meta.MinBucket().value() == bucket &&
+ meta.MaxBucket().value() == bucket) {
+ // Every entry belongs to this bucket; a projection pass
cannot prune rows.
+ PAIMON_RETURN_NOT_OK(
+ manifest_file_->Read(meta.FileName(),
/*filter=*/nullptr, &bucket_entries));
+ } else {
+
PAIMON_RETURN_NOT_OK(manifest_file_->ReadBucketEntries(meta.FileName(), bucket,
Review Comment:
It's no need to cache when you only have few manifest files. So, a second
full parse plus a second zstd decompression is not expensive.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
