WZhuo commented on code in PR #875: URL: https://github.com/apache/iceberg-cpp/pull/875#discussion_r4594567701
########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} + +Result<std::vector<std::byte>> ReadInputFile(InputFile& input_file, int64_t read_size, + std::string_view file_location) { + if (read_size < 0) { + return Invalid("Invalid negative file size {} for {}", read_size, file_location); + } + if (static_cast<uint64_t>(read_size) > + static_cast<uint64_t>(std::numeric_limits<size_t>::max())) { + return Invalid("File size {} exceeds size_t max for {}", read_size, file_location); + } + + auto size = static_cast<size_t>(read_size); + std::vector<std::byte> content(size); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file.Open()); + Status read_status = {}; + if (size > 0) { + read_status = stream->ReadFully(/*position=*/0, content); + } + ICEBERG_RETURN_UNEXPECTED( + FinishWithCloseStatus(std::move(read_status), stream->Close())); + return content; +} + +class CachedSeekableInputStream : public SeekableInputStream { + public: + explicit CachedSeekableInputStream(MetadataContent content) + : content_(std::move(content)) {} + + Result<int64_t> Position() const override { return position_; } + + Status Seek(int64_t position) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + position_ = position; + return {}; + } + + Result<int64_t> Read(std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + auto position = static_cast<size_t>(position_); + auto bytes_to_read = std::min(out.size(), content_->size() - position); + if (bytes_to_read > 0) { + std::memcpy(out.data(), content_->data() + position, bytes_to_read); + position_ += static_cast<int64_t>(bytes_to_read); + } + return static_cast<int64_t>(bytes_to_read); + } + + Status ReadFully(int64_t position, std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + auto offset = static_cast<size_t>(position); + ICEBERG_PRECHECK(out.size() <= content_->size() - offset, + "Read out of bounds: offset {} + length {} exceeds file size {}", + position, out.size(), content_->size()); + if (!out.empty()) { + std::memcpy(out.data(), content_->data() + offset, out.size()); + } + return {}; + } + + Status Close() override { + closed_ = true; + return {}; + } + + private: + MetadataContent content_; + int64_t position_ = 0; + bool closed_ = false; +}; + +class CachedInputFile : public InputFile { + public: + CachedInputFile(std::unique_ptr<InputFile> input, std::shared_ptr<ContentCache> cache, + int64_t size) + : input_(std::move(input)), + cache_(std::move(cache)), + location_(input_->location()), + size_(size) {} + CachedInputFile(std::string location, MetadataContent content) + : location_(std::move(location)), + size_(static_cast<int64_t>(content->size())), + content_(std::move(content)) {} + std::string_view location() const override { return location_; } + Result<int64_t> Size() const override { return size_; } + Result<std::unique_ptr<SeekableInputStream>> Open() override { + if (content_) return std::make_unique<CachedSeekableInputStream>(content_); + if (auto cached = cache_->Find(location_)) { + return std::make_unique<CachedSeekableInputStream>(std::move(cached)); + } + auto loaded = ReadInputFile(*input_, size_, location_); + if (!loaded) { + auto error = std::move(loaded).error(); + if (error.kind != ErrorKind::kIOError) return unexpected<Error>(std::move(error)); + auto fallback = input_->Open(); + if (!fallback) { + error.message += "; fallback open failed: " + fallback.error().message; + return unexpected<Error>(std::move(error)); + } + return fallback; + } + auto content = + std::make_shared<const std::vector<std::byte>>(std::move(loaded).value()); + cache_->Insert(location_, content); + return std::make_unique<CachedSeekableInputStream>(std::move(content)); + } + + private: + std::unique_ptr<InputFile> input_; + std::shared_ptr<ContentCache> cache_; + std::string location_; + int64_t size_; + MetadataContent content_; +}; + +// Only manifest readers receive this adapter. Output and deletion operations are +// forwarded unchanged, allowing manifest maintenance to use the same read context. +class ManifestFileIO : public FileIO { + public: + ManifestFileIO(std::shared_ptr<FileIO> io, std::shared_ptr<ContentCache> cache) + : io_(std::move(io)), cache_(std::move(cache)) {} + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location) override { + return cache_->NewInputFile(io_, std::move(location)); + } + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location, + size_t length) override { + return cache_->NewInputFile(io_, std::move(location), length); + } + Result<std::unique_ptr<OutputFile>> NewOutputFile(std::string location) override { + return io_->NewOutputFile(std::move(location)); + } + Status DeleteFile(const std::string& location) override { + return io_->DeleteFile(location); + } + Status DeleteFiles(const std::vector<std::string>& locations) override { + return io_->DeleteFiles(locations); + } + const std::unordered_map<std::string, std::string>& properties() const override { + return io_->properties(); + } + + private: + std::shared_ptr<FileIO> io_; + std::shared_ptr<ContentCache> cache_; +}; + +} // namespace + +Result<ContentCacheOptions> ContentCacheOptions::FromProperties( + const std::unordered_map<std::string, std::string>& properties) { + ContentCacheOptions options; + if (auto it = properties.find(std::string(kEnabled)); it != properties.end()) { + options.enabled = StringUtils::ParseBoolean(it->second); + } Review Comment: If `enabled` is false, there is no need to parse the remaining options. Could we return early here, e.g. ```cpp if (!options.enabled) { return options; } ``` ########## src/iceberg/table.cc: ########## @@ -75,10 +78,12 @@ Result<std::shared_ptr<Table>> Table::Make( if (catalog == nullptr) [[unlikely]] { return InvalidArgument("Catalog cannot be null"); } - return std::shared_ptr<Table>(new Table(std::move(identifier), std::move(metadata), - std::move(metadata_location), std::move(io), - std::move(catalog), std::move(full_name), - std::move(reporter), std::move(labels))); + auto table = std::shared_ptr<Table>( + new Table(std::move(identifier), std::move(metadata), std::move(metadata_location), + std::move(io), std::move(catalog), std::move(full_name), + std::move(reporter), std::move(labels))); + ICEBERG_RETURN_UNEXPECTED(table->InitializeManifestCache(io_properties)); Review Comment: This can turn a successful commit into data loss. The flow is `Transaction::Commit()` -> `Catalog::UpdateTable()` -> `Table::Make()`: if the cache properties are invalid, `Table::Make()` fails with `InvalidArgument` *after* the catalog has already committed the new metadata. Since that error is not `kCommitStateUnknown`, `Transaction::Commit()` treats it as a failed commit and calls `CleanupUpdates()`, which deletes the staged manifest/manifest list files that the committed metadata now references. An invalid cache config shouldn't cause `Table::Make()` to fail. Could we validate these properties earlier, or fall back to disabling the cache (with a warning) instead of returning an error here? ########## src/iceberg/table.h: ########## @@ -55,12 +59,15 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> { /// \param[in] reporter Optional metrics reporter for this table. Defaults to nullptr /// (noop). /// \param[in] labels Labels for this table, or nullptr when it has none. + /// \param[in] io_properties Catalog I/O defaults for the table manifest cache. + /// Table properties override these defaults; the budget is per loaded table. static Result<std::shared_ptr<Table>> Make( TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata, std::string metadata_location, std::shared_ptr<FileIO> io, std::shared_ptr<Catalog> catalog, std::string full_name = "", std::shared_ptr<MetricsReporter> reporter = nullptr, - std::unique_ptr<Labels> labels = nullptr); + std::unique_ptr<Labels> labels = nullptr, + const std::unordered_map<std::string, std::string>& io_properties = {}); Review Comment: This adds yet another defaulted trailing parameter to a public factory, so callers end up writing `/*labels=*/nullptr, properties_`. It is also an untyped map whose only consumer is the manifest cache, while the name `io_properties` suggests general I/O config. Could we either drop it and read `io->properties()` (as Java does), or take a typed object such as `std::shared_ptr<ContentCache>` / a `TableOptions` struct? A struct would also stop the default-argument list from growing with every new table-level option. Same for `StagedTable::Make`. ########## src/iceberg/table.cc: ########## @@ -92,16 +97,55 @@ Table::Table(TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata metadata_(std::move(metadata)), metadata_location_(std::move(metadata_location)), io_(std::move(io)), + manifest_io_(io_), catalog_(std::move(catalog)), reporter_(std::move(reporter)), metadata_cache_(std::make_unique<TableMetadataCache>(metadata_.get())), labels_(labels && !labels->empty() ? std::move(labels) : nullptr) {} +Status Table::InitializeManifestCache( + const std::unordered_map<std::string, std::string>& io_properties) { + auto properties = io_->properties(); + for (const auto& [key, value] : io_properties) properties[key] = value; + for (const auto& [key, value] : metadata_->properties.configs()) { + properties[key] = value; + } + ICEBERG_ASSIGN_OR_RAISE(auto options, + internal::ContentCacheOptions::FromProperties(properties)); + if (options.enabled) { + content_cache_ = std::make_shared<internal::ContentCache>(options); Review Comment: The cache is attached to a single `Table` handle, so every `LoadTable` call starts with an empty cache. Many engines call `LoadTable` once per query, so with that usage pattern the "reuse across repeated scans" this PR is aiming for will rarely kick in. There is also no process-wide memory limit: the total footprint is effectively "number of tables × 100 MiB". For comparison, Java keys the content cache by `FileIO` (which is shared by tables loaded from the same catalog), keeps at most 8 FileIO caches (`iceberg.io.manifest.cache.fileio-max`), and holds them as soft values so they can be reclaimed under memory pressure. ########## src/iceberg/arrow/arrow_io.cc: ########## @@ -428,24 +429,41 @@ class ArrowInputFile : public InputFile { } ::arrow::fs::FileInfo file_info(path_, ::arrow::fs::FileType::File); ICEBERG_ARROW_ASSIGN_OR_RETURN(auto input, fs_->OpenInputFile(file_info)); - ICEBERG_ARROW_ASSIGN_OR_RETURN(auto size, input->GetSize()); + auto size_result = input->GetSize(); + auto close_status = input->Close(); + if (!size_result.ok() && !close_status.ok()) { + auto status = size_result.status().WithMessage( + size_result.status().message(), + "; additionally failed to close file: ", close_status.ToString()); + ICEBERG_ARROW_RETURN_NOT_OK(status); + } + ICEBERG_ARROW_ASSIGN_OR_RETURN(auto size, std::move(size_result)); + ICEBERG_ARROW_RETURN_NOT_OK(close_status); + file_size_ = size; Review Comment: If possible, could we use `fs_->GetFileInfo(path_)` here instead? It returns the file size directly, without having to open and close the file, which also removes the need for the combined size/close error handling. For example: ```cpp ICEBERG_ARROW_ASSIGN_OR_RETURN(auto info, fs_->GetFileInfo(path_)); // check info.type() == ::arrow::fs::FileType::File and info.size() != ::arrow::fs::kNoSize file_size_ = info.size(); return info.size(); ``` ########## src/iceberg/table.cc: ########## @@ -92,16 +97,55 @@ Table::Table(TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata metadata_(std::move(metadata)), metadata_location_(std::move(metadata_location)), io_(std::move(io)), + manifest_io_(io_), catalog_(std::move(catalog)), reporter_(std::move(reporter)), metadata_cache_(std::make_unique<TableMetadataCache>(metadata_.get())), labels_(labels && !labels->empty() ? std::move(labels) : nullptr) {} +Status Table::InitializeManifestCache( + const std::unordered_map<std::string, std::string>& io_properties) { + auto properties = io_->properties(); + for (const auto& [key, value] : io_properties) properties[key] = value; + for (const auto& [key, value] : metadata_->properties.configs()) { + properties[key] = value; + } Review Comment: This merges three layers of config: `io_->properties()`, `io_properties`, and table properties. Java only reads the cache settings from `io.properties()`. Could we keep a single source here? ########## src/iceberg/catalog/rest/rest_table.cc: ########## @@ -48,13 +48,20 @@ Result<std::shared_ptr<RestTable>> RestTable::Make( std::shared_ptr<Catalog> catalog, std::string full_name, std::shared_ptr<MetricsReporter> reporter, RestScanContext rest_context, std::unique_ptr<Labels> labels) { + ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); if (metadata == nullptr) { return InvalidArgument("Metadata cannot be null"); } - return std::shared_ptr<RestTable>(new RestTable( + auto table = std::shared_ptr<RestTable>(new RestTable( std::move(identifier), std::move(metadata), std::move(metadata_location), std::move(io), std::move(catalog), std::move(full_name), std::move(reporter), std::move(rest_context), std::move(labels))); + auto properties = table->rest_context_->catalog_config; + for (const auto& [key, value] : table->rest_context_->table_config) { + properties[key] = value; + } Review Comment: This recomputes the catalog/table config merge again. The table `FileIO` is already built from `MergeFileIOProperties(catalog_config, table_config)`, so `io_->properties()` already contains these keys and this extra layer has no effect. ########## src/iceberg/catalog/memory/in_memory_catalog.cc: ########## @@ -445,7 +448,8 @@ Result<std::shared_ptr<Table>> InMemoryCatalog::CreateTable( root_namespace_->UpdateTableMetadataLocation(identifier, metadata_file_location)); return Table::Make(identifier, std::move(table_metadata), std::move(metadata_file_location), file_io_, shared_from_this(), - CatalogUtil::FullTableName(name(), identifier), reporter_); + CatalogUtil::FullTableName(name(), identifier), reporter_, + /*labels=*/nullptr, properties_); Review Comment: Every catalog has to remember to pass `properties_` on every `Table::Make` / `StagedTable::Make` call (here and some in `SqlCatalog`). Forgetting one is not an error; the catalog-level config is just silently ignored. The next catalog will likely hit this, e.g. `HiveCatalog::LoadTable` is still `NotImplemented`. ########## src/iceberg/catalog/rest/rest_catalog.cc: ########## @@ -923,12 +923,14 @@ Result<std::shared_ptr<Transaction>> RestCatalog::StageCreateTable( auto table_catalog = std::make_shared<TableScopedCatalog>( shared_from_this(), context, identifier, table_config, std::move(table_session), table_io); + auto manifest_properties = config_.configs(); + for (const auto& [key, value] : table_config) manifest_properties[key] = value; Review Comment: This is the third copy of the same catalog/table config merge (also in `MakeTable` and `RestTable::Make`). As with `RestTable::Make`, the table `FileIO` is already built from the same merged map, so this seems redundant. ########## src/iceberg/update/expire_snapshots.cc: ########## @@ -1038,12 +1038,12 @@ Status ExpireSnapshots::Finalize(const TableMetadata& metadata_after_expiration) !HasNonMainSnapshots(metadata_after_expiration); if (can_use_incremental) { - return IncrementalFileCleanup(ctx_->table->io(), delete_func_, delete_executor_, - plan_executor_) + return IncrementalFileCleanup(ctx_->table->manifest_io(), delete_func_, + delete_executor_, plan_executor_) Review Comment: Cleanup now reads the expired manifest lists and manifests through `manifest_io()`, so they are loaded into the cache (evicting hot entries) right before being deleted. Since they will not be read again, would it be better to keep using `io()` here? ########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} + +Result<std::vector<std::byte>> ReadInputFile(InputFile& input_file, int64_t read_size, + std::string_view file_location) { + if (read_size < 0) { + return Invalid("Invalid negative file size {} for {}", read_size, file_location); + } + if (static_cast<uint64_t>(read_size) > + static_cast<uint64_t>(std::numeric_limits<size_t>::max())) { + return Invalid("File size {} exceeds size_t max for {}", read_size, file_location); + } + + auto size = static_cast<size_t>(read_size); + std::vector<std::byte> content(size); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file.Open()); + Status read_status = {}; + if (size > 0) { + read_status = stream->ReadFully(/*position=*/0, content); + } + ICEBERG_RETURN_UNEXPECTED( + FinishWithCloseStatus(std::move(read_status), stream->Close())); + return content; +} + +class CachedSeekableInputStream : public SeekableInputStream { + public: + explicit CachedSeekableInputStream(MetadataContent content) + : content_(std::move(content)) {} + + Result<int64_t> Position() const override { return position_; } + + Status Seek(int64_t position) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + position_ = position; + return {}; + } + + Result<int64_t> Read(std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + auto position = static_cast<size_t>(position_); + auto bytes_to_read = std::min(out.size(), content_->size() - position); + if (bytes_to_read > 0) { + std::memcpy(out.data(), content_->data() + position, bytes_to_read); + position_ += static_cast<int64_t>(bytes_to_read); + } + return static_cast<int64_t>(bytes_to_read); + } + + Status ReadFully(int64_t position, std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + auto offset = static_cast<size_t>(position); + ICEBERG_PRECHECK(out.size() <= content_->size() - offset, + "Read out of bounds: offset {} + length {} exceeds file size {}", + position, out.size(), content_->size()); + if (!out.empty()) { + std::memcpy(out.data(), content_->data() + offset, out.size()); + } + return {}; + } + + Status Close() override { + closed_ = true; + return {}; + } + + private: + MetadataContent content_; + int64_t position_ = 0; + bool closed_ = false; +}; + +class CachedInputFile : public InputFile { + public: + CachedInputFile(std::unique_ptr<InputFile> input, std::shared_ptr<ContentCache> cache, + int64_t size) + : input_(std::move(input)), + cache_(std::move(cache)), + location_(input_->location()), + size_(size) {} + CachedInputFile(std::string location, MetadataContent content) + : location_(std::move(location)), + size_(static_cast<int64_t>(content->size())), + content_(std::move(content)) {} + std::string_view location() const override { return location_; } + Result<int64_t> Size() const override { return size_; } + Result<std::unique_ptr<SeekableInputStream>> Open() override { + if (content_) return std::make_unique<CachedSeekableInputStream>(content_); + if (auto cached = cache_->Find(location_)) { + return std::make_unique<CachedSeekableInputStream>(std::move(cached)); + } + auto loaded = ReadInputFile(*input_, size_, location_); + if (!loaded) { + auto error = std::move(loaded).error(); + if (error.kind != ErrorKind::kIOError) return unexpected<Error>(std::move(error)); + auto fallback = input_->Open(); + if (!fallback) { + error.message += "; fallback open failed: " + fallback.error().message; + return unexpected<Error>(std::move(error)); + } + return fallback; + } + auto content = + std::make_shared<const std::vector<std::byte>>(std::move(loaded).value()); + cache_->Insert(location_, content); + return std::make_unique<CachedSeekableInputStream>(std::move(content)); + } + + private: + std::unique_ptr<InputFile> input_; + std::shared_ptr<ContentCache> cache_; + std::string location_; + int64_t size_; + MetadataContent content_; +}; + +// Only manifest readers receive this adapter. Output and deletion operations are +// forwarded unchanged, allowing manifest maintenance to use the same read context. +class ManifestFileIO : public FileIO { + public: + ManifestFileIO(std::shared_ptr<FileIO> io, std::shared_ptr<ContentCache> cache) + : io_(std::move(io)), cache_(std::move(cache)) {} + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location) override { + return cache_->NewInputFile(io_, std::move(location)); + } + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location, + size_t length) override { + return cache_->NewInputFile(io_, std::move(location), length); + } + Result<std::unique_ptr<OutputFile>> NewOutputFile(std::string location) override { + return io_->NewOutputFile(std::move(location)); + } + Status DeleteFile(const std::string& location) override { + return io_->DeleteFile(location); + } + Status DeleteFiles(const std::vector<std::string>& locations) override { + return io_->DeleteFiles(locations); + } + const std::unordered_map<std::string, std::string>& properties() const override { Review Comment: Deletes are forwarded without invalidating the cache, so bytes of deleted files stay cached until evicted. Should these also drop the corresponding entries? ########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} + +Result<std::vector<std::byte>> ReadInputFile(InputFile& input_file, int64_t read_size, + std::string_view file_location) { + if (read_size < 0) { + return Invalid("Invalid negative file size {} for {}", read_size, file_location); + } + if (static_cast<uint64_t>(read_size) > + static_cast<uint64_t>(std::numeric_limits<size_t>::max())) { + return Invalid("File size {} exceeds size_t max for {}", read_size, file_location); + } + + auto size = static_cast<size_t>(read_size); + std::vector<std::byte> content(size); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file.Open()); + Status read_status = {}; + if (size > 0) { + read_status = stream->ReadFully(/*position=*/0, content); + } + ICEBERG_RETURN_UNEXPECTED( + FinishWithCloseStatus(std::move(read_status), stream->Close())); + return content; +} + +class CachedSeekableInputStream : public SeekableInputStream { + public: + explicit CachedSeekableInputStream(MetadataContent content) + : content_(std::move(content)) {} + + Result<int64_t> Position() const override { return position_; } + + Status Seek(int64_t position) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + position_ = position; + return {}; + } + + Result<int64_t> Read(std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + auto position = static_cast<size_t>(position_); + auto bytes_to_read = std::min(out.size(), content_->size() - position); + if (bytes_to_read > 0) { + std::memcpy(out.data(), content_->data() + position, bytes_to_read); + position_ += static_cast<int64_t>(bytes_to_read); + } + return static_cast<int64_t>(bytes_to_read); + } + + Status ReadFully(int64_t position, std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + auto offset = static_cast<size_t>(position); + ICEBERG_PRECHECK(out.size() <= content_->size() - offset, + "Read out of bounds: offset {} + length {} exceeds file size {}", + position, out.size(), content_->size()); + if (!out.empty()) { + std::memcpy(out.data(), content_->data() + offset, out.size()); + } + return {}; + } + + Status Close() override { + closed_ = true; + return {}; + } + + private: + MetadataContent content_; + int64_t position_ = 0; + bool closed_ = false; +}; + +class CachedInputFile : public InputFile { + public: + CachedInputFile(std::unique_ptr<InputFile> input, std::shared_ptr<ContentCache> cache, + int64_t size) + : input_(std::move(input)), + cache_(std::move(cache)), + location_(input_->location()), + size_(size) {} + CachedInputFile(std::string location, MetadataContent content) + : location_(std::move(location)), + size_(static_cast<int64_t>(content->size())), + content_(std::move(content)) {} + std::string_view location() const override { return location_; } + Result<int64_t> Size() const override { return size_; } + Result<std::unique_ptr<SeekableInputStream>> Open() override { + if (content_) return std::make_unique<CachedSeekableInputStream>(content_); + if (auto cached = cache_->Find(location_)) { + return std::make_unique<CachedSeekableInputStream>(std::move(cached)); + } + auto loaded = ReadInputFile(*input_, size_, location_); + if (!loaded) { + auto error = std::move(loaded).error(); + if (error.kind != ErrorKind::kIOError) return unexpected<Error>(std::move(error)); + auto fallback = input_->Open(); + if (!fallback) { + error.message += "; fallback open failed: " + fallback.error().message; + return unexpected<Error>(std::move(error)); + } + return fallback; + } + auto content = + std::make_shared<const std::vector<std::byte>>(std::move(loaded).value()); + cache_->Insert(location_, content); + return std::make_unique<CachedSeekableInputStream>(std::move(content)); + } + + private: + std::unique_ptr<InputFile> input_; + std::shared_ptr<ContentCache> cache_; + std::string location_; + int64_t size_; + MetadataContent content_; +}; + +// Only manifest readers receive this adapter. Output and deletion operations are +// forwarded unchanged, allowing manifest maintenance to use the same read context. +class ManifestFileIO : public FileIO { + public: + ManifestFileIO(std::shared_ptr<FileIO> io, std::shared_ptr<ContentCache> cache) + : io_(std::move(io)), cache_(std::move(cache)) {} + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location) override { + return cache_->NewInputFile(io_, std::move(location)); + } + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location, + size_t length) override { + return cache_->NewInputFile(io_, std::move(location), length); + } + Result<std::unique_ptr<OutputFile>> NewOutputFile(std::string location) override { + return io_->NewOutputFile(std::move(location)); + } + Status DeleteFile(const std::string& location) override { + return io_->DeleteFile(location); + } + Status DeleteFiles(const std::vector<std::string>& locations) override { + return io_->DeleteFiles(locations); + } + const std::unordered_map<std::string, std::string>& properties() const override { + return io_->properties(); + } + + private: + std::shared_ptr<FileIO> io_; + std::shared_ptr<ContentCache> cache_; +}; + +} // namespace + +Result<ContentCacheOptions> ContentCacheOptions::FromProperties( + const std::unordered_map<std::string, std::string>& properties) { + ContentCacheOptions options; + if (auto it = properties.find(std::string(kEnabled)); it != properties.end()) { + options.enabled = StringUtils::ParseBoolean(it->second); + } + ICEBERG_ASSIGN_OR_RAISE( + options.max_total_bytes, + ParseNumberProperty(properties, kMaxTotalBytes, options.max_total_bytes)); + ICEBERG_ASSIGN_OR_RAISE( + options.max_content_length, + ParseNumberProperty(properties, kMaxContentLength, options.max_content_length)); + if (options.enabled) { + ICEBERG_PRECHECK( + options.max_total_bytes > 0 && + std::cmp_less_equal(options.max_total_bytes, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum total bytes must be positive and fit in int64_t"); + ICEBERG_PRECHECK( + options.max_content_length > 0 && + std::cmp_less_equal(options.max_content_length, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum content length must be positive and fit in int64_t"); + } + return options; +} + +ContentCache::ContentCache(ContentCacheOptions options) + : options_(options), + lru_(static_cast<int64_t>(options.max_total_bytes)), + aliases_( + static_cast<int64_t>(std::min<size_t>(options.max_total_bytes, 64 * 1024))) {} +std::shared_ptr<FileIO> ContentCache::Wrap(std::shared_ptr<FileIO> io) { + return std::make_shared<ManifestFileIO>(std::move(io), shared_from_this()); +} +MetadataContent ContentCache::Find(std::string_view location, + std::string* canonical_location) { + std::lock_guard lock(mutex_); + if (auto* content = lru_.Find(location)) { + if (canonical_location) *canonical_location = location; + return *content; + } + if (auto* canonical = aliases_.Find(location)) { + if (auto* content = lru_.Find(*canonical)) { + if (canonical_location) *canonical_location = *canonical; + return *content; + } + } + return nullptr; +} +void ContentCache::Insert(const std::string& location, MetadataContent content) { + // Empty/corrupt metadata must not accumulate unbounded zero-weight entries. + if (content->empty()) return; + std::lock_guard lock(mutex_); + lru_.Replace(location, std::move(content)); +} +void ContentCache::Clear() { + std::lock_guard lock(mutex_); + lru_.Clear(); + aliases_.Clear(); +} +Result<std::unique_ptr<InputFile>> ContentCache::NewInputFile( + const std::shared_ptr<FileIO>& io, std::string location, + std::optional<size_t> length) { + if (length && std::cmp_greater(*length, std::numeric_limits<int64_t>::max())) { + return InvalidArgument("File length {} exceeds int64_t max", *length); + } + // A hit needs neither storage resolution nor working remote credentials. + std::string canonical_location; + if (auto content = Find(location, &canonical_location)) { + return std::make_unique<CachedInputFile>(std::move(canonical_location), + std::move(content)); + } + std::unique_ptr<InputFile> input; + if (length) { + ICEBERG_ASSIGN_OR_RAISE(input, io->NewInputFile(location, *length)); + } else { + ICEBERG_ASSIGN_OR_RAISE(input, io->NewInputFile(location)); + } + if (input->location() != location) { + std::lock_guard lock(mutex_); + aliases_.Replace(std::move(location), std::string(input->location())); + } + if (auto content = Find(input->location())) { + return std::make_unique<CachedInputFile>(std::string(input->location()), + std::move(content)); + } + int64_t size; + if (length) { + size = static_cast<int64_t>(*length); + } else { Review Comment: For `FileIO::ReadFile(loc, n)`, `n` is the number of bytes to read, not the file length (`file_io.h`). Via `manifest_io()`, a partial read reaches here with `size = n`, so the truncated prefix is cached as the whole file and a later `NewInputFile(loc)` returns truncated content. Could `ManifestFileIO` refuse or skip caching for partial `ReadFile` calls? ########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} + +Result<std::vector<std::byte>> ReadInputFile(InputFile& input_file, int64_t read_size, + std::string_view file_location) { + if (read_size < 0) { + return Invalid("Invalid negative file size {} for {}", read_size, file_location); + } + if (static_cast<uint64_t>(read_size) > + static_cast<uint64_t>(std::numeric_limits<size_t>::max())) { + return Invalid("File size {} exceeds size_t max for {}", read_size, file_location); + } + + auto size = static_cast<size_t>(read_size); + std::vector<std::byte> content(size); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file.Open()); + Status read_status = {}; + if (size > 0) { + read_status = stream->ReadFully(/*position=*/0, content); + } + ICEBERG_RETURN_UNEXPECTED( + FinishWithCloseStatus(std::move(read_status), stream->Close())); + return content; +} + +class CachedSeekableInputStream : public SeekableInputStream { + public: + explicit CachedSeekableInputStream(MetadataContent content) + : content_(std::move(content)) {} + + Result<int64_t> Position() const override { return position_; } + + Status Seek(int64_t position) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + position_ = position; + return {}; + } + + Result<int64_t> Read(std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + auto position = static_cast<size_t>(position_); + auto bytes_to_read = std::min(out.size(), content_->size() - position); + if (bytes_to_read > 0) { + std::memcpy(out.data(), content_->data() + position, bytes_to_read); + position_ += static_cast<int64_t>(bytes_to_read); + } + return static_cast<int64_t>(bytes_to_read); + } + + Status ReadFully(int64_t position, std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + auto offset = static_cast<size_t>(position); + ICEBERG_PRECHECK(out.size() <= content_->size() - offset, + "Read out of bounds: offset {} + length {} exceeds file size {}", + position, out.size(), content_->size()); + if (!out.empty()) { + std::memcpy(out.data(), content_->data() + offset, out.size()); + } + return {}; + } + + Status Close() override { + closed_ = true; + return {}; + } + + private: + MetadataContent content_; + int64_t position_ = 0; + bool closed_ = false; +}; + +class CachedInputFile : public InputFile { + public: + CachedInputFile(std::unique_ptr<InputFile> input, std::shared_ptr<ContentCache> cache, + int64_t size) + : input_(std::move(input)), + cache_(std::move(cache)), + location_(input_->location()), + size_(size) {} + CachedInputFile(std::string location, MetadataContent content) + : location_(std::move(location)), + size_(static_cast<int64_t>(content->size())), + content_(std::move(content)) {} + std::string_view location() const override { return location_; } + Result<int64_t> Size() const override { return size_; } + Result<std::unique_ptr<SeekableInputStream>> Open() override { + if (content_) return std::make_unique<CachedSeekableInputStream>(content_); + if (auto cached = cache_->Find(location_)) { + return std::make_unique<CachedSeekableInputStream>(std::move(cached)); + } + auto loaded = ReadInputFile(*input_, size_, location_); + if (!loaded) { + auto error = std::move(loaded).error(); + if (error.kind != ErrorKind::kIOError) return unexpected<Error>(std::move(error)); + auto fallback = input_->Open(); + if (!fallback) { + error.message += "; fallback open failed: " + fallback.error().message; + return unexpected<Error>(std::move(error)); + } + return fallback; + } + auto content = + std::make_shared<const std::vector<std::byte>>(std::move(loaded).value()); + cache_->Insert(location_, content); + return std::make_unique<CachedSeekableInputStream>(std::move(content)); + } + + private: + std::unique_ptr<InputFile> input_; + std::shared_ptr<ContentCache> cache_; + std::string location_; + int64_t size_; + MetadataContent content_; +}; + +// Only manifest readers receive this adapter. Output and deletion operations are +// forwarded unchanged, allowing manifest maintenance to use the same read context. +class ManifestFileIO : public FileIO { + public: + ManifestFileIO(std::shared_ptr<FileIO> io, std::shared_ptr<ContentCache> cache) + : io_(std::move(io)), cache_(std::move(cache)) {} + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location) override { + return cache_->NewInputFile(io_, std::move(location)); + } + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location, + size_t length) override { + return cache_->NewInputFile(io_, std::move(location), length); + } + Result<std::unique_ptr<OutputFile>> NewOutputFile(std::string location) override { + return io_->NewOutputFile(std::move(location)); + } + Status DeleteFile(const std::string& location) override { + return io_->DeleteFile(location); + } + Status DeleteFiles(const std::vector<std::string>& locations) override { + return io_->DeleteFiles(locations); + } + const std::unordered_map<std::string, std::string>& properties() const override { + return io_->properties(); + } + + private: + std::shared_ptr<FileIO> io_; + std::shared_ptr<ContentCache> cache_; +}; + +} // namespace + +Result<ContentCacheOptions> ContentCacheOptions::FromProperties( + const std::unordered_map<std::string, std::string>& properties) { + ContentCacheOptions options; + if (auto it = properties.find(std::string(kEnabled)); it != properties.end()) { + options.enabled = StringUtils::ParseBoolean(it->second); + } + ICEBERG_ASSIGN_OR_RAISE( + options.max_total_bytes, + ParseNumberProperty(properties, kMaxTotalBytes, options.max_total_bytes)); + ICEBERG_ASSIGN_OR_RAISE( + options.max_content_length, + ParseNumberProperty(properties, kMaxContentLength, options.max_content_length)); + if (options.enabled) { + ICEBERG_PRECHECK( + options.max_total_bytes > 0 && + std::cmp_less_equal(options.max_total_bytes, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum total bytes must be positive and fit in int64_t"); + ICEBERG_PRECHECK( + options.max_content_length > 0 && + std::cmp_less_equal(options.max_content_length, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum content length must be positive and fit in int64_t"); + } + return options; +} + +ContentCache::ContentCache(ContentCacheOptions options) + : options_(options), + lru_(static_cast<int64_t>(options.max_total_bytes)), + aliases_( + static_cast<int64_t>(std::min<size_t>(options.max_total_bytes, 64 * 1024))) {} +std::shared_ptr<FileIO> ContentCache::Wrap(std::shared_ptr<FileIO> io) { + return std::make_shared<ManifestFileIO>(std::move(io), shared_from_this()); +} Review Comment: nit: there are no blank lines between the `ContentCache` method definitions here, it's somehow hard to read. ########## src/iceberg/data_table_scan.cc: ########## @@ -164,8 +164,12 @@ Result<FileScanTaskStreamPtr> DataTableScan::PlanFilesStream() const { ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); SnapshotReader snapshot_reader(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE(auto data_manifests, snapshot_reader.DataManifests(io_)); - ICEBERG_ASSIGN_OR_RAISE(auto delete_manifests, snapshot_reader.DeleteManifests(io_)); + ICEBERG_ASSIGN_OR_RAISE( + auto data_manifests, + snapshot_reader.DataManifests(context_.manifest_io ? context_.manifest_io : io_)); Review Comment: nit: `context_.manifest_io ? context_.manifest_io : io_` is repeated many times across `data_table_scan.cc`, `incremental_append_scan.cc` and `incremental_changelog_scan.cc`. Could we add a `manifest_io()` helper to `TableScan` instead? ########## src/iceberg/arrow/arrow_io.cc: ########## @@ -530,6 +548,9 @@ Result<std::shared_ptr<::arrow::io::RandomAccessFile>> OpenArrowInputStream( } else { ICEBERG_ASSIGN_OR_RAISE(input_file, io->NewInputFile(path)); } + if (auto* arrow_input = dynamic_cast<ArrowInputFile*>(input_file.get())) { + return arrow_input->OpenArrowFile(); + } Review Comment: With the cache enabled, `manifest_io()` is no longer an `ArrowFileSystemFileIO`, so the fast path above misses; I assume this `dynamic_cast` works around that. But it also changes uncached reads through `ResolvingFileIO` (which hands out `ArrowInputFile`), which seems unrelated to this PR. Could that be split out, or the identity issue handled differently? ########## src/iceberg/table.cc: ########## @@ -92,16 +97,55 @@ Table::Table(TableIdentifier identifier, std::shared_ptr<TableMetadata> metadata metadata_(std::move(metadata)), metadata_location_(std::move(metadata_location)), io_(std::move(io)), + manifest_io_(io_), catalog_(std::move(catalog)), reporter_(std::move(reporter)), metadata_cache_(std::make_unique<TableMetadataCache>(metadata_.get())), labels_(labels && !labels->empty() ? std::move(labels) : nullptr) {} +Status Table::InitializeManifestCache( + const std::unordered_map<std::string, std::string>& io_properties) { + auto properties = io_->properties(); + for (const auto& [key, value] : io_properties) properties[key] = value; + for (const auto& [key, value] : metadata_->properties.configs()) { + properties[key] = value; + } + ICEBERG_ASSIGN_OR_RAISE(auto options, + internal::ContentCacheOptions::FromProperties(properties)); + if (options.enabled) { + content_cache_ = std::make_shared<internal::ContentCache>(options); + manifest_io_ = content_cache_->Wrap(io_); + } + return {}; +} + +void Table::ClearManifestCache() { + if (content_cache_) content_cache_->Clear(); +} + +void Table::ReuseManifestCache(const Table& source) { + if (!content_cache_ || !source.content_cache_ || uuid() != source.uuid() || + content_cache_->options() != source.content_cache_->options()) { + return; + } + // A replaced storage context with no comparable properties may represent a different + // endpoint or filesystem. Only preserve bytes when its identity is known to match. + if (io_ != source.io_ && + (io_->properties().empty() || io_->properties() != source.io_->properties())) { + return; + } Review Comment: Many REST servers vend short-lived credentials through the `LoadTable` response `config` (e.g. `s3.access-key-id`, `s3.session-token`, expiration timestamps). Since a non-empty config creates a new table `FileIO` on every load, `io_->properties()` differs on every `Refresh()`, so this check fails and the cache starts from scratch each time. ########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} + +Result<std::vector<std::byte>> ReadInputFile(InputFile& input_file, int64_t read_size, + std::string_view file_location) { + if (read_size < 0) { + return Invalid("Invalid negative file size {} for {}", read_size, file_location); + } + if (static_cast<uint64_t>(read_size) > + static_cast<uint64_t>(std::numeric_limits<size_t>::max())) { + return Invalid("File size {} exceeds size_t max for {}", read_size, file_location); + } + + auto size = static_cast<size_t>(read_size); + std::vector<std::byte> content(size); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file.Open()); + Status read_status = {}; + if (size > 0) { + read_status = stream->ReadFully(/*position=*/0, content); + } + ICEBERG_RETURN_UNEXPECTED( + FinishWithCloseStatus(std::move(read_status), stream->Close())); + return content; +} + +class CachedSeekableInputStream : public SeekableInputStream { + public: + explicit CachedSeekableInputStream(MetadataContent content) + : content_(std::move(content)) {} + + Result<int64_t> Position() const override { return position_; } + + Status Seek(int64_t position) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + position_ = position; + return {}; + } + + Result<int64_t> Read(std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + auto position = static_cast<size_t>(position_); + auto bytes_to_read = std::min(out.size(), content_->size() - position); + if (bytes_to_read > 0) { + std::memcpy(out.data(), content_->data() + position, bytes_to_read); + position_ += static_cast<int64_t>(bytes_to_read); + } + return static_cast<int64_t>(bytes_to_read); + } + + Status ReadFully(int64_t position, std::span<std::byte> out) override { + ICEBERG_PRECHECK(!closed_, "Input stream is closed"); + ICEBERG_PRECHECK(position >= 0, "Position must not be negative: {}", position); + ICEBERG_PRECHECK(static_cast<uint64_t>(position) <= content_->size(), + "Position {} exceeds file size {}", position, content_->size()); + auto offset = static_cast<size_t>(position); + ICEBERG_PRECHECK(out.size() <= content_->size() - offset, + "Read out of bounds: offset {} + length {} exceeds file size {}", + position, out.size(), content_->size()); + if (!out.empty()) { + std::memcpy(out.data(), content_->data() + offset, out.size()); + } + return {}; + } + + Status Close() override { + closed_ = true; + return {}; + } + + private: + MetadataContent content_; + int64_t position_ = 0; + bool closed_ = false; +}; + +class CachedInputFile : public InputFile { + public: + CachedInputFile(std::unique_ptr<InputFile> input, std::shared_ptr<ContentCache> cache, + int64_t size) + : input_(std::move(input)), + cache_(std::move(cache)), + location_(input_->location()), + size_(size) {} + CachedInputFile(std::string location, MetadataContent content) + : location_(std::move(location)), + size_(static_cast<int64_t>(content->size())), + content_(std::move(content)) {} + std::string_view location() const override { return location_; } + Result<int64_t> Size() const override { return size_; } + Result<std::unique_ptr<SeekableInputStream>> Open() override { + if (content_) return std::make_unique<CachedSeekableInputStream>(content_); + if (auto cached = cache_->Find(location_)) { + return std::make_unique<CachedSeekableInputStream>(std::move(cached)); + } + auto loaded = ReadInputFile(*input_, size_, location_); + if (!loaded) { + auto error = std::move(loaded).error(); + if (error.kind != ErrorKind::kIOError) return unexpected<Error>(std::move(error)); + auto fallback = input_->Open(); + if (!fallback) { + error.message += "; fallback open failed: " + fallback.error().message; + return unexpected<Error>(std::move(error)); + } + return fallback; + } + auto content = + std::make_shared<const std::vector<std::byte>>(std::move(loaded).value()); + cache_->Insert(location_, content); + return std::make_unique<CachedSeekableInputStream>(std::move(content)); + } + + private: + std::unique_ptr<InputFile> input_; + std::shared_ptr<ContentCache> cache_; + std::string location_; + int64_t size_; + MetadataContent content_; +}; + +// Only manifest readers receive this adapter. Output and deletion operations are +// forwarded unchanged, allowing manifest maintenance to use the same read context. +class ManifestFileIO : public FileIO { + public: + ManifestFileIO(std::shared_ptr<FileIO> io, std::shared_ptr<ContentCache> cache) + : io_(std::move(io)), cache_(std::move(cache)) {} + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location) override { + return cache_->NewInputFile(io_, std::move(location)); + } + Result<std::unique_ptr<InputFile>> NewInputFile(std::string location, + size_t length) override { + return cache_->NewInputFile(io_, std::move(location), length); + } + Result<std::unique_ptr<OutputFile>> NewOutputFile(std::string location) override { + return io_->NewOutputFile(std::move(location)); + } + Status DeleteFile(const std::string& location) override { + return io_->DeleteFile(location); + } + Status DeleteFiles(const std::vector<std::string>& locations) override { + return io_->DeleteFiles(locations); + } + const std::unordered_map<std::string, std::string>& properties() const override { + return io_->properties(); + } + + private: + std::shared_ptr<FileIO> io_; + std::shared_ptr<ContentCache> cache_; +}; + +} // namespace + +Result<ContentCacheOptions> ContentCacheOptions::FromProperties( + const std::unordered_map<std::string, std::string>& properties) { + ContentCacheOptions options; + if (auto it = properties.find(std::string(kEnabled)); it != properties.end()) { + options.enabled = StringUtils::ParseBoolean(it->second); + } + ICEBERG_ASSIGN_OR_RAISE( + options.max_total_bytes, + ParseNumberProperty(properties, kMaxTotalBytes, options.max_total_bytes)); + ICEBERG_ASSIGN_OR_RAISE( + options.max_content_length, + ParseNumberProperty(properties, kMaxContentLength, options.max_content_length)); + if (options.enabled) { + ICEBERG_PRECHECK( + options.max_total_bytes > 0 && + std::cmp_less_equal(options.max_total_bytes, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum total bytes must be positive and fit in int64_t"); + ICEBERG_PRECHECK( + options.max_content_length > 0 && + std::cmp_less_equal(options.max_content_length, + std::numeric_limits<int64_t>::max()), + "Manifest cache maximum content length must be positive and fit in int64_t"); + } + return options; +} + +ContentCache::ContentCache(ContentCacheOptions options) + : options_(options), + lru_(static_cast<int64_t>(options.max_total_bytes)), + aliases_( + static_cast<int64_t>(std::min<size_t>(options.max_total_bytes, 64 * 1024))) {} +std::shared_ptr<FileIO> ContentCache::Wrap(std::shared_ptr<FileIO> io) { + return std::make_shared<ManifestFileIO>(std::move(io), shared_from_this()); +} +MetadataContent ContentCache::Find(std::string_view location, + std::string* canonical_location) { + std::lock_guard lock(mutex_); + if (auto* content = lru_.Find(location)) { + if (canonical_location) *canonical_location = location; + return *content; + } + if (auto* canonical = aliases_.Find(location)) { + if (auto* content = lru_.Find(*canonical)) { + if (canonical_location) *canonical_location = *canonical; + return *content; + } + } + return nullptr; +} +void ContentCache::Insert(const std::string& location, MetadataContent content) { + // Empty/corrupt metadata must not accumulate unbounded zero-weight entries. + if (content->empty()) return; + std::lock_guard lock(mutex_); + lru_.Replace(location, std::move(content)); +} +void ContentCache::Clear() { + std::lock_guard lock(mutex_); + lru_.Clear(); + aliases_.Clear(); +} +Result<std::unique_ptr<InputFile>> ContentCache::NewInputFile( + const std::shared_ptr<FileIO>& io, std::string location, + std::optional<size_t> length) { + if (length && std::cmp_greater(*length, std::numeric_limits<int64_t>::max())) { + return InvalidArgument("File length {} exceeds int64_t max", *length); + } + // A hit needs neither storage resolution nor working remote credentials. + std::string canonical_location; + if (auto content = Find(location, &canonical_location)) { + return std::make_unique<CachedInputFile>(std::move(canonical_location), + std::move(content)); + } + std::unique_ptr<InputFile> input; + if (length) { + ICEBERG_ASSIGN_OR_RAISE(input, io->NewInputFile(location, *length)); + } else { + ICEBERG_ASSIGN_OR_RAISE(input, io->NewInputFile(location)); + } + if (input->location() != location) { + std::lock_guard lock(mutex_); + aliases_.Replace(std::move(location), std::string(input->location())); + } Review Comment: This alias mapping only triggers when a `FileIO` returns an `InputFile` whose `location()` differs from the requested location. None of the built-in FileIOs (`ArrowFileSystemFileIO`, `ResolvingFileIO`, `ArrowS3FileIO`) do that; only the test mock does. Java simply keys the cache by `input.location()`. Could we drop `aliases_` to keep this simpler, and add it later if a real use case appears? ########## src/iceberg/table.h: ########## @@ -127,6 +134,16 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> { /// \brief Returns a FileIO to read and write table data and metadata files const std::shared_ptr<FileIO>& io() const; + /// \brief I/O context for immutable manifest lists and manifests only. + /// Each loaded table has its own optional content cache, shared by its readers. + const std::shared_ptr<FileIO>& manifest_io() const { return manifest_io_; } Review Comment: `manifest_io()` is public, but the returned `FileIO` is only a partial adapter (see comments in `content_cache_internal.cc`). Also, despite the doc comment, nothing restricts it to manifests: anything read through it (metadata JSON, puffin files, ...) gets cached. Could this be an internal accessor used only by `update/*` and scans? If it stays public, please document which reads are cached. ########## src/iceberg/catalog/rest/rest_catalog.cc: ########## @@ -1074,6 +1076,8 @@ Result<std::shared_ptr<Table>> RestCatalog::MakeTable( std::shared_ptr<Catalog> table_catalog, std::shared_ptr<MetricsReporter> reporter, const std::unordered_map<std::string, std::string>& table_config, std::shared_ptr<auth::AuthSession> table_session, std::unique_ptr<Labels> labels) { + auto manifest_properties = config_.configs(); + for (const auto& [key, value] : table_config) manifest_properties[key] = value; Review Comment: `manifest_properties` is built before checking the scan planning mode, but the server-mode branch (`RestTable::Make`) never uses it and recomputes the same merge from `rest_context_`. Could we move this into the branch that uses it, or remove it (see the comment on `rest_table.cc`)? ########## src/iceberg/content_cache_internal.cc: ########## @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/content_cache_internal.h" + +#include <algorithm> +#include <cstring> +#include <limits> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg::internal { +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T fallback) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) return fallback; + auto result = StringUtils::ParseNumber<T>(it->second); + if (!result) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, result.error().message); + } + return result; +} + +Status FinishWithCloseStatus(Status operation_status, Status close_status) { + if (!operation_status.has_value()) { + auto error = operation_status.error(); + if (!close_status.has_value()) { + error.message += "; additionally failed to close stream: "; + error.message += close_status.error().message; + } + return ::iceberg::unexpected<Error>(std::move(error)); + } + return close_status; +} Review Comment: nit: this is an exact copy of `FinishWithCloseStatus` in `file_io.cc`. Could we move it to a shared internal helper (e.g. under `util/`) and reuse it in both places? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
