manuzhang commented on code in PR #875: URL: https://github.com/apache/iceberg-cpp/pull/875#discussion_r4469551000
########## src/iceberg/metadata_cache.cc: ########## @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/metadata_cache.h" + +#include <algorithm> +#include <chrono> +#include <condition_variable> +#include <limits> +#include <list> +#include <mutex> +#include <utility> + +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" + +namespace iceberg { + +namespace { + +template <typename T> +Result<T> ParseNumberProperty( + const std::unordered_map<std::string, std::string>& properties, std::string_view key, + T default_value) { + auto it = properties.find(std::string(key)); + if (it == properties.end()) { + return default_value; + } + auto value = StringUtils::ParseNumber<T>(it->second); + if (!value.has_value()) { + return InvalidArgument("Invalid numeric value '{}' for property '{}': {}", it->second, + key, value.error().message); + } + return value; +} + +} // namespace + +Result<MetadataCacheOptions> MetadataCacheOptions::FromProperties( + const std::unordered_map<std::string, std::string>& properties) { + MetadataCacheOptions options; + if (auto it = properties.find(std::string(kEnabled)); it != properties.end()) { + options.enabled = StringUtils::ParseBoolean(it->second); + } + + ICEBERG_ASSIGN_OR_RAISE(options.expiration_interval_ms, + ParseNumberProperty(properties, kExpirationIntervalMs, + options.expiration_interval_ms)); + ICEBERG_ASSIGN_OR_RAISE( + options.max_total_bytes, + ParseNumberProperty(properties, kMaxTotalBytes, options.max_total_bytes)); + ICEBERG_ASSIGN_OR_RAISE( + options.max_content_length, + ParseNumberProperty(properties, kMaxContentLength, options.max_content_length)); + return options; +} + +class MetadataCache::Impl { + public: + using Clock = std::chrono::steady_clock; + + explicit Impl(MetadataCacheOptions options) : options_(std::move(options)) {} + + struct Entry { + bool loading = true; + Content content; + std::optional<Error> error; + Clock::time_point last_access; + std::list<std::string>::iterator lru_position; + std::condition_variable loaded; + }; + + bool IsExpired(const Entry& entry, Clock::time_point now) const { + return options_.expiration_interval_ms > 0 && + now - entry.last_access >= + std::chrono::milliseconds(options_.expiration_interval_ms); Review Comment: This revision removes time-based expiration, so this comparison no longer exists. Cached manifests and manifest lists are immutable, so entries only leave the cache through size-based eviction or `ClearMetadataCache()`. `io.manifest.cache.expiration-interval-ms` is no longer read. ########## src/iceberg/test/metadata_cache_test.cc: ########## @@ -0,0 +1,628 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/metadata_cache.h" + +#include <atomic> +#include <barrier> +#include <chrono> +#include <future> +#include <memory> +#include <stdexcept> +#include <string> +#include <string_view> +#include <unordered_map> +#include <utility> + +#include <gtest/gtest.h> + +#include "iceberg/file_io_registry.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +using namespace std::chrono_literals; + +MetadataCacheOptions EnabledOptions(size_t max_total_bytes = 100, + size_t max_content_length = 100) { + return { + .enabled = true, + .expiration_interval_ms = 0, + .max_total_bytes = max_total_bytes, + .max_content_length = max_content_length, + }; +} + +class CountingInputFile : public InputFile { + public: + CountingInputFile(std::unique_ptr<InputFile> input_file, int* open_count, + int* size_count) + : input_file_(std::move(input_file)), + open_count_(open_count), + size_count_(size_count) {} + + std::string_view location() const override { return input_file_->location(); } + + Result<int64_t> Size() const override { + ++*size_count_; + return input_file_->Size(); + } + + Result<std::unique_ptr<SeekableInputStream>> Open() override { + ++*open_count_; + return input_file_->Open(); + } + + private: + std::unique_ptr<InputFile> input_file_; + int* open_count_; + int* size_count_; +}; + +class CountingMockFileIO : public MockFileIO { + public: + Result<std::unique_ptr<InputFile>> NewInputFile(std::string file_location) override { + ICEBERG_ASSIGN_OR_RAISE(auto input_file, + MockFileIO::NewInputFile(std::move(file_location))); + return std::make_unique<CountingInputFile>(std::move(input_file), &open_count, + &size_count); + } + + int open_count = 0; + int size_count = 0; +}; + +class NormalizingMockFileIO : public CountingMockFileIO { + public: + Result<std::unique_ptr<InputFile>> NewInputFile(std::string file_location) override { + if (fail_resolution) { + return IOError("Cannot resolve file location"); + } + constexpr std::string_view kAliasPrefix = "alias://"; + if (file_location.starts_with(kAliasPrefix)) { + file_location.erase(0, kAliasPrefix.size()); + } + return CountingMockFileIO::NewInputFile(std::move(file_location)); + } + + bool fail_resolution = false; +}; + +struct FailingOpenState { + int open_count = 0; + Error first_error; + Error fallback_error; +}; + +class FailingOpenInputFile : public InputFile { + public: + explicit FailingOpenInputFile(std::shared_ptr<FailingOpenState> state) + : state_(std::move(state)) {} + + std::string_view location() const override { return "manifest.avro"; } + Result<int64_t> Size() const override { return 8; } + + Result<std::unique_ptr<SeekableInputStream>> Open() override { + ++state_->open_count; + if (state_->open_count == 1) { + return unexpected<Error>(state_->first_error); + } + return unexpected<Error>(state_->fallback_error); + } + + private: + std::shared_ptr<FailingOpenState> state_; +}; + +class FailingOpenFileIO : public FileIO { + public: + explicit FailingOpenFileIO(std::shared_ptr<FailingOpenState> state) + : state_(std::move(state)) {} + + Result<std::unique_ptr<InputFile>> NewInputFile( + std::string /*file_location*/) override { + return std::make_unique<FailingOpenInputFile>(state_); + } + + private: + std::shared_ptr<FailingOpenState> state_; +}; + +TEST(MetadataCacheTest, ReusesContentAcrossLoads) { + auto cache_result = MetadataCache::Make(EnabledOptions()); + ASSERT_THAT(cache_result, IsOk()); + auto cache = std::move(cache_result).value(); + + int loads = 0; + auto loader = [&]() -> Result<MetadataCache::Content> { + ++loads; + return std::make_shared<const std::string>("metadata"); + }; + + auto first = cache->Get("s3://warehouse/table/metadata/v1.metadata.json", 8, loader); + auto second = cache->Get("s3://warehouse/table/metadata/v1.metadata.json", 8, loader); + + ASSERT_THAT(first, IsOk()); + ASSERT_THAT(second, IsOk()); + EXPECT_EQ(loads, 1); + EXPECT_EQ(first.value(), second.value()); + EXPECT_EQ(cache->size(), 1); + EXPECT_EQ(cache->total_bytes(), 8); +} + +TEST(MetadataCacheTest, FileIOReusesCachedContentAcrossInputFiles) { + CountingMockFileIO file_io; + file_io.AddFile("manifest.avro", "manifest"); + ASSERT_THAT(file_io.ConfigureMetadataCache( + {{std::string(MetadataCacheOptions::kEnabled), "true"}}), + IsOk()); + + auto first_file = file_io.NewCachedInputFile("manifest.avro"); + ASSERT_THAT(first_file, IsOk()); + auto first_stream = first_file.value()->Open(); + ASSERT_THAT(first_stream, IsOk()); + ASSERT_THAT(first_stream.value()->Close(), IsOk()); + + auto second_file = file_io.NewCachedInputFile("manifest.avro"); + ASSERT_THAT(second_file, IsOk()); + auto second_stream = second_file.value()->Open(); + ASSERT_THAT(second_stream, IsOk()); + ASSERT_THAT(second_stream.value()->Close(), IsOk()); + + EXPECT_EQ(file_io.open_count, 1); Review Comment: Added `TestManifestReader.ReadsManifestListAndManifestFromCachedContent`, run for format versions 1 to 3. It writes a real manifest and manifest list, then reads both twice with caching enabled through `ManifestListReader` and `ManifestReader`. It checks the decoded manifest file and data file rows, and uses a counting Arrow mock filesystem to assert that the second read opens neither file. At the end, a FileIO without caching reopens the manifest, which shows the counter tracks real reads. The test fails if the readers stop setting `cache_content`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
