wgtmac commented on code in PR #875:
URL: https://github.com/apache/iceberg-cpp/pull/875#discussion_r4425800164
##########
src/iceberg/file_reader.h:
##########
@@ -102,6 +102,8 @@ struct ICEBERG_EXPORT ReaderOptions {
std::optional<Split> split;
/// \brief FileIO instance to open the file.
std::shared_ptr<class FileIO> io;
+ /// \brief Cache this immutable metadata file's content when FileIO caching
is enabled.
+ bool cache_content = false;
Review Comment:
Please append this field after the existing members. Inserting it before
`projection` breaks valid callers such as `ReaderOptions{path, {}, {}, io,
schema}`. Add coverage for the old aggregate initialization form.
##########
src/iceberg/metadata_cache.cc:
##########
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/metadata_cache.h"
+
+#include <algorithm>
+#include <chrono>
+#include <condition_variable>
+#include <limits>
+#include <list>
+#include <mutex>
+#include <utility>
+
+#include "iceberg/util/macros.h"
+#include "iceberg/util/string_util.h"
+
+namespace iceberg {
+
+namespace {
+
+template <typename T>
+Result<T> ParseNumberProperty(
+ const std::unordered_map<std::string, std::string>& properties,
std::string_view key,
+ T default_value) {
+ auto it = properties.find(std::string(key));
+ if (it == properties.end()) {
+ return default_value;
+ }
+ auto value = StringUtils::ParseNumber<T>(it->second);
+ if (!value.has_value()) {
+ return InvalidArgument("Invalid numeric value '{}' for property '{}': {}",
it->second,
+ key, value.error().message);
+ }
+ return value;
+}
+
+} // namespace
+
+Result<MetadataCacheOptions> MetadataCacheOptions::FromProperties(
+ const std::unordered_map<std::string, std::string>& properties) {
+ MetadataCacheOptions options;
+ if (auto it = properties.find(std::string(kEnabled)); it !=
properties.end()) {
+ options.enabled = StringUtils::ParseBoolean(it->second);
+ }
+
+ ICEBERG_ASSIGN_OR_RAISE(options.expiration_interval_ms,
+ ParseNumberProperty(properties,
kExpirationIntervalMs,
+ options.expiration_interval_ms));
+ ICEBERG_ASSIGN_OR_RAISE(
+ options.max_total_bytes,
+ ParseNumberProperty(properties, kMaxTotalBytes,
options.max_total_bytes));
+ ICEBERG_ASSIGN_OR_RAISE(
+ options.max_content_length,
+ ParseNumberProperty(properties, kMaxContentLength,
options.max_content_length));
+ return options;
+}
+
+class MetadataCache::Impl {
+ public:
+ using Clock = std::chrono::steady_clock;
+
+ explicit Impl(MetadataCacheOptions options) : options_(std::move(options)) {}
+
+ struct Entry {
+ bool loading = true;
+ Content content;
+ std::optional<Error> error;
+ Clock::time_point last_access;
+ std::list<std::string>::iterator lru_position;
+ std::condition_variable loaded;
+ };
+
+ bool IsExpired(const Entry& entry, Clock::time_point now) const {
+ return options_.expiration_interval_ms > 0 &&
+ now - entry.last_access >=
+ std::chrono::milliseconds(options_.expiration_interval_ms);
Review Comment:
`INT64_MAX` milliseconds passes validation, but this comparison converts it
to nanoseconds on libc++ and overflows. Compare elapsed milliseconds instead,
or reject values that cannot be converted safely, and cover the upper boundary.
##########
src/iceberg/test/metadata_cache_test.cc:
##########
@@ -0,0 +1,628 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/metadata_cache.h"
+
+#include <atomic>
+#include <barrier>
+#include <chrono>
+#include <future>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include <gtest/gtest.h>
+
+#include "iceberg/file_io_registry.h"
+#include "iceberg/test/matchers.h"
+#include "iceberg/test/mock_io.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+namespace {
+
+using namespace std::chrono_literals;
+
+MetadataCacheOptions EnabledOptions(size_t max_total_bytes = 100,
+ size_t max_content_length = 100) {
+ return {
+ .enabled = true,
+ .expiration_interval_ms = 0,
+ .max_total_bytes = max_total_bytes,
+ .max_content_length = max_content_length,
+ };
+}
+
+class CountingInputFile : public InputFile {
+ public:
+ CountingInputFile(std::unique_ptr<InputFile> input_file, int* open_count,
+ int* size_count)
+ : input_file_(std::move(input_file)),
+ open_count_(open_count),
+ size_count_(size_count) {}
+
+ std::string_view location() const override { return input_file_->location();
}
+
+ Result<int64_t> Size() const override {
+ ++*size_count_;
+ return input_file_->Size();
+ }
+
+ Result<std::unique_ptr<SeekableInputStream>> Open() override {
+ ++*open_count_;
+ return input_file_->Open();
+ }
+
+ private:
+ std::unique_ptr<InputFile> input_file_;
+ int* open_count_;
+ int* size_count_;
+};
+
+class CountingMockFileIO : public MockFileIO {
+ public:
+ Result<std::unique_ptr<InputFile>> NewInputFile(std::string file_location)
override {
+ ICEBERG_ASSIGN_OR_RAISE(auto input_file,
+
MockFileIO::NewInputFile(std::move(file_location)));
+ return std::make_unique<CountingInputFile>(std::move(input_file),
&open_count,
+ &size_count);
+ }
+
+ int open_count = 0;
+ int size_count = 0;
+};
+
+class NormalizingMockFileIO : public CountingMockFileIO {
+ public:
+ Result<std::unique_ptr<InputFile>> NewInputFile(std::string file_location)
override {
+ if (fail_resolution) {
+ return IOError("Cannot resolve file location");
+ }
+ constexpr std::string_view kAliasPrefix = "alias://";
+ if (file_location.starts_with(kAliasPrefix)) {
+ file_location.erase(0, kAliasPrefix.size());
+ }
+ return CountingMockFileIO::NewInputFile(std::move(file_location));
+ }
+
+ bool fail_resolution = false;
+};
+
+struct FailingOpenState {
+ int open_count = 0;
+ Error first_error;
+ Error fallback_error;
+};
+
+class FailingOpenInputFile : public InputFile {
+ public:
+ explicit FailingOpenInputFile(std::shared_ptr<FailingOpenState> state)
+ : state_(std::move(state)) {}
+
+ std::string_view location() const override { return "manifest.avro"; }
+ Result<int64_t> Size() const override { return 8; }
+
+ Result<std::unique_ptr<SeekableInputStream>> Open() override {
+ ++state_->open_count;
+ if (state_->open_count == 1) {
+ return unexpected<Error>(state_->first_error);
+ }
+ return unexpected<Error>(state_->fallback_error);
+ }
+
+ private:
+ std::shared_ptr<FailingOpenState> state_;
+};
+
+class FailingOpenFileIO : public FileIO {
+ public:
+ explicit FailingOpenFileIO(std::shared_ptr<FailingOpenState> state)
+ : state_(std::move(state)) {}
+
+ Result<std::unique_ptr<InputFile>> NewInputFile(
+ std::string /*file_location*/) override {
+ return std::make_unique<FailingOpenInputFile>(state_);
+ }
+
+ private:
+ std::shared_ptr<FailingOpenState> state_;
+};
+
+TEST(MetadataCacheTest, ReusesContentAcrossLoads) {
+ auto cache_result = MetadataCache::Make(EnabledOptions());
+ ASSERT_THAT(cache_result, IsOk());
+ auto cache = std::move(cache_result).value();
+
+ int loads = 0;
+ auto loader = [&]() -> Result<MetadataCache::Content> {
+ ++loads;
+ return std::make_shared<const std::string>("metadata");
+ };
+
+ auto first = cache->Get("s3://warehouse/table/metadata/v1.metadata.json", 8,
loader);
+ auto second = cache->Get("s3://warehouse/table/metadata/v1.metadata.json",
8, loader);
+
+ ASSERT_THAT(first, IsOk());
+ ASSERT_THAT(second, IsOk());
+ EXPECT_EQ(loads, 1);
+ EXPECT_EQ(first.value(), second.value());
+ EXPECT_EQ(cache->size(), 1);
+ EXPECT_EQ(cache->total_bytes(), 8);
+}
+
+TEST(MetadataCacheTest, FileIOReusesCachedContentAcrossInputFiles) {
+ CountingMockFileIO file_io;
+ file_io.AddFile("manifest.avro", "manifest");
+ ASSERT_THAT(file_io.ConfigureMetadataCache(
+ {{std::string(MetadataCacheOptions::kEnabled), "true"}}),
+ IsOk());
+
+ auto first_file = file_io.NewCachedInputFile("manifest.avro");
+ ASSERT_THAT(first_file, IsOk());
+ auto first_stream = first_file.value()->Open();
+ ASSERT_THAT(first_stream, IsOk());
+ ASSERT_THAT(first_stream.value()->Close(), IsOk());
+
+ auto second_file = file_io.NewCachedInputFile("manifest.avro");
+ ASSERT_THAT(second_file, IsOk());
+ auto second_stream = second_file.value()->Open();
+ ASSERT_THAT(second_stream, IsOk());
+ ASSERT_THAT(second_stream.value()->Close(), IsOk());
+
+ EXPECT_EQ(file_io.open_count, 1);
Review Comment:
This only opens and closes the cached stream. Add a regression that reads a
real manifest and manifest list twice through their readers with caching
enabled, checks the decoded rows, and verifies the underlying open counts.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
