wgtmac commented on code in PR #1033: URL: https://github.com/apache/iceberg-cpp/pull/1033#discussion_r4320399968
########## src/iceberg/util/scheduled_executor.cc: ########## @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <map> +#include <thread> +#include <unordered_map> +#include <utility> + +#include "iceberg/util/scheduled_executor_internal.h" + +namespace iceberg::internal { + +// The timer thread retains its state until Run returns. This also allows the last +// timer owner to be released on the timer thread during process teardown. +class SharedTimerState : public std::enable_shared_from_this<SharedTimerState> { + public: + ~SharedTimerState() { Shutdown(); } + + void Start() { + timer_thread_ = std::thread([self = shared_from_this()] { self->Run(); }); + } + + uint64_t Schedule(std::weak_ptr<ScheduledExecutorState> state, + std::chrono::steady_clock::time_point deadline, ExecutorTask task, + std::optional<ScheduledTaskRejection> on_rejected) { + std::lock_guard lock(mutex_); + auto id = next_id_++; + auto [it, inserted] = tasks_.emplace(std::pair{deadline, id}, + Task{.state = std::move(state), + .task = std::move(task), + .on_rejected = std::move(on_rejected)}); + by_id_.emplace(id, it); + cv_.notify_one(); + return id; + } + + void Cancel(uint64_t id) { + std::optional<Task> cancelled; + { + std::lock_guard lock(mutex_); + auto it = by_id_.find(id); + if (it == by_id_.end()) return; + cancelled.emplace(std::move(it->second->second)); + tasks_.erase(it->second); + by_id_.erase(it); + } + cv_.notify_one(); + // Release callback captures outside the queue lock. + } + + bool IsTimerThread() const { + return timer_thread_.get_id() == std::this_thread::get_id(); + } + + void Shutdown() { + Queue discarded; + { + std::lock_guard lock(mutex_); + if (shutdown_) return; + shutdown_ = true; + tasks_.swap(discarded); + by_id_.clear(); + } + cv_.notify_one(); + if (!timer_thread_.joinable()) return; + if (IsTimerThread()) { + timer_thread_.detach(); + } else { + timer_thread_.join(); + } + } + + private: + struct Task { + std::weak_ptr<ScheduledExecutorState> state; + ExecutorTask task; + std::optional<ScheduledTaskRejection> on_rejected; + }; + + using Key = std::pair<std::chrono::steady_clock::time_point, uint64_t>; + using Queue = std::map<Key, Task>; + + void Run() { + while (true) { + std::optional<Task> task; + uint64_t id; + { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return shutdown_ || !tasks_.empty(); }); + if (shutdown_) return; + + auto key = tasks_.begin()->first; + if (cv_.wait_until(lock, key.first, [this, key] { + return shutdown_ || tasks_.empty() || tasks_.begin()->first != key; + })) { + continue; + } + auto it = tasks_.begin(); + id = it->first.second; + task.emplace(std::move(it->second)); + by_id_.erase(id); + tasks_.erase(it); + } + if (auto state = task->state.lock()) { + state->Dispatch(id, std::move(task->task), std::move(task->on_rejected)); + } + } + } + + std::mutex mutex_; + std::condition_variable cv_; + Queue tasks_; + std::unordered_map<uint64_t, Queue::iterator> by_id_; + uint64_t next_id_ = 1; + bool shutdown_ = false; + std::thread timer_thread_; +}; + +class SharedTimer { + public: + static std::shared_ptr<SharedTimer> Instance() { + static auto timer = std::make_shared<SharedTimer>(); + return timer; + } + + SharedTimer() : state_(std::make_shared<SharedTimerState>()) { state_->Start(); } + + ~SharedTimer() { state_->Shutdown(); } + + uint64_t Schedule(std::weak_ptr<ScheduledExecutorState> state, + std::chrono::steady_clock::time_point deadline, ExecutorTask task, + std::optional<ScheduledTaskRejection> on_rejected) { + return state_->Schedule(std::move(state), deadline, std::move(task), + std::move(on_rejected)); + } + + void Cancel(uint64_t id) { state_->Cancel(id); } + + bool IsTimerThread() const { return state_->IsTimerThread(); } + + private: + std::shared_ptr<SharedTimerState> state_; +}; + +void ScheduledTask::Cancel() { + if (id_.has_value()) { + if (auto state = state_.lock()) state->Cancel(*id_); + } + state_.reset(); + id_.reset(); +} + +ScheduledExecutorState::ScheduledExecutorState(Executor& executor) + : executor_(executor), timer_(SharedTimer::Instance()) {} + +ScheduledExecutorState::~ScheduledExecutorState() { Shutdown(); } + +ScheduledTask ScheduledExecutorState::Schedule( + std::chrono::steady_clock::time_point deadline, ExecutorTask task, + std::optional<ScheduledTaskRejection> on_rejected) { + std::lock_guard lock(mutex_); + if (shutdown_) return {}; + auto id = timer_->Schedule(weak_from_this(), deadline, std::move(task), + std::move(on_rejected)); + scheduled_.insert(id); + return {weak_from_this(), id}; +} + +void ScheduledExecutorState::Cancel(uint64_t id) { + { + std::lock_guard lock(mutex_); + if (scheduled_.erase(id) == 0) return; + } + timer_->Cancel(id); +} + +void ScheduledExecutorState::Shutdown() { + std::unordered_set<uint64_t> pending; + { + std::lock_guard lock(mutex_); + shutdown_ = true; + scheduled_.swap(pending); + } + for (auto id : pending) timer_->Cancel(id); + if (!timer_->IsTimerThread()) { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return active_dispatches_ == 0; }); + } +} Review Comment: Shutdown is already idempotent. The first call closes the owner and swaps out its registrations; later calls swap an empty set and preserve the closed state. Rechecking active_dispatches_ is deliberate: every off-timer Shutdown caller must wait for in-flight Submit calls to finish. An early return based only on shutdown_ would let a concurrent second caller return before that lifetime barrier. The repeated-Shutdown test covers the existing behavior. ########## src/iceberg/catalog/rest/rest_catalog.h: ########## @@ -58,14 +58,19 @@ class ICEBERG_REST_EXPORT RestCatalog final std::function<Result<std::shared_ptr<HttpClient>>(const RestCatalogProperties&)>; /// \brief Create a RestCatalog instance. + /// \param client_factory Optional HTTP client factory, called with the initial config + /// and then the merged server config. When empty, uses the default HTTP client. + /// \param auth_executor Optional non-owning executor for OAuth2 token refresh. When + /// null, background token refresh is disabled. When non-null, it must outlive the + /// catalog and all authentication sessions created by it. It must enqueue work + /// asynchronously without blocking the shared timer thread. /// \param metrics_executor Optional non-owning executor for asynchronous REST metrics. /// It must outlive the catalog and all tables created by it. When null, REST metrics /// are reported synchronously. - /// \param client_factory Optional HTTP client factory, called with the initial config - /// and then the merged server config. When empty, uses the default HTTP client. static Result<std::shared_ptr<RestCatalog>> Make(const RestCatalogProperties& config, - Executor* metrics_executor = nullptr, - ClientFactory client_factory = {}); + ClientFactory client_factory = {}, + Executor* auth_executor = nullptr, + Executor* metrics_executor = nullptr); Review Comment: The source-level change is intentional. This PR explicitly adopts Make(config, client_factory, auth_executor, metrics_executor), and all repository call sites have been updated. We accept the positional API change rather than retain a second compatibility overload or add an options type. Resolving this as an accepted API decision, not as a claim that the old positional signature remains compatible. ########## src/iceberg/util/scheduled_executor.cc: ########## @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <map> +#include <thread> +#include <unordered_map> +#include <utility> + +#include "iceberg/util/scheduled_executor_internal.h" + +namespace iceberg::internal { + +// The timer thread retains its state until Run returns. This also allows the last +// timer owner to be released on the timer thread during process teardown. +class SharedTimerState : public std::enable_shared_from_this<SharedTimerState> { + public: + ~SharedTimerState() { Shutdown(); } + + void Start() { + timer_thread_ = std::thread([self = shared_from_this()] { self->Run(); }); + } + + uint64_t Schedule(std::weak_ptr<ScheduledExecutorState> state, + std::chrono::steady_clock::time_point deadline, ExecutorTask task, + std::optional<ScheduledTaskRejection> on_rejected) { + std::lock_guard lock(mutex_); + auto id = next_id_++; + auto [it, inserted] = tasks_.emplace(std::pair{deadline, id}, + Task{.state = std::move(state), + .task = std::move(task), + .on_rejected = std::move(on_rejected)}); + by_id_.emplace(id, it); + cv_.notify_one(); + return id; + } + + void Cancel(uint64_t id) { + std::optional<Task> cancelled; + { + std::lock_guard lock(mutex_); + auto it = by_id_.find(id); + if (it == by_id_.end()) return; + cancelled.emplace(std::move(it->second->second)); + tasks_.erase(it->second); + by_id_.erase(it); + } + cv_.notify_one(); + // Release callback captures outside the queue lock. + } + + bool IsTimerThread() const { + return timer_thread_.get_id() == std::this_thread::get_id(); + } + + void Shutdown() { + Queue discarded; + { + std::lock_guard lock(mutex_); + if (shutdown_) return; + shutdown_ = true; + tasks_.swap(discarded); + by_id_.clear(); + } + cv_.notify_one(); + if (!timer_thread_.joinable()) return; + if (IsTimerThread()) { + timer_thread_.detach(); + } else { Review Comment: There is no unused-variable build failure here. The structured-binding declaration is used through it, which is stored in by_id_; an unused element does not make the whole declaration unused. The non-trivial optional<Task> named cancelled is deliberately retained until after the queue lock is released, so destroying its callback captures can safely re-enter scheduler code. That lifetime is documented next to the scope. The current head 780956480 passes cpp-linter and the Linux, macOS and Windows build checks; no warning suppression or behavior change is needed. ########## src/iceberg/util/scheduled_executor.cc: ########## @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <map> +#include <thread> +#include <unordered_map> +#include <utility> + +#include "iceberg/util/scheduled_executor_internal.h" + +namespace iceberg::internal { + +// The timer thread retains its state until Run returns. This also allows the last +// timer owner to be released on the timer thread during process teardown. +class SharedTimerState : public std::enable_shared_from_this<SharedTimerState> { + public: + ~SharedTimerState() { Shutdown(); } + + void Start() { + timer_thread_ = std::thread([self = shared_from_this()] { self->Run(); }); + } + + uint64_t Schedule(std::weak_ptr<ScheduledExecutorState> state, + std::chrono::steady_clock::time_point deadline, ExecutorTask task, + std::optional<ScheduledTaskRejection> on_rejected) { + std::lock_guard lock(mutex_); + auto id = next_id_++; + auto [it, inserted] = tasks_.emplace(std::pair{deadline, id}, + Task{.state = std::move(state), + .task = std::move(task), + .on_rejected = std::move(on_rejected)}); + by_id_.emplace(id, it); + cv_.notify_one(); + return id; + } + + void Cancel(uint64_t id) { + std::optional<Task> cancelled; + { + std::lock_guard lock(mutex_); + auto it = by_id_.find(id); + if (it == by_id_.end()) return; + cancelled.emplace(std::move(it->second->second)); + tasks_.erase(it->second); + by_id_.erase(it); + } + cv_.notify_one(); + // Release callback captures outside the queue lock. + } + + bool IsTimerThread() const { + return timer_thread_.get_id() == std::this_thread::get_id(); + } + + void Shutdown() { + Queue discarded; + { + std::lock_guard lock(mutex_); + if (shutdown_) return; + shutdown_ = true; + tasks_.swap(discarded); + by_id_.clear(); + } + cv_.notify_one(); + if (!timer_thread_.joinable()) return; + if (IsTimerThread()) { + timer_thread_.detach(); + } else { + timer_thread_.join(); + } + } + + private: + struct Task { + std::weak_ptr<ScheduledExecutorState> state; + ExecutorTask task; + std::optional<ScheduledTaskRejection> on_rejected; + }; + + using Key = std::pair<std::chrono::steady_clock::time_point, uint64_t>; + using Queue = std::map<Key, Task>; + + void Run() { Review Comment: There is no unused-variable build failure here. The structured-binding declaration is used through it, which is stored in by_id_; an unused element does not make the whole declaration unused. The non-trivial optional<Task> named cancelled is deliberately retained until after the queue lock is released, so destroying its callback captures can safely re-enter scheduler code. That lifetime is documented next to the scope. The current head 780956480 passes cpp-linter and the Linux, macOS and Windows build checks; no warning suppression or behavior change is needed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
