zy-kkk commented on code in PR #54304: URL: https://github.com/apache/doris/pull/54304#discussion_r2335451866
########## be/src/runtime/plugin/cloud_plugin_downloader.cpp: ########## @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "runtime/plugin/cloud_plugin_downloader.h" + +#include <fmt/format.h> + +#include "cloud/cloud_storage_engine.h" +#include "io/fs/local_file_system.h" +#include "io/fs/remote_file_system.h" +#include "runtime/exec_env.h" + +namespace doris { + +// Use 10MB buffer for all downloads - same as cloud_warm_up_manager +static constexpr size_t DOWNLOAD_BUFFER_SIZE = 10 * 1024 * 1024; // 10MB + +// Static mutex definition for synchronizing downloads +std::mutex CloudPluginDownloader::_download_mutex; + +Status CloudPluginDownloader::download_from_cloud(PluginType type, const std::string& name, + const std::string& local_path, + std::string* result_path) { + // Use lock_guard to synchronize concurrent downloads + std::lock_guard<std::mutex> lock(_download_mutex); + + if (name.empty()) { + return Status::InvalidArgument("Plugin name cannot be empty"); + } + + CloudPluginDownloader downloader; + + // 1. Get FileSystem + io::RemoteFileSystemSPtr filesystem; + RETURN_IF_ERROR(downloader._get_cloud_filesystem(&filesystem)); + + // 2. Build remote plugin path + std::string remote_path; + RETURN_IF_ERROR(downloader._build_plugin_path(type, name, &remote_path)); + LOG(INFO) << "Downloading plugin: " << name << " -> " << local_path; + + // 3. Prepare local environment + RETURN_IF_ERROR(downloader._prepare_local_path(local_path)); + + // 4. Download remote file to local + RETURN_IF_ERROR(downloader._download_remote_file(filesystem, remote_path, local_path)); + + *result_path = local_path; + LOG(INFO) << "Successfully downloaded plugin: " << name << " to " << local_path; + + return Status::OK(); +} + +Status CloudPluginDownloader::_build_plugin_path(PluginType type, const std::string& name, + std::string* path) { + std::string type_name; + switch (type) { + case PluginType::JDBC_DRIVERS: + type_name = "jdbc_drivers"; + break; + case PluginType::JAVA_UDF: + type_name = "java_udf"; + break; + default: + return Status::InvalidArgument("Unsupported plugin type: {}", static_cast<int>(type)); + } + *path = fmt::format("plugins/{}/{}", type_name, name); + return Status::OK(); +} + +Status CloudPluginDownloader::_get_cloud_filesystem(io::RemoteFileSystemSPtr* filesystem) { + BaseStorageEngine& base_engine = ExecEnv::GetInstance()->storage_engine(); + CloudStorageEngine* cloud_engine = dynamic_cast<CloudStorageEngine*>(&base_engine); + if (!cloud_engine) { + return Status::NotFound("CloudStorageEngine not found, not in cloud mode"); + } + + *filesystem = cloud_engine->latest_fs(); Review Comment: Only SAAS scenarios are considered, and multi-vault custom deployment and storage-computing separation scenarios are not considered. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
