HappenLee commented on code in PR #57329:
URL: https://github.com/apache/doris/pull/57329#discussion_r2464406580
##########
be/src/runtime/user_function_cache.cpp:
##########
@@ -249,12 +255,115 @@ Status UserFunctionCache::_load_cache_entry(const
std::string& url,
RETURN_IF_ERROR(_download_lib(url, entry));
}
+ if (!entry->is_unziped && entry->type == LibType::PY_ZIP) {
+ RETURN_IF_ERROR(_unzip_lib(entry->lib_file));
+ entry->lib_file = entry->lib_file.substr(0, entry->lib_file.size() -
4);
+ entry->is_unziped = true;
+ }
+
if (entry->type == LibType::SO) {
RETURN_IF_ERROR(_load_cache_entry_internal(entry));
- } else if (entry->type != LibType::JAR) {
+ } else if (entry->type != LibType::JAR && entry->type != LibType::PY_ZIP) {
return Status::InvalidArgument(
- "Unsupported lib type! Make sure your lib type is one of 'so'
and 'jar'!");
+ "Unsupported lib type! Make sure your lib type is one of 'so'
and 'jar' and "
+ "python 'zip'!");
+ }
+ return Status::OK();
+}
+
+Status UserFunctionCache::_check_cache_is_python_udf(const std::string& dir,
+ const std::string& file) {
+ const std::string& full_path = dir + "/" + file;
+ RETURN_IF_ERROR(_unzip_lib(full_path));
+ std::string unzip_dir = full_path.substr(0, full_path.size() - 4);
+
+ bool has_python_file = false;
+
+ auto scan_cb = [&has_python_file](const io::FileInfo& file) {
+ if (file.is_file && ends_with(file.file_name, ".py")) {
+ has_python_file = true;
+ return false; // Stop iteration once we find a Python file
+ }
+ return true;
+ };
+
RETURN_IF_ERROR(io::global_local_filesystem()->iterate_directory(unzip_dir,
scan_cb));
+ if (!has_python_file) {
+ return Status::InternalError("No Python file found in the unzipped
directory.");
+ }
+ return Status::OK();
+}
+
+Status UserFunctionCache::_unzip_lib(const std::string& zip_file) {
+ std::string unzip_dir = zip_file.substr(0, zip_file.size() - 4);
+
RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(unzip_dir));
+
+ unzFile zip_file_handle = unzOpen(zip_file.c_str());
+ if (zip_file_handle == nullptr) {
+ return Status::InternalError("Failed to open zip file: " + zip_file);
+ }
+
+ unz_global_info global_info;
+ if (unzGetGlobalInfo(zip_file_handle, &global_info) != UNZ_OK) {
+ unzClose(zip_file_handle);
+ return Status::InternalError("Failed to get global info from zip file:
" + zip_file);
+ }
+
+ for (uLong i = 0; i < global_info.number_entry; ++i) {
+ unz_file_info file_info;
+ char filename[256];
+ if (unzGetCurrentFileInfo(zip_file_handle, &file_info, filename,
sizeof(filename), nullptr, 0, nullptr, 0) != UNZ_OK) {
+ unzClose(zip_file_handle);
+ return Status::InternalError("Failed to get file info from zip
file: " + zip_file);
+ }
+
+ if (std::string(filename).find("__MACOSX") != std::string::npos) {
+ if ((i + 1) < global_info.number_entry) {
+ if (unzGoToNextFile(zip_file_handle) != UNZ_OK) {
+ unzClose(zip_file_handle);
+ return Status::InternalError("Failed to go to next file in
zip: " + zip_file);
+ }
+ }
+ continue;
+ }
+ std::string full_filename = unzip_dir + "/" + filename;
+ if (full_filename.length() > PATH_MAX) {
+ unzClose(zip_file_handle);
+ return Status::InternalError("File path is too long: " +
full_filename);
Review Comment:
also return the PATH_MAX user know how long is better
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
