This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b4466bc35bf branch-3.1: [fix](plugin) fix plugin dir compatible issue
(#56060)
b4466bc35bf is described below
commit b4466bc35bfc45e7441e354a4b4323088f79b645
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Sep 16 23:01:14 2025 -0700
branch-3.1: [fix](plugin) fix plugin dir compatible issue (#56060)
Followup #52921
Some code is missing after cherry-pick
When upgrading from 3.0.x to 3.1.x, if user put jdbc driver jar under
`${DORIS_HOME}/jdbc_drivers`,
the system will fail to find the jar, which is unexpected.
For compatibility, the system should check both
`${DORIS_HOME}/jdbc_drivers`(old) and
`${DORIS_HOME}/plugins/jdbc_drivers`(new) to find the jar.
This PR fix it.
Only for branch-3.1. No issue on master branch
---
be/src/common/config.cpp | 3 +
be/src/common/config.h | 3 +
be/src/runtime/user_function_cache.cpp | 6 +-
be/src/runtime/user_function_cache.h | 2 -
be/src/util/path_util.cpp | 46 +++++++++++++
be/src/util/path_util.h | 16 +++++
be/src/vec/exec/vjdbc_connector.cpp | 11 +--
be/src/vec/exec/vjdbc_connector.h | 2 -
be/test/util/path_util_test.cpp | 119 +++++++++++++++++++++++++++++++++
9 files changed, 195 insertions(+), 13 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 243cd65351a..931130994da 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -54,6 +54,9 @@ DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf");
// Dir of jdbc drivers
DEFINE_String(jdbc_drivers_dir, "${DORIS_HOME}/plugins/jdbc_drivers");
+// Dir of java udf
+DEFINE_String(java_udf_dir, "${DORIS_HOME}/plugins/java_udf");
+
// cluster id
DEFINE_Int32(cluster_id, "-1");
// port on which BackendService is exported
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 55591fc36c7..2b3691c9186 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -88,6 +88,9 @@ DECLARE_String(custom_config_dir);
// Dir of jdbc drivers
DECLARE_String(jdbc_drivers_dir);
+// Dir of java udf
+DECLARE_String(java_udf_dir);
+
// cluster id
DECLARE_Int32(cluster_id);
// port on which BackendService is exported
diff --git a/be/src/runtime/user_function_cache.cpp
b/be/src/runtime/user_function_cache.cpp
index e058b76ad33..5ec3c633d15 100644
--- a/be/src/runtime/user_function_cache.cpp
+++ b/be/src/runtime/user_function_cache.cpp
@@ -42,6 +42,7 @@
#include "runtime/exec_env.h"
#include "util/dynamic_util.h"
#include "util/md5.h"
+#include "util/path_util.h"
#include "util/string_util.h"
namespace doris {
@@ -271,10 +272,12 @@ Status UserFunctionCache::_download_lib(const
std::string& url,
return Status::InternalError("fail to open file");
}
+ std::string udf_path =
+ doris::path_util::get_real_plugin_url(url,
doris::config::java_udf_dir, "java_udf");
Md5Digest digest;
HttpClient client;
int64_t file_size = 0;
- RETURN_IF_ERROR(client.init(url));
+ RETURN_IF_ERROR(client.init(udf_path));
Status status;
auto download_cb = [&status, &tmp_file, &fp, &digest, &file_size](const
void* data,
size_t
length) {
@@ -382,4 +385,5 @@ std::vector<std::string>
UserFunctionCache::_split_string_by_checksum(const std:
return result;
}
+
} // namespace doris
diff --git a/be/src/runtime/user_function_cache.h
b/be/src/runtime/user_function_cache.h
index 2537d9376ec..93759c261e2 100644
--- a/be/src/runtime/user_function_cache.h
+++ b/be/src/runtime/user_function_cache.h
@@ -75,8 +75,6 @@ private:
std::string _get_file_name_from_url(const std::string& url) const;
std::vector<std::string> _split_string_by_checksum(const std::string&
file);
- std::string _check_and_return_default_driver_url(const std::string& url);
-
private:
std::string _lib_dir;
void* _current_process_handle = nullptr;
diff --git a/be/src/util/path_util.cpp b/be/src/util/path_util.cpp
index e05b9371e49..4b6f6adcbed 100644
--- a/be/src/util/path_util.cpp
+++ b/be/src/util/path_util.cpp
@@ -20,6 +20,10 @@
// Use the POSIX version of dirname(3). See `man 3 dirname`
#include <libgen.h>
+#include <cstdlib>
+#include <filesystem>
+
+#include "common/config.h"
#include "gutil/strings/split.h"
#include "gutil/strings/strip.h"
@@ -64,5 +68,47 @@ std::string file_extension(const string& path) {
return pos == string::npos ? "" : file_name.substr(pos);
}
+std::string get_real_plugin_url(const std::string& url, const std::string&
plugin_dir_config_value,
+ const std::string& plugin_dir_name, const
std::string& doris_home) {
+ if (url.find(":/") == std::string::npos) {
+ return check_and_return_default_plugin_url(url,
plugin_dir_config_value, plugin_dir_name,
+ doris_home);
+ }
+ return url;
+}
+
+std::string check_and_return_default_plugin_url(const std::string& url,
+ const std::string&
plugin_dir_config_value,
+ const std::string&
plugin_dir_name,
+ const std::string& doris_home)
{
+ std::string home_dir = doris_home;
+ if (home_dir.empty()) {
+ const char* env_home = std::getenv("DORIS_HOME");
+ if (env_home) {
+ home_dir = std::string(env_home);
+ } else {
+ return "file://" + plugin_dir_config_value + "/" + url;
+ }
+ }
+
+ std::string default_url = home_dir + "/plugins/" + plugin_dir_name;
+ std::string default_old_url = home_dir + "/" + plugin_dir_name;
+
+ if (plugin_dir_config_value == default_url) {
+ // If true, which means user does not set `jdbc_drivers_dir` and use
the default one.
+ // Because in 2.1.8, we change the default value of `jdbc_drivers_dir`
+ // from `DORIS_HOME/jdbc_drivers` to `DORIS_HOME/plugins/jdbc_drivers`,
+ // so we need to check the old default dir for compatibility.
+ std::filesystem::path file = default_url + "/" + url;
+ if (std::filesystem::exists(file)) {
+ return "file://" + default_url + "/" + url;
+ } else {
+ return "file://" + default_old_url + "/" + url;
+ }
+ } else {
+ return "file://" + plugin_dir_config_value + "/" + url;
+ }
+}
+
} // namespace path_util
} // namespace doris
diff --git a/be/src/util/path_util.h b/be/src/util/path_util.h
index ee68122f4fa..61231e6239d 100644
--- a/be/src/util/path_util.h
+++ b/be/src/util/path_util.h
@@ -55,5 +55,21 @@ std::string base_name(const std::string& path);
// NOTE: path can be either one file's full path or only file name
std::string file_extension(const std::string& path);
+// Get the real URL for plugins (e.g., JDBC drivers). If the URL doesn't
contain ":/",
+// it will be treated as a relative path and converted to a file:// URL using
provided dirs.
+// plugin_dir_config_value is the configured plugin dir; plugin_dir_name is
the dir name (e.g. "jdbc_drivers").
+// doris_home is optional; if empty, will use DORIS_HOME environment variable.
+std::string get_real_plugin_url(const std::string& url, const std::string&
plugin_dir_config_value,
+ const std::string& plugin_dir_name,
+ const std::string& doris_home = "");
+
+// Check and return the default plugin URL using provided directories.
+// plugin_dir_config_value is the configured drivers dir; plugin_dir_name is
dir name.
+// doris_home is optional; if empty, will use DORIS_HOME environment variable.
+std::string check_and_return_default_plugin_url(const std::string& url,
+ const std::string&
plugin_dir_config_value,
+ const std::string&
plugin_dir_name,
+ const std::string& doris_home
= "");
+
} // namespace path_util
} // namespace doris
diff --git a/be/src/vec/exec/vjdbc_connector.cpp
b/be/src/vec/exec/vjdbc_connector.cpp
index 17e4781a5a6..d4cf31f4f78 100644
--- a/be/src/vec/exec/vjdbc_connector.cpp
+++ b/be/src/vec/exec/vjdbc_connector.cpp
@@ -36,6 +36,7 @@
#include "runtime/types.h"
#include "runtime/user_function_cache.h"
#include "util/jni-util.h"
+#include "util/path_util.h"
#include "util/runtime_profile.h"
#include "vec/columns/column_nullable.h"
#include "vec/core/block.h"
@@ -126,7 +127,8 @@ Status JdbcConnector::open(RuntimeState* state, bool read) {
// Add a scoped cleanup jni reference object. This cleans up local refs
made below.
JniLocalFrame jni_frame;
{
- std::string driver_path = _get_real_url(_conn_param.driver_path);
+ std::string driver_path = doris::path_util::get_real_plugin_url(
+ _conn_param.driver_path, doris::config::jdbc_drivers_dir,
"jdbc_drivers", "");
TJdbcExecutorCtorParams ctor_params;
ctor_params.__set_statement(_sql_str);
@@ -633,11 +635,4 @@ Status JdbcConnector::_get_java_table_type(JNIEnv* env,
TOdbcTableType::type tab
return Status::OK();
}
-std::string JdbcConnector::_get_real_url(const std::string& url) {
- if (url.find(":/") == std::string::npos) {
- return "file://" + config::jdbc_drivers_dir + "/" + url;
- }
- return url;
-}
-
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/vjdbc_connector.h
b/be/src/vec/exec/vjdbc_connector.h
index 8838ce5984d..b514389b5da 100644
--- a/be/src/vec/exec/vjdbc_connector.h
+++ b/be/src/vec/exec/vjdbc_connector.h
@@ -141,8 +141,6 @@ private:
Status _get_java_table_type(JNIEnv* env, TOdbcTableType::type table_type,
jobject* java_enum_obj);
- std::string _get_real_url(const std::string& url);
-
bool _closed = false;
jclass _executor_factory_clazz;
jclass _executor_clazz;
diff --git a/be/test/util/path_util_test.cpp b/be/test/util/path_util_test.cpp
index 8806febb133..3ec95c5a1ac 100644
--- a/be/test/util/path_util_test.cpp
+++ b/be/test/util/path_util_test.cpp
@@ -20,6 +20,9 @@
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
#include <string>
#include <vector>
@@ -99,4 +102,120 @@ TEST(TestPathUtil, file_extension_test) {
EXPECT_EQ(".", path_util::file_extension("a.b.c."));
}
+// Helpers for plugin URL tests
+namespace {
+
+std::string create_temp_home() {
+ namespace fs = std::filesystem;
+ fs::path base = fs::temp_directory_path() / "doris_path_util_test";
+ fs::create_directories(base);
+ // ensure unique subdir per test run
+ fs::path home = base / std::to_string(reinterpret_cast<uintptr_t>(&base));
+ fs::create_directories(home);
+ return home.string();
+}
+
+void touch_file(const std::string& dir, const std::string& filename) {
+ namespace fs = std::filesystem;
+ fs::create_directories(dir);
+ std::ofstream ofs(fs::path(dir) / filename, std::ios::binary);
+ ofs << "x";
+}
+
+} // namespace
+
+TEST(TestPathUtil, get_real_plugin_url_absolute_passthrough) {
+ // absolute style URLs containing ":/" should be returned as-is
+ EXPECT_EQ("http://example.com/a.jar";,
+ path_util::get_real_plugin_url("http://example.com/a.jar";,
"/any/dir", "jdbc_drivers",
+ ""));
+ EXPECT_EQ("file:///opt/driver/a.jar",
+ path_util::get_real_plugin_url("file:///opt/driver/a.jar",
"/any/dir", "jdbc_drivers",
+ ""));
+}
+
+TEST(TestPathUtil,
check_and_return_default_plugin_url_prefers_new_default_when_exists) {
+ namespace fs = std::filesystem;
+ std::string home = create_temp_home();
+ std::string plugin_name = "jdbc_drivers";
+ std::string default_new = home + "/plugins/" + plugin_name;
+ std::string default_old = home + "/" + plugin_name;
+ std::string fname = "drv.jar";
+
+ touch_file(default_new, fname);
+
+ std::string expected = "file://" + default_new + "/" + fname;
+ EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname,
default_new,
+
plugin_name, home));
+}
+
+TEST(TestPathUtil,
check_and_return_default_plugin_url_falls_back_to_old_default) {
+ std::string home = create_temp_home();
+ std::string plugin_name = "jdbc_drivers";
+ std::string default_new = home + "/plugins/" + plugin_name;
+ std::string default_old = home + "/" + plugin_name;
+ std::string fname = "drv.jar";
+
+ // create only old default file
+ touch_file(default_old, fname);
+
+ std::string expected = "file://" + default_old + "/" + fname;
+ EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname,
default_new,
+
plugin_name, home));
+}
+
+TEST(TestPathUtil, check_and_return_default_plugin_url_old_even_if_missing) {
+ std::string home = create_temp_home();
+ std::string plugin_name = "jdbc_drivers";
+ std::string default_new = home + "/plugins/" + plugin_name;
+ std::string default_old = home + "/" + plugin_name;
+ std::string fname = "drv.jar";
+
+ // neither new nor old has the file; should still point to old default
+ std::string expected = "file://" + default_old + "/" + fname;
+ EXPECT_EQ(expected, path_util::check_and_return_default_plugin_url(fname,
default_new,
+
plugin_name, home));
+}
+
+TEST(TestPathUtil, check_and_return_default_plugin_url_custom_config_dir) {
+ std::string home = create_temp_home();
+ std::string plugin_name = "jdbc_drivers";
+ std::string custom_dir = home + "/custom/plugins";
+ std::string fname = "drv.jar";
+ touch_file(custom_dir, fname);
+
+ std::string expected = "file://" + custom_dir + "/" + fname;
+ EXPECT_EQ(expected,
+ path_util::check_and_return_default_plugin_url(fname,
custom_dir, plugin_name, home));
+}
+
+TEST(TestPathUtil, get_real_plugin_url_relative_paths) {
+ std::string home = create_temp_home();
+ std::string plugin_name = "jdbc_drivers";
+ std::string default_new = home + "/plugins/" + plugin_name;
+ std::string default_old = home + "/" + plugin_name;
+ std::string fname = "drv.jar";
+
+ // When new default exists
+ touch_file(default_new, fname);
+ std::string expected_new = "file://" + default_new + "/" + fname;
+ EXPECT_EQ(expected_new, path_util::get_real_plugin_url(fname, default_new,
plugin_name, home));
+
+ // When only old default exists
+ std::string home2 = create_temp_home();
+ std::string default_new2 = home2 + "/plugins/" + plugin_name;
+ std::string default_old2 = home2 + "/" + plugin_name;
+ touch_file(default_old2, fname);
+ std::string expected_old = "file://" + default_old2 + "/" + fname;
+ EXPECT_EQ(expected_old,
+ path_util::get_real_plugin_url(fname, default_new2, plugin_name,
home2));
+
+ // When using a custom configured dir (not equal to default new path)
+ std::string custom_dir = home + "/custom";
+ touch_file(custom_dir, fname);
+ std::string expected_custom = "file://" + custom_dir + "/" + fname;
+ EXPECT_EQ(expected_custom,
+ path_util::get_real_plugin_url(fname, custom_dir, plugin_name,
home));
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
