This is an automated email from the ASF dual-hosted git repository.
zclll pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a680f5fca0 [Fix](pythonUdf) Fix pythonUdf helper cmd coredump when
`enable_python_udf_support` is not set (#61196)
2a680f5fca0 is described below
commit 2a680f5fca0ed4c490558ad542beeff221be6bea
Author: linrrarity <[email protected]>
AuthorDate: Thu Mar 12 16:41:01 2026 +0800
[Fix](pythonUdf) Fix pythonUdf helper cmd coredump when
`enable_python_udf_support` is not set (#61196)
When `enable_python_udf_support` is not set to true, a null pointer
dereference occurs when invoking helper commands (`SHOW PYTHON VERSION/
PACKAGES IN`).
```text
/home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38:
runtime error: member call on null pointer of type 'doris::PythonEnvScanner'
#0 0x559edede162c in doris::PythonVersionManager::env_infos_to_thrift()
const
/home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38
#1 0x559ede63acd4 in
doris::BaseBackendService::get_python_envs(std::vector>&)
/home/zcp/repo_center/doris_master/doris/be/src/service/backend_service.cpp:1316:47
#2 0x559edf99796a in
doris::BackendServiceProcessor::process_get_python_envs(int,
apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*,
void*)
/home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/BackendService.cpp:7789:13
#3 0x559edf94a69c in
doris::BackendServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*,
apache::thrift::protocol::TProtocol*, std::__cxx11::basic_string,
std::allocator> const&, int, void*)
/home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/BackendService.cpp:6466:3
#4 0x559eac6bad06 in
apache::thrift::TDispatchProcessor::process(std::shared_ptr, std::shared_ptr,
void*)
/home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/thrift/TDispatchProcessor.h:121:12
#5 0x559ee47c071a in apache::thrift::server::TConnectedClient::run()
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1c871a)
#6 0x559ee47c1a45 in
apache::thrift::server::TThreadedServer::TConnectedClientRunner::run()
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1c9a45)
#7 0x559ee47c557f in
apache::thrift::concurrency::Thread::threadMain(std::shared_ptr)
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd57f)
#8 0x559ee47c52a5 in void std::__invoke_impl),
std::shared_ptr>(std::__invoke_other, void (*&&)(std::shared_ptr),
std::shared_ptr&&)
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd2a5)
#9 0x559ee47c521c in std::__invoke_result), std::shared_ptr>::type
std::__invoke), std::shared_ptr>(void (*&&)(std::shared_ptr),
std::shared_ptr&&)
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd21c)
#10 0x559ee47c51f1 in void std::thread::_Invoker),
std::shared_ptr>>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd1f1)
#11 0x559ee47c51b4 in std::thread::_Invoker),
std::shared_ptr>>::operator()()
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd1b4)
#12 0x559ee47c4fd8 in std::thread::_State_impl),
std::shared_ptr>>>::_M_run()
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1ccfd8)
#13 0x559ef61b5f3f in execute_native_thread_routine archive64.c
#14 0x559eac535d26 in asan_thread_start(void*)
(/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x51f3dd26)
#15 0x7fcba62bfac2 in start_thread nptl/pthread_create.c:442:8
#16 0x7fcba635184f misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior
/home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38
```
now:
```text
Doris> show python packages in '3.19.18';
ERROR 1105 (HY000): errCode = 2, detailMessage = Failed to get python
packages from any backend: [E42] Set 'python_venv_interpreter_paths' in be.conf
to enable PythonUDF feature
Doris> show python versions;
ERROR 1105 (HY000): errCode = 2, detailMessage = Failed to get python envs
from any backend: [E-236] Set 'python_venv_interpreter_paths' in be.conf to
enable PythonUDF feature
```
---
be/src/udf/python/python_env.cpp | 8 ++---
be/src/udf/python/python_env.h | 36 ++++++++++++++++++----
.../plans/commands/ShowPythonPackagesCommand.java | 9 ++----
.../plans/commands/ShowPythonVersionsCommand.java | 8 +----
4 files changed, 37 insertions(+), 24 deletions(-)
diff --git a/be/src/udf/python/python_env.cpp b/be/src/udf/python/python_env.cpp
index 7fea4aea312..ef7eb77eb7a 100644
--- a/be/src/udf/python/python_env.cpp
+++ b/be/src/udf/python/python_env.cpp
@@ -267,8 +267,8 @@ std::string VenvEnvScanner::to_string() const {
return ss.str();
}
-Status PythonVersionManager::init(PythonEnvType env_type, const fs::path&
python_root_path,
- const std::string&
python_venv_interpreter_paths) {
+Status PythonEnvScannerHolder::init(PythonEnvType env_type, const fs::path&
python_root_path,
+ const std::string&
python_venv_interpreter_paths) {
switch (env_type) {
case PythonEnvType::CONDA: {
if (!fs::exists(python_root_path) ||
!fs::is_directory(python_root_path)) {
@@ -302,10 +302,10 @@ Status PythonVersionManager::init(PythonEnvType env_type,
const fs::path& python
std::vector<TPythonEnvInfo> PythonVersionManager::env_infos_to_thrift() const {
std::vector<TPythonEnvInfo> infos;
- const auto& envs = _env_scanner->get_envs();
+ const auto& envs = this->get_envs();
infos.reserve(envs.size());
- const auto env_type_str =
_python_env_type_to_string(_env_scanner->env_type());
+ const auto env_type_str = _python_env_type_to_string(this->env_type());
for (const auto& env : envs) {
TPythonEnvInfo info;
info.__set_env_name(env.env_name);
diff --git a/be/src/udf/python/python_env.h b/be/src/udf/python/python_env.h
index 897398b815f..89f6f67a252 100644
--- a/be/src/udf/python/python_env.h
+++ b/be/src/udf/python/python_env.h
@@ -22,6 +22,7 @@
#include <filesystem>
#include <utility>
+#include "common/exception.h"
#include "common/status.h"
namespace doris {
@@ -137,6 +138,27 @@ private:
std::vector<std::string> _interpreter_paths;
};
+// Holds a PythonEnvScanner instance and centralizes the initialization check
+// to avoid accessing the scanner when the Python UDF feature is disabled.
+// This class is intended for internal use by PythonVersionManager only.
+class PythonEnvScannerHolder {
+public:
+ Status init(PythonEnvType env_type, const fs::path& python_root_path,
+ const std::string& python_venv_interpreter_paths);
+
+ const PythonEnvScanner& get() const {
+ if (!_env_scanner) {
+ throw Exception(ErrorCode::NOT_INITIALIZED,
+ "Set 'enable_python_udf_support = true' in be.conf
to enable PythonUDF "
+ "feature");
+ }
+ return *_env_scanner;
+ }
+
+private:
+ std::unique_ptr<PythonEnvScanner> _env_scanner;
+};
+
class PythonVersionManager {
public:
static PythonVersionManager& instance() {
@@ -145,17 +167,19 @@ public:
}
Status init(PythonEnvType env_type, const fs::path& python_root_path,
- const std::string& python_venv_interpreter_paths);
+ const std::string& python_venv_interpreter_paths) {
+ return _holder.init(env_type, python_root_path,
python_venv_interpreter_paths);
+ }
Status get_version(const std::string& runtime_version, PythonVersion*
version) const {
- return _env_scanner->get_version(runtime_version, version);
+ return _holder.get().get_version(runtime_version, version);
}
- const std::vector<PythonEnvironment>& get_envs() const { return
_env_scanner->get_envs(); }
+ const std::vector<PythonEnvironment>& get_envs() const { return
_holder.get().get_envs(); }
- PythonEnvType env_type() const { return _env_scanner->env_type(); }
+ PythonEnvType env_type() const { return _holder.get().env_type(); }
- std::string to_string() const { return _env_scanner->to_string(); }
+ std::string to_string() const { return _holder.get().to_string(); }
std::vector<TPythonEnvInfo> env_infos_to_thrift() const;
@@ -163,7 +187,7 @@ public:
const std::vector<std::pair<std::string, std::string>>& packages)
const;
private:
- std::unique_ptr<PythonEnvScanner> _env_scanner;
+ PythonEnvScannerHolder _holder;
};
// List installed pip packages for a given Python version.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java
index f4b5ad8f99a..2cb49271a94 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -37,8 +38,6 @@ import org.apache.doris.thrift.TPythonPackageInfo;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashMap;
@@ -50,8 +49,6 @@ import java.util.Map;
* Shows pip packages installed for the given Python version, collected from
all alive BEs.
*/
public class ShowPythonPackagesCommand extends ShowCommand {
- private static final Logger LOG =
LogManager.getLogger(ShowPythonPackagesCommand.class);
-
private static final String[] TITLE_NAMES = {"Package", "Version"};
private static final String[] TITLE_NAMES_INCONSISTENT = {"Package",
"Version", "Consistent", "Backends"};
@@ -116,8 +113,6 @@ public class ShowPythonPackagesCommand extends ShowCommand {
}
allBePackages.add(pkgMap);
beIdentifiers.add(backend.getHost() + ":" +
backend.getBePort());
- } catch (Exception e) {
- LOG.warn("Failed to get python packages from backend[{}]",
backend.getId(), e);
} finally {
if (ok) {
ClientPool.backendPool.returnObject(address, client);
@@ -128,7 +123,7 @@ public class ShowPythonPackagesCommand extends ShowCommand {
}
if (allBePackages.isEmpty()) {
- return new ShowResultSet(getMetaData(), Lists.newArrayList());
+ throw new AnalysisException("No alive backends found to get python
packages");
}
// Check consistency across BEs
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java
index 16fab957bce..102a92b7131 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java
@@ -37,8 +37,6 @@ import org.apache.doris.thrift.TPythonEnvInfo;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashSet;
@@ -50,8 +48,6 @@ import java.util.Set;
* Shows Python versions available on all alive backends (intersection).
*/
public class ShowPythonVersionsCommand extends ShowCommand {
- private static final Logger LOG =
LogManager.getLogger(ShowPythonVersionsCommand.class);
-
private static final String[] TITLE_NAMES = {
"Version", "EnvName", "EnvType", "BasePath", "ExecutablePath"
};
@@ -108,8 +104,6 @@ public class ShowPythonVersionsCommand extends ShowCommand {
} else {
commonVersions.retainAll(versions);
}
- } catch (Exception e) {
- LOG.warn("Failed to get python envs from backend[{}]",
backend.getId(), e);
} finally {
if (ok) {
ClientPool.backendPool.returnObject(address, client);
@@ -120,7 +114,7 @@ public class ShowPythonVersionsCommand extends ShowCommand {
}
List<List<String>> rows = Lists.newArrayList();
- if (commonVersions != null && !allBeEnvs.isEmpty()) {
+ if (commonVersions != null) {
// Use envs from the first BE as reference, filtered to common
versions
for (TPythonEnvInfo env : allBeEnvs.get(0)) {
if (commonVersions.contains(env.getFullVersion())) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
