github-actions[bot] commented on code in PR #69255:
URL: https://github.com/apache/doris/pull/69255#discussion_r4599310628
##########
be/src/udf/python/python_server.py:
##########
@@ -964,63 +948,105 @@ def load(self) -> AdaptivePythonUDF:
return AdaptivePythonUDF(self.python_udf_meta, func)
+class UDFModuleFinder(importlib.abc.MetaPathFinder):
+ """
+ Imports user modules by their own names from the UDF directory on
sys.path, except a
+ top-level name the server has already imported
(SERVER_TOP_LEVEL_MODULE_NAMES). Such a
+ module, e.g. the user's types.py, is imported as
"__doris_pyudf_<function_id>.types"
+ under a root package whose __path__ is the UDF directory, so the server
keeps its own
+ types module.
+
+ Other modules keep their names, so a UDAF state class from state_helper.py
is still
+ pickled as "state_helper.State" and stays readable by workers of older
versions.
+
+ Absolute imports in user code use the same rule, e.g. "from json import
helper" in a
+ UDF directory containing json/ loads
"__doris_pyudf_<function_id>.json.helper".
+ """
+
+ def __init__(self, function_id: int, location: str):
+ self.root_package = self.root_package_name(function_id)
+ self.root_package_prefix = self.root_package + "."
+ # Match absolute module __file__ paths even when user_function_dir is
relative.
+ self.location = os.path.abspath(location)
+ self.udf_file_prefix = os.path.join(self.location, "")
+ self.original_import = builtins.__import__
+ # Server top-level name -> whether the UDF directory also provides it.
+ self.shadowed_server_module_cache: Dict[str, bool] = {}
+ # importlib.import_module bypasses builtins.__import__, so user code
gets
+ # an importlib whose import_module adds the prefix.
+ self.udf_importlib = ModuleType("importlib")
+ self.udf_importlib.__dict__.update(vars(importlib))
+ self.udf_importlib.import_module = self.import_module_for_udf_code
+ self.udf_importlib.__getattr__ = lambda name: getattr(importlib, name)
+
+ @staticmethod
+ def root_package_name(function_id: int) -> str:
+ return f"__doris_pyudf_{function_id}"
+
+ def install(self) -> None:
+ sys.path.insert(0, self.location)
+ sys.meta_path.insert(0, self)
+ builtins.__import__ = self.import_for_udf_code
+
+ def find_spec(self, fullname, path=None, target=None):
+ # Only creates the root package; PathFinder loads every user module
with its own
+ # loader, through sys.path or the root package __path__.
+ if fullname != self.root_package:
+ return None
+ spec = importlib.machinery.ModuleSpec(fullname, None, is_package=True)
+ spec.submodule_search_locations = [self.location]
+ return spec
+
+ def is_shadowed_server_module(self, top_level_name: str) -> bool:
+ if top_level_name not in SERVER_TOP_LEVEL_MODULE_NAMES:
+ return False
+ shadowed = self.shadowed_server_module_cache.get(top_level_name)
+ if shadowed is None:
+ shadowed = (
+ importlib.machinery.PathFinder.find_spec(top_level_name,
[self.location])
+ is not None
+ )
+ self.shadowed_server_module_cache[top_level_name] = shadowed
+ return shadowed
+
+ def add_root_package_prefix(self, name: str) -> str:
+ if self.is_shadowed_server_module(name.partition(".")[0]):
+ return self.root_package_prefix + name
+ return name
+
+ def import_for_udf_code(self, name, globals=None, locals=None,
fromlist=(), level=0):
+ """builtins.__import__ that adds the prefix to absolute imports in
user code."""
+ if (
+ level != 0
+ or globals is None
Review Comment:
[P2] Route direct dynamic imports through the UDF namespace
Direct calls such as `__import__('types')` [omit `globals` by
default](https://docs.python.org/3.10/library/functions.html#__import__), so
this branch bypasses the UDF routing even when called from user code. A
previously valid `types.py` can define `class State: pass`, then evaluate
`StateType = __import__('types').State` at module scope before defining
`evaluate` (`symbol='types.evaluate'`). At the merge base, the loader evicts
the bare `types` entry and Python registers the user module there during
initialization, so this succeeds. Here the entry module loads as
`__doris_pyudf_<id>.types`, but the dynamic call returns the cached stdlib
`types`, raises `AttributeError`, and prevents the function from loading.
Initial entry-point resolution now succeeds; this is a separate failure in a
subsequent dynamic import inside that module. Preserve caller-aware routing for
direct `__import__` calls with omitted globals and cover this initializer case
alongside ordinary imports.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]