yuqi1129 commented on code in PR #7384:
URL: https://github.com/apache/gravitino/pull/7384#discussion_r2153578826
##########
clients/client-python/gravitino/filesystem/gvfs_base_operations.py:
##########
@@ -133,6 +134,14 @@ def __init__(
)
self._current_location_name = self._init_current_location_name()
+ def _register_custom_storage_handlers(self):
+ """Register custom storage handler providers from configuration."""
+ try:
+ from gravitino.filesystem.gvfs_storage_handler import
register_storage_handler_providers
+ register_storage_handler_providers(self._options)
+ except Exception as e:
+ logger.warning(f"Failed to register custom storage handlers:
{str(e)}")
Review Comment:
The indentation here is incorrect.
##########
clients/client-python/gravitino/filesystem/gvfs_storage_handler.py:
##########
@@ -737,3 +740,103 @@ def get_storage_handler_by_path(actual_path: str) ->
StorageHandler:
raise GravitinoRuntimeException(
f"Storage type doesn't support now. Path:{actual_path}"
)
+
+def register_storage_handler_providers(options: Dict[str, str]):
+ """
+ Register storage handler providers from configuration.
+
+ This function dynamically loads and registers storage handler providers
+ specified in the configuration, allowing users to extend GVFS with custom
+ storage systems without modifying core code.
+
+ :param options: Configuration options dictionary
+ """
+ if not options:
+ return
+
+ providers_config = options.get("storage_handler_providers")
+ if not providers_config:
+ return
+
+ # Import here to avoid circular imports
+ from gravitino.filesystem.storage_handler_provider import
StorageHandlerProvider
+
+ for provider_class_name in providers_config.split(','):
+ provider_class_name = provider_class_name.strip()
+ if not provider_class_name:
+ continue
+
+ try:
+ # Dynamic import and instantiation
+ module_name, class_name = provider_class_name.rsplit(".", 1)
+ module = importlib.import_module(module_name)
+ provider_class = getattr(module, class_name)
+
+ # Validate that it's a StorageHandlerProvider
+ if not issubclass(provider_class, StorageHandlerProvider):
+ raise GravitinoRuntimeException(
+ f"Class {provider_class_name} must extend
StorageHandlerProvider"
+ )
+
+ provider = provider_class()
+ storage_handler = provider.get_storage_handler()
+ scheme = provider.scheme()
+
+ # Check for scheme conflicts
+ scheme_enum = None
+ for storage_type in StorageType:
+ if storage_type.value == scheme:
+ scheme_enum = storage_type
+ break
+
+ if scheme_enum is None:
+ # Create a new storage type dynamically for custom schemes
+ # Note: This is a limitation of using Enum, but we can work
around it
+ # by directly adding to the dictionaries
+ pass
+ else:
+ # Check if scheme already exists
+ if scheme_enum in storage_handlers:
+ raise GravitinoRuntimeException(
+ f"Storage handler for scheme '{scheme}' already
exists. "
+ f"Provider: {provider_class_name}"
+ )
+
+ # Register the storage handler
+ if scheme_enum:
+ storage_handlers[scheme_enum] = storage_handler
+ else:
+ # For custom schemes, we need to extend our lookup mechanism
+ # Add to storage_handlers with a custom key approach
+ custom_storage_type = type('CustomStorageType', (), {'value':
scheme})()
+ storage_handlers[custom_storage_type] = storage_handler
Review Comment:
As the class name `custom_storage_type` is constant, do we only support
registering one kind of custom storage handler?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
