This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
The following commit(s) were added to refs/heads/main by this push:
new 691879c8f feat(portal): de-Thrift user/experiment storage listing to
gRPC (Track D, D4.3) (#190)
691879c8f is described below
commit 691879c8f7b951f482f3a471bd5485fabf209f00
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Tue Jun 9 02:26:38 2026 -0400
feat(portal): de-Thrift user/experiment storage listing to gRPC (Track D,
D4.3) (#190)
Repoint the storage path views and the create-user-storage-dir signal from
the
legacy airavata_django_portal_sdk user_storage helpers to the gRPC storage
facade:
- UserStoragePathView get/post/put/delete + ExperimentStoragePathView →
storage.dir_exists/list_dir/get_file_metadata/create_dir/delete_dir/delete_file
(+ _storage_upload_and_register for uploads). New _user_storage_path
resolves a
request path to the absolute ~/-prefixed path the facade expects,
including
experiment-relative paths (via the experiment data dir).
- New grpc_adapters.user_storage_file / user_storage_directory map the gRPC
FileMetadataResponse
(name/path/size/modified_time/content_type/data_product_uri,
populated server-side by apache/airavata#652) to the dicts the storage
serializers read.
- UserStorageFileSerializer.downloadURL now builds the portal download-file
URL
directly from the file's data product URI;
UserHasWriteAccessToPathSerializer
drops the remote-API user_storage.listdir branch (the gRPC path talks to
the
backend directly, not a remote portal proxy).
- signals.create_user_storage_dir → storage.create_dir/create_symlink.
- The deprecated download_file view redirects to the gRPC download-file
endpoint.
- Drop the now-unused user_storage (and queue_settings_calculators) imports
from
views.py/serializers.py.
REST/JSON contract unchanged. Depends on apache/airavata#652 (per-file
modified-time/content-type/data-product-uri in the listing + SFTP file
delete).
Validated live against the running backend: UserStoragePathView GET lists
~/tmp
(200) with each file rendering name/size/createdTime/modifiedTime/mimeType/
dataProductURI/downloadURL/userHasWriteAccess; upload (create dir +
register),
content overwrite, and DELETE (204, file removed) all round-trip; manage.py
check
clean.
---
.../django_airavata/apps/api/grpc_adapters.py | 45 +++++++
.../django_airavata/apps/api/serializers.py | 39 ++----
.../django_airavata/apps/api/signals.py | 11 +-
.../django_airavata/apps/api/views.py | 146 ++++++++++++++-------
4 files changed, 160 insertions(+), 81 deletions(-)
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
index 9d9111946..29b08f85c 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
@@ -1159,3 +1159,48 @@ def grid_ftp_data_movement(pb):
'SECURITY_PROTOCOL_'),
gridFTPEndPoints=list(pb.grid_ftp_end_points),
)
+
+
+# --- User storage file/directory listings -----------------------------------
+# The storage serializers (UserStorageFileSerializer /
UserStorageDirectorySerializer)
+# read plain dicts keyed the way the legacy user_storage.listdir produced. The
gRPC
+# FileMetadataResponse carries name/path/size/modified_time(epoch
ms)/content_type/
+# data_product_uri; map it to that dict shape. ``relative_path`` overrides the
path the
+# serializer reports (e.g. experiment-dir listings expose a path relative to
the data dir).
+
+def _epoch_millis_to_datetime(value):
+ """Epoch milliseconds -> aware UTC datetime, or None when unset (0)."""
+ if not value:
+ return None
+ import datetime
+ return datetime.datetime.fromtimestamp(value / 1000,
tz=datetime.timezone.utc)
+
+
+def user_storage_file(pb, relative_path=None):
+ """gRPC ``FileMetadataResponse`` (a file) -> UserStorageFileSerializer
dict."""
+ modified = _epoch_millis_to_datetime(pb.modified_time)
+ return {
+ 'name': pb.name,
+ 'path': relative_path if relative_path is not None else pb.path,
+ 'data-product-uri': pb.data_product_uri or None,
+ # The adaptor exposes only a modified time; reuse it for created time
so
+ # the serializer's createdTime/modifiedTime both render (SFTP has no
ctime).
+ 'created_time': modified,
+ 'modified_time': modified,
+ 'mime_type': pb.content_type or None,
+ 'size': pb.size,
+ 'hidden': False,
+ }
+
+
+def user_storage_directory(pb, relative_path=None):
+ """gRPC ``FileMetadataResponse`` (a directory) ->
UserStorageDirectorySerializer dict."""
+ modified = _epoch_millis_to_datetime(pb.modified_time)
+ return {
+ 'name': pb.name,
+ 'path': relative_path if relative_path is not None else pb.path,
+ 'created_time': modified,
+ 'modified_time': modified,
+ 'size': pb.size,
+ 'hidden': False,
+ }
diff --git a/airavata-django-portal/django_airavata/apps/api/serializers.py
b/airavata-django-portal/django_airavata/apps/api/serializers.py
index 9d22aa40f..257e7ce35 100644
--- a/airavata-django-portal/django_airavata/apps/api/serializers.py
+++ b/airavata-django-portal/django_airavata/apps/api/serializers.py
@@ -67,9 +67,7 @@ from airavata.model.workspace.ttypes import (
Project
)
from airavata_django_portal_sdk import (
- experiment_util,
- queue_settings_calculators,
- user_storage
+ experiment_util
)
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -1871,26 +1869,8 @@ class
UserHasWriteAccessToPathSerializer(serializers.Serializer):
def get_userHasWriteAccess(self, instance):
request = self.context['request']
- # Special handling when using remote API to access user data storage
- if hasattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API'):
- if "userHasWriteAccess" in instance:
- return instance["userHasWriteAccess"]
- elif instance.get("isDir", False):
- path = Path(instance.get("path", ""))
- if path != Path(""):
- # get parent directory listing and use that to figure out
if
- # there is write access to this directory
- directories, _ = user_storage.listdir(request, path.parent)
- for d in directories:
- if Path(d["path"]) == path:
- return d.get("userHasWriteAccess", False)
- return False
- else:
- # User always has write access on home directory
- return True
- else:
- return False
-
+ if "userHasWriteAccess" in instance:
+ return instance["userHasWriteAccess"]
is_shared_path = view_utils.is_shared_path(instance["path"])
if is_shared_path:
return request.is_gateway_admin
@@ -1909,9 +1889,18 @@ class
UserStorageFileSerializer(UserHasWriteAccessToPathSerializer):
hidden = serializers.BooleanField()
def get_downloadURL(self, file):
- """Getter for downloadURL field."""
+ """Lazy portal URL to the byte-streaming download endpoint for this
file.
+
+ Returns None when the file has no data product URI; resolving the
bytes is
+ deferred to the endpoint, so this getter makes no backend call.
+ """
request = self.context['request']
- return user_storage.get_lazy_download_url(request,
data_product_uri=file['data-product-uri'])
+ data_product_uri = file.get('data-product-uri')
+ if not data_product_uri:
+ return None
+ base = request.build_absolute_uri(
+ reverse('django_airavata_api:download-file'))
+ return base + '?data-product-uri=' + quote(data_product_uri)
class UserStorageDirectorySerializer(UserHasWriteAccessToPathSerializer):
diff --git a/airavata-django-portal/django_airavata/apps/api/signals.py
b/airavata-django-portal/django_airavata/apps/api/signals.py
index ceab29b86..62d2a4363 100644
--- a/airavata-django-portal/django_airavata/apps/api/signals.py
+++ b/airavata-django-portal/django_airavata/apps/api/signals.py
@@ -2,7 +2,6 @@
import logging
-from airavata_django_portal_sdk import user_storage
from django.conf import settings
from django.contrib.auth.signals import user_logged_in
from django.dispatch import Signal, receiver
@@ -18,12 +17,12 @@ user_added_to_group = Signal()
# Receivers
@receiver(user_logged_in)
def create_user_storage_dir(sender, request, user, **kwargs):
- """Create user's home direct in gateway storage."""
- path = ""
- if not user_storage.dir_exists(request, path):
- user_storage.create_user_dir(request, path)
+ """Create user's home directory in gateway storage (gRPC storage
facade)."""
+ storage = request.airavata.storage
+ if not storage.dir_exists("~/"):
+ storage.create_dir("~/")
log.info("Created home directory for user {}".format(user.username))
if hasattr(settings, 'GATEWAY_DATA_SHARED_DIRECTORIES'):
for name, entry in settings.GATEWAY_DATA_SHARED_DIRECTORIES.items():
- user_storage.create_symlink(request, entry['path'], name)
+ storage.create_symlink(entry['path'], "~/" + name)
diff --git a/airavata-django-portal/django_airavata/apps/api/views.py
b/airavata-django-portal/django_airavata/apps/api/views.py
index fd49b2a54..79a2ef19c 100644
--- a/airavata-django-portal/django_airavata/apps/api/views.py
+++ b/airavata-django-portal/django_airavata/apps/api/views.py
@@ -5,6 +5,7 @@ import logging
import os
import warnings
from datetime import datetime, timedelta
+from urllib.parse import quote
from airavata.model.appcatalog.computeresource.ttypes import (
CloudJobSubmission,
@@ -33,8 +34,7 @@ from airavata.model.group.ttypes import ResourcePermissionType
from airavata.model.user.ttypes import Status
from airavata_django_portal_sdk import (
experiment_util,
- queue_settings_calculators,
- user_storage
+ queue_settings_calculators
)
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -88,11 +88,12 @@ log = logging.getLogger(__name__)
def _storage_upload_and_register(request, dir_path, uploaded_file, name=None,
- content_type=None):
+ content_type=None, experiment_id=None):
"""Upload a file to user storage and register a data product for it (gRPC).
Writes the bytes via the ``storage`` facade (the path is the full file
path,
- ``~/``-prefixed so the backend resolves it against the storage root), then
+ ``~/``-prefixed so the backend resolves it against the storage root, or
+ relative to the experiment data dir when ``experiment_id`` is given), then
registers a data product via the ``research`` facade so the file has a
canonical product URI. Returns the registered data product adapted to the
``DataProductSerializer`` shape. Replaces the legacy
@@ -101,8 +102,9 @@ def _storage_upload_and_register(request, dir_path,
uploaded_file, name=None,
"""
storage = request.airavata.storage
name = name or os.path.basename(getattr(uploaded_file, 'name', '') or '')
- # Full file path, ~/-prefixed so resolvePath expands it to the storage
root.
- upload_path = "~/" + os.path.join(dir_path, name).lstrip("/")
+ # Full file path resolved against the storage root (or experiment data
dir).
+ upload_path = _user_storage_path(
+ os.path.join(dir_path, name), experiment_id, request)
content = uploaded_file.read()
storage.upload_file(
path=upload_path, content=content, name=name,
@@ -969,10 +971,12 @@ def tus_upload_finish(request):
@api_view()
def download_file(request):
# TODO: remove this deprecated view
- warnings.warn("download_file view has moved to SDK", DeprecationWarning)
- # redirect to /sdk/download
+ warnings.warn("download_file view is deprecated; use 'download-file'",
DeprecationWarning)
+ # Redirect to the gRPC byte-streaming download endpoint.
data_product_uri = request.GET.get('data-product-uri', '')
- return redirect(user_storage.get_download_url(request,
data_product_uri=data_product_uri))
+ return redirect(
+
request.build_absolute_uri(reverse('django_airavata_api:download-file'))
+ + '?data-product-uri=' + quote(data_product_uri))
@api_view()
@@ -1659,6 +1663,28 @@ class ParserViewSet(mixins.CreateModelMixin,
self.request.airavata.research.save_parser(grpc_requests.parser(parser))
+def _user_storage_path(path, experiment_id=None, request=None):
+ """Resolve a user-storage path to the absolute, ``~/``-prefixed path the
gRPC
+ storage facade expects.
+
+ A bare relative path is taken relative to the user's storage root (``~/``).
+ When ``experiment_id`` is given, the path is relative to that experiment's
+ data directory (resolved via the experiment's userConfigurationData).
+ """
+ rel = (path or "").lstrip("/")
+ if experiment_id:
+ experiment = grpc_adapters.experiment(
+ request.airavata.research.get_experiment(experiment_id))
+ data_dir = (experiment.userConfigurationData.experimentDataDir
+ if experiment.userConfigurationData else None) or ""
+ base = data_dir.rstrip("/")
+ full = base + ("/" + rel if rel else "")
+ return full if (full.startswith("/") or full.startswith("~/")) else
"~/" + full
+ if rel.startswith("~"):
+ return rel
+ return "~/" + rel
+
+
class UserStoragePathView(APIView):
serializer_class = serializers.UserStoragePathSerializer
permission_classes = (IsAuthenticated, UserStorageSharedDirPermission)
@@ -1672,19 +1698,18 @@ class UserStoragePathView(APIView):
def post(self, request, path="/", format=None):
path = request.data.get('path', path)
experiment_id = request.data.get('experiment-id')
- if not user_storage.dir_exists(request, path,
experiment_id=experiment_id):
- _, resource_path = user_storage.create_user_dir(request, path,
experiment_id=experiment_id)
- # create_user_dir may create the directory with a different name
- # than requested, for example, converting spaces to underscores, so
- # use as the path the path that is returned by create_user_dir
- path = resource_path
+ storage = request.airavata.storage
+ resolved = _user_storage_path(path, experiment_id, request)
+ if not storage.dir_exists(resolved):
+ storage.create_dir(resolved)
data_product = None
# Handle direct upload
if 'file' in request.FILES:
user_file = request.FILES['file']
- data_product = user_storage.save(
- request, path, user_file, content_type=user_file.content_type,
+ data_product = _storage_upload_and_register(
+ request, path, user_file, name=user_file.name,
+ content_type=user_file.content_type,
experiment_id=experiment_id)
# Handle a tus upload
elif 'uploadURL' in request.POST:
@@ -1692,26 +1717,26 @@ class UserStoragePathView(APIView):
def save_file(file_path, file_name, file_type):
with open(file_path, 'rb') as uploaded_file:
- return user_storage.save(request, path, uploaded_file,
- name=file_name,
content_type=file_type,
- experiment_id=experiment_id)
+ return _storage_upload_and_register(
+ request, path, uploaded_file, name=file_name,
+ content_type=file_type, experiment_id=experiment_id)
data_product = tus.save_tus_upload(uploadURL, save_file)
return self._create_response(request, path, uploaded=data_product,
experiment_id=experiment_id)
- # Accept wither to replace file or to replace file content text.
+ # Accept either to replace file or to replace file content text.
def put(self, request, path="/", format=None):
path = request.POST.get('path', path)
# Replace the file if the request has a file upload.
if 'file' in request.FILES:
self.delete(request=request, path=path, format=format)
dir_path, file_name = os.path.split(path)
- self.post(request=request, path=dir_path, format=format,
file_name=file_name)
+ self.post(request=request, path=dir_path, format=format)
# Replace only the file content if the request body has the
`fileContentText`
elif request.data and "fileContentText" in request.data:
- user_storage.update_file_content(
- request=request,
- path=path,
- fileContentText=request.data["fileContentText"])
+ request.airavata.storage.upload_file(
+ path=_user_storage_path(path),
+ content=request.data["fileContentText"].encode("utf-8"),
+ name=os.path.basename(path))
else:
return Response(status=status.HTTP_400_BAD_REQUEST)
@@ -1720,20 +1745,25 @@ class UserStoragePathView(APIView):
def delete(self, request, path="/", format=None):
path = request.data.get('path', path)
experiment_id = request.data.get('experiment-id')
- if user_storage.dir_exists(request, path, experiment_id=experiment_id):
- user_storage.delete_dir(request, path, experiment_id=experiment_id)
+ storage = request.airavata.storage
+ resolved = _user_storage_path(path, experiment_id, request)
+ if storage.dir_exists(resolved):
+ storage.delete_dir(resolved)
else:
- user_storage.delete_user_file(request, path,
experiment_id=experiment_id)
+ storage.delete_file(resolved)
return Response(status=204)
def _create_response(self, request, path, uploaded=None,
experiment_id=None):
- if user_storage.dir_exists(request, path, experiment_id=experiment_id):
- directories, files = user_storage.listdir(request, path,
experiment_id=experiment_id)
+ storage = request.airavata.storage
+ resolved = _user_storage_path(path, experiment_id, request)
+ if storage.dir_exists(resolved):
+ listing = storage.list_dir(resolved)
data = {
'isDir': True,
- 'directories': directories,
- 'files': files
+ 'directories': [
+ grpc_adapters.user_storage_directory(d) for d in
listing.directories],
+ 'files': [grpc_adapters.user_storage_file(f) for f in
listing.files],
}
if uploaded is not None:
data['uploaded'] = uploaded
@@ -1743,7 +1773,7 @@ class UserStoragePathView(APIView):
data, context={'request': request})
return Response(serializer.data)
else:
- file = user_storage.get_file_metadata(request, path,
experiment_id=experiment_id)
+ file =
grpc_adapters.user_storage_file(storage.get_file_metadata(resolved))
data = {
'isDir': False,
'directories': [],
@@ -1774,23 +1804,39 @@ class ExperimentStoragePathView(APIView):
return self._create_response(request, experiment_id, path)
def _create_response(self, request, experiment_id, path):
- if user_storage.experiment_dir_exists(request, experiment_id, path):
- directories, files = user_storage.list_experiment_dir(request,
experiment_id, path)
-
- def add_expid(d):
- d['experiment_id'] = experiment_id
- return d
- data = {
- 'isDir': True,
- 'directories': map(add_expid, directories),
- 'files': map(add_expid, files)
- }
- data['parts'] = self._split_path(path)
- serializer = self.serializer_class(
- data, context={'request': request})
- return Response(serializer.data)
- else:
+ storage = request.airavata.storage
+ resolved = _user_storage_path(path, experiment_id, request)
+ if not storage.dir_exists(resolved):
raise Http404(f"Path '{path}' does not exist for {experiment_id}")
+ listing = storage.list_dir(resolved)
+
+ def rel(entry_path):
+ # Expose the path relative to the experiment data dir, as the
legacy
+ # list_experiment_dir did (resolved is the absolute experiment
path).
+ base = resolved.rstrip("/")
+ p = entry_path
+ if p.startswith(base + "/"):
+ return p[len(base) + 1:]
+ return os.path.basename(p)
+
+ def add_expid(d):
+ d['experiment_id'] = experiment_id
+ return d
+ data = {
+ 'isDir': True,
+ 'directories': [
+ add_expid(grpc_adapters.user_storage_directory(
+ d, relative_path=os.path.join(path, rel(d.path)) if path
else rel(d.path)))
+ for d in listing.directories],
+ 'files': [
+ add_expid(grpc_adapters.user_storage_file(
+ f, relative_path=os.path.join(path, rel(f.path)) if path
else rel(f.path)))
+ for f in listing.files],
+ }
+ data['parts'] = self._split_path(path)
+ serializer = self.serializer_class(
+ data, context={'request': request})
+ return Response(serializer.data)
def _split_path(self, path):
head, tail = os.path.split(path)
