This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
The following commit(s) were added to refs/heads/main by this push:
new 9d7a3e1fe feat(portal): de-Thrift the file upload/write path to gRPC
(Track D, D4.2) (#187)
9d7a3e1fe is described below
commit 9d7a3e1fe7a6db13930afa116fc1ae72e7e68565
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Tue Jun 9 01:34:48 2026 -0400
feat(portal): de-Thrift the file upload/write path to gRPC (Track D, D4.2)
(#187)
Repoint the upload/write data-product views from the legacy
airavata_django_portal_sdk user_storage helpers to the gRPC storage +
research
facades:
- upload_input_file / tus_upload_finish: new _storage_upload_and_register
helper
uploads bytes via storage.upload_file (full ~/-prefixed path under the
input
staging dir 'tmp') then registers the data product via
research.register_data_product, returning the registered product adapted
to
the DataProductSerializer shape. Replaces user_storage.save_input_file.
- DataProductView.put (fileContentText): overwrite the file in place via
storage.upload_file at the replica's path. Replaces
user_storage.update_data_product_content.
- grpc_requests.data_product_for_upload builds the proto DataProductModel
(FILE type, single GATEWAY_DATA_STORE/TRANSIENT replica, mime-type
metadata,
product size) the way the legacy SDK's _create_data_product did.
The REST/JSON contract to the Vue frontend is unchanged (same
DataProductSerializer output). Listing/delete in UserStoragePathView /
ExperimentStoragePathView and the create_user_storage_dir signal remain on
user_storage and migrate in D4.3.
Depends on apache/airavata#651 (data-product register/read round-trip).
Validation: manage.py check clean; live end-to-end against the tilt backend
—
upload via the view helper registers a product with its replica, the
DataProductSerializer renders productUri/downloadURL/isInputFileUpload=True/
filesize/userHasWriteAccess, and DataProductView.put overwrites the bytes
(download returns the new content).
---
.../django_airavata/apps/api/grpc_requests.py | 30 +++++++++++
.../django_airavata/apps/api/views.py | 62 +++++++++++++++++++---
2 files changed, 84 insertions(+), 8 deletions(-)
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
index 3bc156d4c..3a5704d0c 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
@@ -555,3 +555,33 @@ def group(t):
members=list(t.members or []),
admins=list(t.admins or []),
)
+
+
+def data_product_for_upload(*, gateway_id, owner_name, product_name, file_path,
+ storage_resource_id, content_type=None,
product_size=0):
+ """Build a proto ``DataProductModel`` to register for a freshly uploaded
file.
+
+ The gRPC ``storage.upload_file`` only transfers the bytes and returns a
+ minimal ``DataProductModel``; the portal registers the full data product
via
+ ``research.register_data_product`` so the file gets a canonical product
URI.
+ Mirrors the legacy ``user_storage._create_data_product`` shape: a single
+ GATEWAY_DATA_STORE / TRANSIENT replica pointing at ``file_path``, with the
+ content type recorded under ``mime-type`` metadata.
+ """
+ rc = _pb2("data.replica.replica_catalog_pb2")
+ product_metadata = {"mime-type": content_type} if content_type else {}
+ return rc.DataProductModel(
+ gateway_id=gateway_id,
+ owner_name=owner_name,
+ product_name=product_name,
+ data_product_type=rc.DataProductType.FILE,
+ product_size=product_size or 0,
+ product_metadata=product_metadata,
+ replica_locations=[rc.DataReplicaLocationModel(
+ replica_name="{} gateway data store copy".format(product_name),
+
replica_location_category=rc.ReplicaLocationCategory.GATEWAY_DATA_STORE,
+ replica_persistent_type=rc.ReplicaPersistentType.TRANSIENT,
+ storage_resource_id=storage_resource_id or '',
+ file_path=file_path,
+ )],
+ )
diff --git a/airavata-django-portal/django_airavata/apps/api/views.py
b/airavata-django-portal/django_airavata/apps/api/views.py
index c08c15beb..fd49b2a54 100644
--- a/airavata-django-portal/django_airavata/apps/api/views.py
+++ b/airavata-django-portal/django_airavata/apps/api/views.py
@@ -80,9 +80,49 @@ from . import (
READ_PERMISSION_TYPE = '{}:READ'
+# Input files uploaded for an experiment are staged under this directory in the
+# user's storage (mirrors the legacy SDK's TMP_INPUT_FILE_UPLOAD_DIR).
+TMP_INPUT_FILE_UPLOAD_DIR = "tmp"
+
log = logging.getLogger(__name__)
+def _storage_upload_and_register(request, dir_path, uploaded_file, name=None,
+ content_type=None):
+ """Upload a file to user storage and register a data product for it (gRPC).
+
+ Writes the bytes via the ``storage`` facade (the path is the full file
path,
+ ``~/``-prefixed so the backend resolves it against the storage root), then
+ registers a data product via the ``research`` facade so the file has a
+ canonical product URI. Returns the registered data product adapted to the
+ ``DataProductSerializer`` shape. Replaces the legacy
+ ``user_storage.save``/``save_input_file`` (which transferred bytes and
+ registered the data product in one call).
+ """
+ storage = request.airavata.storage
+ name = name or os.path.basename(getattr(uploaded_file, 'name', '') or '')
+ # Full file path, ~/-prefixed so resolvePath expands it to the storage
root.
+ upload_path = "~/" + os.path.join(dir_path, name).lstrip("/")
+ content = uploaded_file.read()
+ storage.upload_file(
+ path=upload_path, content=content, name=name,
+ content_type=content_type or '')
+ # The upload response is minimal; resolve the absolute path the backend
wrote
+ # to and register the full data product.
+ metadata = storage.get_file_metadata(upload_path)
+ product_uri = request.airavata.research.register_data_product(
+ grpc_requests.data_product_for_upload(
+ gateway_id=settings.GATEWAY_ID,
+ owner_name=request.user.username,
+ product_name=name,
+ file_path=metadata.path,
+ storage_resource_id=storage.get_default_storage_resource_id(),
+ content_type=content_type,
+ product_size=metadata.size))
+ return grpc_adapters.data_product(
+ request.airavata.research.get_data_product(product_uri))
+
+
class GroupViewSet(APIBackedViewSet):
serializer_class = serializers.GroupSerializer
lookup_field = 'group_id'
@@ -875,10 +915,14 @@ class DataProductView(APIView):
data_product = grpc_adapters.data_product(
request.airavata.research.get_data_product(data_product_uri))
if request.data and "fileContentText" in request.data:
- user_storage.update_data_product_content(
- request=request,
- data_product=data_product,
- fileContentText=request.data["fileContentText"])
+ file_path = grpc_adapters.data_product_file_path(data_product)
+ if file_path is None:
+ return Response(status=status.HTTP_400_BAD_REQUEST)
+ # Overwrite the file content in place at the replica's path.
+ request.airavata.storage.upload_file(
+ path=file_path,
+ content=request.data["fileContentText"].encode("utf-8"),
+ name=data_product.productName or os.path.basename(file_path))
return self.get(request=request, format=format)
else:
return Response(status=status.HTTP_400_BAD_REQUEST)
@@ -888,8 +932,9 @@ class DataProductView(APIView):
def upload_input_file(request):
try:
input_file = request.FILES['file']
- data_product = user_storage.save_input_file(
- request, input_file, content_type=input_file.content_type)
+ data_product = _storage_upload_and_register(
+ request, TMP_INPUT_FILE_UPLOAD_DIR, input_file,
+ content_type=input_file.content_type)
serializer = serializers.DataProductSerializer(
data_product, context={'request': request})
return JsonResponse({'uploaded': True,
@@ -907,8 +952,9 @@ def tus_upload_finish(request):
def save_upload(file_path, file_name, file_type):
with open(file_path, 'rb') as uploaded_file:
- return user_storage.save_input_file(request, uploaded_file,
- name=file_name,
content_type=file_type)
+ return _storage_upload_and_register(
+ request, TMP_INPUT_FILE_UPLOAD_DIR, uploaded_file,
+ name=file_name, content_type=file_type)
try:
data_product = tus.save_tus_upload(uploadURL, save_upload)
serializer = serializers.DataProductSerializer(
