This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
The following commit(s) were added to refs/heads/main by this push:
new 70eee0d19 feat(portal): repoint compute resource reads to gRPC (Track
D, D2) (#168)
70eee0d19 is described below
commit 70eee0d1984d38cbca7ebe92b3f55d2464c71006
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Mon Jun 8 19:43:08 2026 -0400
feat(portal): repoint compute resource reads to gRPC (Track D, D2) (#168)
Migrate ComputeResourceViewSet (get_instance, all_names, all_names_list,
queues) and the previously-deferred ApplicationDeploymentViewSet.queues
action from the Thrift client to the gRPC compute facade. queues reads a
ComputeResource, which is why it moves with this family; the deployment
queues action also re-reads the deployment via research.get_application_
deployment. Write actions stay on Thrift pending D3.
ComputeResourceDescription is the deepest read model. The new
compute_resource adapter recursively adapts batchQueues (BatchQueue),
jobSubmissionInterfaces (JobSubmissionInterface), the reused
dataMovementInterfaces, and the fileSystems map. Notable bridges:
- fileSystems is a proto map<int32, string> whose int key holds a proto
FileSystems value (HOME=1) while Thrift FileSystems is HOME=0; the
adapter converts each key to the Thrift integer by name and keeps them
plain ints so the serializer's DictField renders the same '0'..'4' keys
the Thrift i32-keyed map produced.
- JobSubmissionProtocol needs the explicit _thrift_enum_mapped bridge:
proto JSP_CLOUD maps to Thrift CLOUD (name divergence), the rest align
by name but differ by one in value.
Verified: manage.py check clean; all_names returns 200 live; a seeded
compute resource round-trips through the portal with real data -- batch
queue fields, hostAliases/ipAddresses, and crucially fileSystems
{"0":"/home","3":"/scratch"} (proto HOME=1/SCRATCH=4 bridged to the
Thrift map keys 0/3) all correct; offline render additionally exercises
the JobSubmissionProtocol map including JSP_CLOUD -> CLOUD.
---
.../django_airavata/apps/api/grpc_adapters.py | 106 +++++++++++++++++++++
.../django_airavata/apps/api/views.py | 24 ++---
2 files changed, 118 insertions(+), 12 deletions(-)
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
index 3cfe01802..ca9942a57 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
@@ -13,6 +13,9 @@ serializers are made protobuf-native.
from types import SimpleNamespace
+from airavata.model.appcatalog.computeresource.ttypes import (
+ JobSubmissionProtocol as _ThriftJobSubmissionProtocol,
+)
from airavata.model.appcatalog.parallelism.ttypes import (
ApplicationParallelismType as _ThriftParallelismType,
)
@@ -90,6 +93,109 @@ def storage_resource(pb):
)
+# proto JobSubmissionProtocol member name -> Thrift JobSubmissionProtocol
value.
+# Mostly aligned, but proto JSP_CLOUD maps to Thrift CLOUD (name divergence).
+_JOB_SUBMISSION_PROTOCOL = {
+ 'LOCAL': _ThriftJobSubmissionProtocol.LOCAL,
+ 'SSH': _ThriftJobSubmissionProtocol.SSH,
+ 'GLOBUS': _ThriftJobSubmissionProtocol.GLOBUS,
+ 'UNICORE': _ThriftJobSubmissionProtocol.UNICORE,
+ 'JSP_CLOUD': _ThriftJobSubmissionProtocol.CLOUD,
+ 'SSH_FORK': _ThriftJobSubmissionProtocol.SSH_FORK,
+ 'LOCAL_FORK': _ThriftJobSubmissionProtocol.LOCAL_FORK,
+}
+
+# proto FileSystems int value -> Thrift FileSystems int value (built lazily by
+# name; the proto map key is a bare int32 with no enum descriptor to read).
+_file_systems_proto_to_thrift = None
+
+
+def _file_systems(pb_map):
+ """proto ``file_systems`` map<int32, string> -> {Thrift FileSystems int:
path}.
+
+ The proto map key is a bare int32 holding a proto ``FileSystems`` value;
+ convert each to the Thrift ``FileSystems`` int (by name — proto HOME=1 vs
+ Thrift HOME=0) so the serializer's ``DictField`` renders the same '0'..'4'
+ keys the Thrift i32-keyed map produced. Keys stay plain ints (not IntEnum)
+ so ``DictField``'s ``str(key)`` yields the digit, as Thrift's map did.
+ Unknown keys (e.g. the proto-only zero sentinel) are dropped.
+ """
+ global _file_systems_proto_to_thrift
+ if _file_systems_proto_to_thrift is None:
+ from airavata.model.appcatalog.computeresource.ttypes import
FileSystems
+ from
airavata_sdk.generated.org.apache.airavata.model.appcatalog.computeresource
import ( # noqa: E501
+ compute_resource_pb2,
+ )
+ proto_fs = compute_resource_pb2.FileSystems
+ _file_systems_proto_to_thrift = {
+ proto_fs.Value(name): int(getattr(FileSystems, name))
+ for name in proto_fs.keys() if hasattr(FileSystems, name)
+ }
+ return {
+ _file_systems_proto_to_thrift[k]: v
+ for k, v in pb_map.items() if k in _file_systems_proto_to_thrift
+ }
+
+
+def _batch_queue(pb):
+ """gRPC ``BatchQueue`` -> ``BatchQueueSerializer`` shape (all scalars)."""
+ return SimpleNamespace(
+ queueName=pb.queue_name,
+ queueDescription=pb.queue_description,
+ maxRunTime=pb.max_run_time,
+ maxNodes=pb.max_nodes,
+ maxProcessors=pb.max_processors,
+ maxJobsInQueue=pb.max_jobs_in_queue,
+ maxMemory=pb.max_memory,
+ cpuPerNode=pb.cpu_per_node,
+ defaultNodeCount=pb.default_node_count,
+ defaultCPUCount=pb.default_cpu_count,
+ defaultWalltime=pb.default_walltime,
+ queueSpecificMacros=pb.queue_specific_macros,
+ isDefaultQueue=pb.is_default_queue,
+ )
+
+
+def _job_submission_interface(pb):
+ """gRPC ``JobSubmissionInterface`` -> auto-generated serializer shape."""
+ return SimpleNamespace(
+ jobSubmissionInterfaceId=pb.job_submission_interface_id,
+ jobSubmissionProtocol=_thrift_enum_mapped(
+ pb, 'job_submission_protocol', _JOB_SUBMISSION_PROTOCOL),
+ priorityOrder=pb.priority_order,
+ )
+
+
+def compute_resource(pb):
+ """gRPC ``ComputeResourceDescription`` ->
``ComputeResourceDescriptionSerializer`` shape.
+
+ The deepest read model: recursively adapts batch queues, the file-systems
+ map, and the job-submission and data-movement interface lists.
+ """
+ return SimpleNamespace(
+ computeResourceId=pb.compute_resource_id,
+ hostName=pb.host_name,
+ hostAliases=list(pb.host_aliases),
+ ipAddresses=list(pb.ip_addresses),
+ resourceDescription=pb.resource_description,
+ enabled=pb.enabled,
+ batchQueues=[_batch_queue(q) for q in pb.batch_queues],
+ fileSystems=_file_systems(pb.file_systems),
+ jobSubmissionInterfaces=[
+ _job_submission_interface(j) for j in
pb.job_submission_interfaces],
+ dataMovementInterfaces=[
+ _data_movement_interface(d) for d in pb.data_movement_interfaces],
+ maxMemoryPerNode=pb.max_memory_per_node,
+ gatewayUsageReporting=pb.gateway_usage_reporting,
+ gatewayUsageModuleLoadCommand=pb.gateway_usage_module_load_command,
+ gatewayUsageExecutable=pb.gateway_usage_executable,
+ cpusPerNode=pb.cpus_per_node,
+ defaultNodeCount=pb.default_node_count,
+ defaultCPUCount=pb.default_cpu_count,
+ defaultWalltime=pb.default_walltime,
+ )
+
+
def proto_summary_type(thrift_summary_type):
"""Thrift ``SummaryType`` -> proto ``SummaryType`` enum value (by name).
diff --git a/airavata-django-portal/django_airavata/apps/api/views.py
b/airavata-django-portal/django_airavata/apps/api/views.py
index 6eea9d420..7c273d867 100644
--- a/airavata-django-portal/django_airavata/apps/api/views.py
+++ b/airavata-django-portal/django_airavata/apps/api/views.py
@@ -659,10 +659,12 @@ class ApplicationDeploymentViewSet(APIBackedViewSet):
@action(detail=True)
def queues(self, request, app_deployment_id):
"""Return queues for this deployment with defaults overridden by
deployment defaults if they exist"""
- app_deployment = self.request.airavata_client.getApplicationDeployment(
- self.authz_token, app_deployment_id)
- compute_resource = request.airavata_client.getComputeResource(
- request.authz_token, app_deployment.computeHostId)
+ app_deployment = grpc_adapters.application_deployment(
+ self.request.airavata.research.get_application_deployment(
+ app_deployment_id))
+ compute_resource = grpc_adapters.compute_resource(
+ request.airavata.compute.get_compute_resource(
+ app_deployment.computeHostId))
# Override defaults with app deployment default queue, if defined
batch_queues = []
for batch_queue in compute_resource.batchQueues:
@@ -686,21 +688,19 @@ class ComputeResourceViewSet(mixins.RetrieveModelMixin,
lookup_field = 'compute_resource_id'
def get_instance(self, lookup_value, format=None):
- return self.request.airavata_client.getComputeResource(
- self.authz_token, lookup_value)
+ return grpc_adapters.compute_resource(
+ self.request.airavata.compute.get_compute_resource(lookup_value))
@action(detail=False)
def all_names(self, request, format=None):
"""Return a map of compute resource names keyed by resource id."""
return Response(
- request.airavata_client.getAllComputeResourceNames(
- request.authz_token))
+ request.airavata.compute.get_all_compute_resource_names())
@action(detail=False)
def all_names_list(self, request, format=None):
"""Return a list of compute resource names keyed by resource id."""
- all_names = request.airavata_client.getAllComputeResourceNames(
- request.authz_token)
+ all_names = request.airavata.compute.get_all_compute_resource_names()
return Response([
{
'host_id': host_id,
@@ -713,8 +713,8 @@ class ComputeResourceViewSet(mixins.RetrieveModelMixin,
@action(detail=True)
def queues(self, request, compute_resource_id, format=None):
- details = request.airavata_client.getComputeResource(
- request.authz_token, compute_resource_id)
+ details = grpc_adapters.compute_resource(
+ request.airavata.compute.get_compute_resource(compute_resource_id))
serializer = self.serializer_class(instance=details,
context={'request': request})
data = serializer.data
