This is an automated email from the ASF dual-hosted git repository.
yasithdev pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airavata-portals.git
The following commit(s) were added to refs/heads/main by this push:
new 43604e2d4 refactor(portal): make experiment + application-interface
serializers proto-native (Track D) (#202)
43604e2d4 is described below
commit 43604e2d4909eecacd5f9e02507dbec7ce1bf017
Author: Yasith Jayawardana <[email protected]>
AuthorDate: Tue Jun 9 04:40:41 2026 -0400
refactor(portal): make experiment + application-interface serializers
proto-native (Track D) (#202)
The largest coupled family: the experiment read tree and the application
interfaces share the InputDataObjectType/OutputDataObjectType serializers,
so
they migrate together.
- Input/Output: read the gRPC InputDataObjectType/OutputDataObjectType
protobuf
directly (DataType renders as the member name; metaData proto-'' -> null
via
new ProtoStoredJSONField); create() builds the proto.
- ApplicationInterfaceDescriptionSerializer: proto-native read + write
(build the
proto ApplicationInterfaceDescription with nested inputs/outputs;
showQueueSettings/queueSettingsCalculatorId DB handling preserved). View
_update_input_metadata reads proto fields.
- Experiment tree: full proto-native ExperimentSerializer replicating the
entire
processes -> tasks -> jobs tree (each with its status list),
userConfiguration +
scheduling, inputs/outputs, status, errors. Preserves the subtle
ISO-vs-int
timestamp quirks byte-for-byte (top-level experimentStatus = ISO; nested
process-tree statuses + the nested job creationTime = raw int; standalone
JobSerializer creationTime = ISO). State/type enums render as Thrift ints
via
the new state field helpers. save() builds the proto ExperimentModel
incl. the
writable nested userConfigurationData + computationalResourceScheduling.
- Repoint every experiment/interface consumer to pass protobuf through:
ExperimentViewSet (instance/create/update/launch/jobs/clone),
ProjectViewSet.experiments, FullExperimentViewSet (+ its nested
module/compute/
project/jobs; DataType comparisons now use the proto enum),
ApplicationInterfaceViewSet, ApplicationModuleViewSet
interface/deployment actions, output_views.get_output_views/generate_data,
workspace.edit_experiment, ExperimentArchiveView, _user_storage_path.
- Remove grpc_adapters/grpc_requests experiment-tree + interface + io
adapters and
the now-unused Thrift imports; views.py drops the
DataType/ExperimentModel Thrift
imports.
NOTE: the EXECUTING-only intermediateOutput enrichment still calls the
legacy
experiment_util (Thrift) with a proto experiment — a parked concern tracked
with
the experiment_util migration; it is best-effort (try/except) and never
runs for
the validated non-EXECUTING cases.
Validated byte-for-byte (experiment full tree, job, input, output,
interface) vs
the old adapter+serializer path; write paths (experiment incl. nested
scheduling,
interface incl. nested I/O) produce equivalent protos. manage.py check
green; api
test failures unchanged vs origin/main.
---
.../django_airavata/apps/api/grpc_adapters.py | 429 --------------
.../django_airavata/apps/api/grpc_requests.py | 157 +----
.../django_airavata/apps/api/output_views.py | 23 +-
.../django_airavata/apps/api/serializers.py | 635 +++++++++++++++++----
.../django_airavata/apps/api/views.py | 174 +++---
.../django_airavata/apps/workspace/views.py | 9 +-
6 files changed, 624 insertions(+), 803 deletions(-)
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
index c5a01bff1..a59bb8b62 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_adapters.py
@@ -16,10 +16,6 @@ from types import SimpleNamespace
from airavata.model.appcatalog.computeresource.ttypes import (
JobSubmissionProtocol as _ThriftJobSubmissionProtocol,
)
-from airavata.model.appcatalog.groupresourceprofile.ttypes import (
- ResourceType as _ThriftResourceType,
-)
-from airavata.model.application.io.ttypes import DataType as _ThriftDataType
from airavata.model.data.replica.ttypes import (
DataProductType as _ThriftDataProductType,
ReplicaLocationCategory as _ThriftReplicaLocationCategory,
@@ -28,16 +24,6 @@ from airavata.model.data.replica.ttypes import (
from airavata.model.data.movement.ttypes import (
DataMovementProtocol as _ThriftDataMovementProtocol,
)
-from airavata.model.experiment.ttypes import (
- ExperimentType as _ThriftExperimentType,
-)
-from airavata.model.status.ttypes import (
- ExperimentState as _ThriftExperimentState,
- JobState as _ThriftJobState,
- ProcessState as _ThriftProcessState,
- TaskState as _ThriftTaskState,
-)
-from airavata.model.task.ttypes import TaskTypes as _ThriftTaskTypes
from airavata.model.user.ttypes import Status as _ThriftStatus
@@ -107,421 +93,6 @@ _JOB_SUBMISSION_PROTOCOL = {
'LOCAL_FORK': _ThriftJobSubmissionProtocol.LOCAL_FORK,
}
-def _input_data_object(pb):
- """gRPC ``InputDataObjectType`` -> ``InputDataObjectTypeSerializer``
shape."""
- return SimpleNamespace(
- name=pb.name,
- value=pb.value,
- # DataType proto/Thrift ints differ per name -> bridge by name; the
- # serializer's EnumChoiceField(DataType) reads ``.name`` off the
member.
- type=_thrift_enum(pb, 'type', _ThriftDataType),
- applicationArgument=pb.application_argument,
- standardInput=pb.standard_input,
- userFriendlyDescription=pb.user_friendly_description,
- # JSON string; empty -> None so StoredJSONField renders null like
Thrift.
- metaData=pb.meta_data or None,
- inputOrder=pb.input_order,
- isRequired=pb.is_required,
- requiredToAddedToCommandLine=pb.required_to_added_to_command_line,
- dataStaged=pb.data_staged,
- storageResourceId=pb.storage_resource_id,
- isReadOnly=pb.is_read_only,
- overrideFilename=pb.override_filename,
- )
-
-
-def _output_data_object(pb):
- """gRPC ``OutputDataObjectType`` -> ``OutputDataObjectTypeSerializer``
shape."""
- return SimpleNamespace(
- name=pb.name,
- value=pb.value,
- type=_thrift_enum(pb, 'type', _ThriftDataType),
- applicationArgument=pb.application_argument,
- isRequired=pb.is_required,
- requiredToAddedToCommandLine=pb.required_to_added_to_command_line,
- dataMovement=pb.data_movement,
- location=pb.location,
- searchQuery=pb.search_query,
- outputStreaming=pb.output_streaming,
- storageResourceId=pb.storage_resource_id,
- metaData=pb.meta_data or None,
- )
-
-
-def application_interface(pb):
- """gRPC ``ApplicationInterfaceDescription`` ->
``ApplicationInterfaceDescriptionSerializer`` shape.
-
- Recursively adapts the nested ``applicationInputs``/``applicationOutputs``
- repeated messages.
- """
- return SimpleNamespace(
- applicationInterfaceId=pb.application_interface_id,
- applicationName=pb.application_name,
- applicationDescription=pb.application_description,
- applicationModules=list(pb.application_modules),
- applicationInputs=[_input_data_object(i) for i in
pb.application_inputs],
- applicationOutputs=[_output_data_object(o) for o in
pb.application_outputs],
- archiveWorkingDirectory=pb.archive_working_directory,
- hasOptionalFileInputs=pb.has_optional_file_inputs,
- )
-
-
-# proto ResourceType member name -> Thrift ResourceType value (names align,
-# ints differ: proto SLURM=1 vs Thrift SLURM=0).
-_RESOURCE_TYPE = {
- 'SLURM': _ThriftResourceType.SLURM,
- 'AWS': _ThriftResourceType.AWS,
-}
-
-
-def _compute_resource_reservation(pb):
- """gRPC ``ComputeResourceReservation`` ->
``ComputeResourceReservationSerializer`` shape."""
- return SimpleNamespace(
- reservationId=pb.reservation_id,
- reservationName=pb.reservation_name,
- queueNames=list(pb.queue_names),
- # serializer overrides start/end with nullable UTC fields.
- startTime=pb.start_time or None,
- endTime=pb.end_time or None,
- )
-
-
-def _group_account_ssh_provisioner_config(pb):
- """gRPC ``GroupAccountSSHProvisionerConfig`` -> auto-generated serializer
shape."""
- return SimpleNamespace(
- resourceId=pb.resource_id,
- groupResourceProfileId=pb.group_resource_profile_id,
- configName=pb.config_name,
- configValue=pb.config_value,
- )
-
-
-def _slurm_compute_resource_preference(pb):
- """gRPC ``SlurmComputeResourcePreference`` -> auto-generated serializer
shape."""
- return SimpleNamespace(
- allocationProjectNumber=pb.allocation_project_number,
- preferredBatchQueue=pb.preferred_batch_queue,
- qualityOfService=pb.quality_of_service,
- usageReportingGatewayId=pb.usage_reporting_gateway_id,
- sshAccountProvisioner=pb.ssh_account_provisioner,
- groupSSHAccountProvisionerConfigs=[
- _group_account_ssh_provisioner_config(c)
- for c in pb.group_ssh_account_provisioner_configs],
-
sshAccountProvisionerAdditionalInfo=pb.ssh_account_provisioner_additional_info,
- reservations=[_compute_resource_reservation(r) for r in
pb.reservations],
- )
-
-
-def _aws_compute_resource_preference(pb):
- """gRPC ``AwsComputeResourcePreference`` -> auto-generated serializer
shape."""
- return SimpleNamespace(
- region=pb.region,
- preferredAmiId=pb.preferred_ami_id,
- preferredInstanceType=pb.preferred_instance_type,
- )
-
-
-def _environment_specific_preferences(pb):
- """proto oneof ``EnvironmentSpecificPreferences`` -> {slurm, aws} (one
set)."""
- which = pb.WhichOneof('preferences')
- return SimpleNamespace(
- slurm=(_slurm_compute_resource_preference(pb.slurm)
- if which == 'slurm' else None),
- aws=(_aws_compute_resource_preference(pb.aws)
- if which == 'aws' else None),
- )
-
-
-def _group_compute_resource_preference(pb):
- """gRPC ``GroupComputeResourcePreference`` -> auto-generated serializer
shape."""
- return SimpleNamespace(
- computeResourceId=pb.compute_resource_id,
- groupResourceProfileId=pb.group_resource_profile_id,
- overridebyAiravata=pb.override_by_airavata,
- loginUserName=pb.login_user_name,
- scratchLocation=pb.scratch_location,
- # rendered as raw ints; bridge by name (incl. the JSP_CLOUD/LOCAL
- # divergences) to the Thrift integer the frontend expects.
- preferredJobSubmissionProtocol=_thrift_enum_mapped(
- pb, 'preferred_job_submission_protocol', _JOB_SUBMISSION_PROTOCOL),
- preferredDataMovementProtocol=_thrift_enum_mapped(
- pb, 'preferred_data_movement_protocol', _DATA_MOVEMENT_PROTOCOL),
- # empty token -> None so the serializer's userHasWriteAccess token READ
- # check (``token is None or ...``) skips unset tokens.
- resourceSpecificCredentialStoreToken=(
- pb.resource_specific_credential_store_token or None),
- resourceType=_thrift_enum_mapped(pb, 'resource_type', _RESOURCE_TYPE),
- specificPreferences=(
- _environment_specific_preferences(pb.specific_preferences)
- if pb.HasField('specific_preferences') else None),
- )
-
-
-def _compute_resource_policy(pb):
- """gRPC ``ComputeResourcePolicy`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- resourcePolicyId=pb.resource_policy_id,
- computeResourceId=pb.compute_resource_id,
- groupResourceProfileId=pb.group_resource_profile_id,
- allowedBatchQueues=list(pb.allowed_batch_queues),
- )
-
-
-def _batch_queue_resource_policy(pb):
- """gRPC ``BatchQueueResourcePolicy`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- resourcePolicyId=pb.resource_policy_id,
- computeResourceId=pb.compute_resource_id,
- groupResourceProfileId=pb.group_resource_profile_id,
- queuename=pb.queuename,
- maxAllowedNodes=pb.max_allowed_nodes,
- maxAllowedCores=pb.max_allowed_cores,
- maxAllowedWalltime=pb.max_allowed_walltime,
- )
-
-
-def group_resource_profile(pb):
- """gRPC ``GroupResourceProfile`` -> ``GroupResourceProfileSerializer``
shape.
-
- Recursively adapts the compute preferences (each carrying a slurm/aws
- union of specific preferences with reservations) and the compute /
- batch-queue resource policies.
- """
- return SimpleNamespace(
- gatewayId=pb.gateway_id,
- groupResourceProfileId=pb.group_resource_profile_id,
- groupResourceProfileName=pb.group_resource_profile_name,
- computePreferences=[
- _group_compute_resource_preference(p) for p in
pb.compute_preferences],
- computeResourcePolicies=[
- _compute_resource_policy(p) for p in pb.compute_resource_policies],
- batchQueueResourcePolicies=[
- _batch_queue_resource_policy(p) for p in
pb.batch_queue_resource_policies],
- creationTime=pb.creation_time or None,
- updatedTime=pb.updated_time or None,
- defaultCredentialStoreToken=pb.default_credential_store_token or None,
- )
-
-
-# --- Experiment tree -------------------------------------------------------
-#
-# getExperiment returns the full ExperimentModel including the processes tree
-# (process -> tasks -> jobs, each with its status list). The status/type enums
-# render as raw integers and are bridged by name with the proto prefix stripped
-# (proto EXPERIMENT_STATE_CREATED -> Thrift CREATED). Status timeOfStateChange
-# stays an int (ExperimentStatusSerializer/ProcessStatusSerializer use a
-# non-nullable UTC field; the nested auto-generated ones render the int);
-# model creation/update times map proto-zero -> None (nullable).
-
-
-def _error_model(pb):
- """gRPC ``ErrorModel`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- errorId=pb.error_id,
- creationTime=pb.creation_time or None,
- actualErrorMessage=pb.actual_error_message,
- userFriendlyMessage=pb.user_friendly_message,
- transientOrPersistent=pb.transient_or_persistent,
- rootCauseErrorIdList=list(pb.root_cause_error_id_list),
- )
-
-
-def _computational_resource_scheduling(pb):
- """gRPC ``ComputationalResourceSchedulingModel`` -> auto-generated
shape."""
- return SimpleNamespace(
- resourceHostId=pb.resource_host_id,
- totalCPUCount=pb.total_cpu_count,
- nodeCount=pb.node_count,
- numberOfThreads=pb.number_of_threads,
- queueName=pb.queue_name,
- wallTimeLimit=pb.wall_time_limit,
- totalPhysicalMemory=pb.total_physical_memory,
- chessisNumber=pb.chessis_number,
- staticWorkingDir=pb.static_working_dir,
- overrideLoginUserName=pb.override_login_user_name,
- overrideScratchLocation=pb.override_scratch_location,
- overrideAllocationProjectNumber=pb.override_allocation_project_number,
- mGroupCount=pb.m_group_count,
- )
-
-
-def _user_configuration_data(pb):
- """gRPC ``UserConfigurationDataModel`` -> auto-generated serializer
shape."""
- return SimpleNamespace(
- airavataAutoSchedule=pb.airavata_auto_schedule,
- overrideManualScheduledParams=pb.override_manual_scheduled_params,
- shareExperimentPublicly=pb.share_experiment_publicly,
- computationalResourceScheduling=(
-
_computational_resource_scheduling(pb.computational_resource_scheduling)
- if pb.HasField('computational_resource_scheduling') else None),
- throttleResources=pb.throttle_resources,
- userDN=pb.user_dn,
- generateCert=pb.generate_cert,
- inputStorageResourceId=pb.input_storage_resource_id,
- outputStorageResourceId=pb.output_storage_resource_id,
- experimentDataDir=pb.experiment_data_dir,
- useUserCRPref=pb.use_user_cr_pref,
- groupResourceProfileId=pb.group_resource_profile_id,
- autoScheduledCompResourceSchedulingList=[
- _computational_resource_scheduling(s)
- for s in pb.auto_scheduled_comp_resource_scheduling_list],
- )
-
-
-def _experiment_status(pb):
- """gRPC ``ExperimentStatus`` -> ``ExperimentStatusSerializer`` shape."""
- return SimpleNamespace(
- state=_thrift_enum_prefixed(
- pb, 'state', _ThriftExperimentState, 'EXPERIMENT_STATE_'),
- timeOfStateChange=pb.time_of_state_change,
- reason=pb.reason,
- statusId=pb.status_id,
- )
-
-
-def _process_status(pb):
- """gRPC ``ProcessStatus`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- state=_thrift_enum_prefixed(
- pb, 'state', _ThriftProcessState, 'PROCESS_STATE_'),
- timeOfStateChange=pb.time_of_state_change,
- reason=pb.reason,
- statusId=pb.status_id,
- processId=pb.process_id,
- )
-
-
-def _task_status(pb):
- """gRPC ``TaskStatus`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- state=_thrift_enum_prefixed(
- pb, 'state', _ThriftTaskState, 'TASK_STATE_'),
- timeOfStateChange=pb.time_of_state_change,
- reason=pb.reason,
- statusId=pb.status_id,
- )
-
-
-def _job_status(pb):
- """gRPC ``JobStatus`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- jobState=_thrift_enum_prefixed(
- pb, 'job_state', _ThriftJobState, 'JOB_STATE_'),
- timeOfStateChange=pb.time_of_state_change,
- reason=pb.reason,
- statusId=pb.status_id,
- )
-
-
-def job_model(pb):
- """gRPC ``JobModel`` -> ``JobSerializer`` shape."""
- return SimpleNamespace(
- jobId=pb.job_id,
- taskId=pb.task_id,
- processId=pb.process_id,
- jobDescription=pb.job_description,
- creationTime=pb.creation_time or None,
- jobStatuses=[_job_status(s) for s in pb.job_statuses],
- computeResourceConsumed=pb.compute_resource_consumed,
- jobName=pb.job_name,
- workingDir=pb.working_dir,
- stdOut=pb.std_out,
- stdErr=pb.std_err,
- exitCode=pb.exit_code,
- )
-
-
-def _task_model(pb):
- """gRPC ``TaskModel`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- taskId=pb.task_id,
- taskType=_thrift_enum_prefixed(
- pb, 'task_type', _ThriftTaskTypes, 'TASK_TYPES_'),
- parentProcessId=pb.parent_process_id,
- creationTime=pb.creation_time or None,
- lastUpdateTime=pb.last_update_time or None,
- taskStatuses=[_task_status(s) for s in pb.task_statuses],
- taskDetail=pb.task_detail,
- subTaskModel=pb.sub_task_model,
- taskErrors=[_error_model(e) for e in pb.task_errors],
- jobs=[job_model(j) for j in pb.jobs],
- maxRetry=pb.max_retry,
- currentRetry=pb.current_retry,
- )
-
-
-def _process_model(pb):
- """gRPC ``ProcessModel`` -> auto-generated serializer shape."""
- return SimpleNamespace(
- processId=pb.process_id,
- experimentId=pb.experiment_id,
- creationTime=pb.creation_time or None,
- lastUpdateTime=pb.last_update_time or None,
- processStatuses=[_process_status(s) for s in pb.process_statuses],
- processDetail=pb.process_detail,
- applicationInterfaceId=pb.application_interface_id,
- applicationDeploymentId=pb.application_deployment_id,
- computeResourceId=pb.compute_resource_id,
- processInputs=[_input_data_object(i) for i in pb.process_inputs],
- processOutputs=[_output_data_object(o) for o in pb.process_outputs],
- processResourceSchedule=(
- _computational_resource_scheduling(pb.process_resource_schedule)
- if pb.HasField('process_resource_schedule') else None),
- tasks=[_task_model(t) for t in pb.tasks],
- taskDag=pb.task_dag,
- processErrors=[_error_model(e) for e in pb.process_errors],
- gatewayExecutionId=pb.gateway_execution_id,
- enableEmailNotification=pb.enable_email_notification,
- emailAddresses=list(pb.email_addresses),
- inputStorageResourceId=pb.input_storage_resource_id,
- outputStorageResourceId=pb.output_storage_resource_id,
- userDn=pb.user_dn,
- generateCert=pb.generate_cert,
- experimentDataDir=pb.experiment_data_dir,
- userName=pb.user_name,
- useUserCRPref=pb.use_user_cr_pref,
- groupResourceProfileId=pb.group_resource_profile_id,
- # the legacy workflow-engine subsystem is not adapted (rarely
populated);
- # an empty list matches the Thrift default for non-workflow processes.
- processWorkflows=[],
- )
-
-
-def experiment(pb):
- """gRPC ``ExperimentModel`` -> ``ExperimentSerializer`` shape.
-
- The deepest read model: recursively adapts the user configuration, the
- experiment input/output and status lists, the errors, and the full
- processes -> tasks -> jobs tree.
- """
- return SimpleNamespace(
- experimentId=pb.experiment_id,
- projectId=pb.project_id,
- gatewayId=pb.gateway_id,
- experimentType=_thrift_enum_prefixed(
- pb, 'experiment_type', _ThriftExperimentType, 'EXPERIMENT_TYPE_'),
- userName=pb.user_name,
- experimentName=pb.experiment_name,
- creationTime=pb.creation_time or None,
- description=pb.description,
- executionId=pb.execution_id,
- gatewayExecutionId=pb.gateway_execution_id,
- gatewayInstanceId=pb.gateway_instance_id,
- enableEmailNotification=pb.enable_email_notification,
- emailAddresses=list(pb.email_addresses),
- userConfigurationData=(
- _user_configuration_data(pb.user_configuration_data)
- if pb.HasField('user_configuration_data') else None),
- experimentInputs=[_input_data_object(i) for i in pb.experiment_inputs],
- experimentOutputs=[_output_data_object(o) for o in
pb.experiment_outputs],
- experimentStatus=[_experiment_status(s) for s in pb.experiment_status],
- errors=[_error_model(e) for e in pb.errors],
- processes=[_process_model(p) for p in pb.processes],
- # legacy workflow-engine subsystem not adapted (rarely populated).
- workflow=None,
- )
-
-
def group(pb):
"""gRPC ``GroupModel`` -> ``GroupSerializer`` shape."""
return SimpleNamespace(
diff --git a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
index 7cb80e006..cf040d581 100644
--- a/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
+++ b/airavata-django-portal/django_airavata/apps/api/grpc_requests.py
@@ -20,13 +20,9 @@ from airavata.model.appcatalog.computeresource.ttypes import
(
from airavata.model.appcatalog.groupresourceprofile.ttypes import (
ResourceType as _ThriftResourceType,
)
-from airavata.model.application.io.ttypes import DataType as _ThriftDataType
from airavata.model.data.movement.ttypes import (
DataMovementProtocol as _ThriftDataMovementProtocol,
)
-from airavata.model.experiment.ttypes import (
- ExperimentType as _ThriftExperimentType,
-)
_GEN = "airavata_sdk.generated.org.apache.airavata.model"
@@ -69,63 +65,19 @@ def password_credential(gateway_id, portal_user_name,
login_user_name,
)
-def _input_data_object(t):
- io = _pb2("application.io.application_io_pb2")
- return io.InputDataObjectType(
- name=t.name or '',
- value=t.value or '',
- type=_proto_enum(io.DataType, _ThriftDataType, t.type),
- application_argument=t.applicationArgument or '',
- standard_input=bool(t.standardInput),
- user_friendly_description=t.userFriendlyDescription or '',
- # StoredJSONField.to_internal_value json.dumps()es metaData to a
string.
- meta_data=t.metaData or '',
- input_order=t.inputOrder or 0,
- is_required=bool(t.isRequired),
- required_to_added_to_command_line=bool(t.requiredToAddedToCommandLine),
- data_staged=bool(t.dataStaged),
- storage_resource_id=t.storageResourceId or '',
- is_read_only=bool(t.isReadOnly),
- override_filename=t.overrideFilename or '',
- )
-
-
-def _output_data_object(t):
- io = _pb2("application.io.application_io_pb2")
- return io.OutputDataObjectType(
- name=t.name or '',
- value=t.value or '',
- type=_proto_enum(io.DataType, _ThriftDataType, t.type),
- application_argument=t.applicationArgument or '',
- is_required=bool(t.isRequired),
- required_to_added_to_command_line=bool(t.requiredToAddedToCommandLine),
- data_movement=bool(t.dataMovement),
- location=t.location or '',
- search_query=t.searchQuery or '',
- output_streaming=bool(t.outputStreaming),
- storage_resource_id=t.storageResourceId or '',
- meta_data=t.metaData or '',
- )
-
-
-def application_interface(t):
- """Thrift ``ApplicationInterfaceDescription`` -> proto message."""
- return
_pb2("appcatalog.appinterface.app_interface_pb2").ApplicationInterfaceDescription(
- application_interface_id=t.applicationInterfaceId or '',
- application_name=t.applicationName or '',
- application_description=t.applicationDescription or '',
- application_modules=list(t.applicationModules or []),
- application_inputs=[_input_data_object(i) for i in
(t.applicationInputs or [])],
- application_outputs=[_output_data_object(o) for o in
(t.applicationOutputs or [])],
- archive_working_directory=bool(t.archiveWorkingDirectory),
- has_optional_file_inputs=bool(t.hasOptionalFileInputs),
- )
+def _proto_enum_rev(proto_enum, rev_map, value):
+ """Thrift enum value -> proto enum value via an EXPLICIT inverse name map
+ (for protocol enums whose proto/Thrift member names diverge). None/unmapped
+ -> 0 (proto *_UNKNOWN)."""
+ if value is None:
+ return 0
+ name = rev_map.get(value)
+ return proto_enum.Value(name) if name is not None else 0
-# proto-name -> Thrift protocol value maps (mirror grpc_adapters); inverted
-# below to go Thrift value -> proto value, preserving the divergent name pairs
+# Thrift protocol value -> proto member name, preserving the divergent name
pairs
# (Thrift CLOUD <-> proto JSP_CLOUD; Thrift LOCAL <-> proto
-# DATA_MOVEMENT_PROTOCOL_LOCAL).
+# DATA_MOVEMENT_PROTOCOL_LOCAL). Used by the group-resource-profile write path.
_JOB_SUBMISSION_PROTOCOL_REV = {
_ThriftJobSubmissionProtocol.LOCAL: 'LOCAL',
_ThriftJobSubmissionProtocol.SSH: 'SSH',
@@ -143,95 +95,6 @@ _DATA_MOVEMENT_PROTOCOL_REV = {
}
-def _proto_enum_rev(proto_enum, rev_map, value):
- """Thrift enum value -> proto enum value via an EXPLICIT inverse name map
- (for protocol enums whose proto/Thrift member names diverge). None/unmapped
- -> 0 (proto *_UNKNOWN)."""
- if value is None:
- return 0
- name = rev_map.get(value)
- return proto_enum.Value(name) if name is not None else 0
-
-
-# --- Experiment tree (write direction) -------------------------------------
-# Reverse of grpc_adapters.experiment. The write path carries only what the
-# user submitted; status/errors/processes/workflow are server-managed.
-
-
-def _computational_resource_scheduling(t):
- """Thrift ``ComputationalResourceSchedulingModel`` -> proto message."""
- return
_pb2("scheduling.scheduling_pb2").ComputationalResourceSchedulingModel(
- resource_host_id=t.resourceHostId or '',
- total_cpu_count=t.totalCPUCount or 0,
- node_count=t.nodeCount or 0,
- number_of_threads=t.numberOfThreads or 0,
- queue_name=t.queueName or '',
- wall_time_limit=t.wallTimeLimit or 0,
- total_physical_memory=t.totalPhysicalMemory or 0,
- chessis_number=t.chessisNumber or '',
- static_working_dir=t.staticWorkingDir or '',
- override_login_user_name=t.overrideLoginUserName or '',
- override_scratch_location=t.overrideScratchLocation or '',
- override_allocation_project_number=t.overrideAllocationProjectNumber
or '',
- m_group_count=t.mGroupCount or 0,
- )
-
-
-def _user_configuration_data(t):
- """Thrift ``UserConfigurationDataModel`` -> proto message."""
- ucd = _pb2("experiment.experiment_pb2").UserConfigurationDataModel(
- airavata_auto_schedule=bool(t.airavataAutoSchedule),
- override_manual_scheduled_params=bool(t.overrideManualScheduledParams),
- share_experiment_publicly=bool(t.shareExperimentPublicly),
- throttle_resources=bool(t.throttleResources),
- user_dn=t.userDN or '',
- generate_cert=bool(t.generateCert),
- input_storage_resource_id=t.inputStorageResourceId or '',
- output_storage_resource_id=t.outputStorageResourceId or '',
- experiment_data_dir=t.experimentDataDir or '',
- use_user_cr_pref=bool(t.useUserCRPref),
- group_resource_profile_id=t.groupResourceProfileId or '',
- auto_scheduled_comp_resource_scheduling_list=[
- _computational_resource_scheduling(s)
- for s in (t.autoScheduledCompResourceSchedulingList or [])],
- )
- if t.computationalResourceScheduling is not None:
- ucd.computational_resource_scheduling.CopyFrom(
-
_computational_resource_scheduling(t.computationalResourceScheduling))
- return ucd
-
-
-def experiment(t):
- """Thrift ``ExperimentModel`` -> proto ``ExperimentModel`` request message.
-
- Only the user-submitted fields are populated; experiment_status, errors and
- processes are server-managed (left empty), workflow omitted.
- """
- exp_pb = _pb2("experiment.experiment_pb2")
- e = exp_pb.ExperimentModel(
- experiment_id=t.experimentId or '',
- project_id=t.projectId or '',
- gateway_id=t.gatewayId or '',
- experiment_type=_proto_enum(
- exp_pb.ExperimentType, _ThriftExperimentType, t.experimentType,
- 'EXPERIMENT_TYPE_'),
- user_name=t.userName or '',
- experiment_name=t.experimentName or '',
- description=t.description or '',
- execution_id=t.executionId or '',
- enable_email_notification=bool(t.enableEmailNotification),
- email_addresses=list(t.emailAddresses or []),
- experiment_inputs=[
- _input_data_object(i) for i in (t.experimentInputs or [])],
- experiment_outputs=[
- _output_data_object(o) for o in (t.experimentOutputs or [])],
- )
- if t.userConfigurationData is not None:
- e.user_configuration_data.CopyFrom(
- _user_configuration_data(t.userConfigurationData))
- return e
-
-
# --- Group resource profile (write direction) ------------------------------
# Reverse of grpc_adapters.group_resource_profile. Reuses the _proto_enum_rev
# protocol maps defined above.
diff --git a/airavata-django-portal/django_airavata/apps/api/output_views.py
b/airavata-django-portal/django_airavata/apps/api/output_views.py
index 67aafa3d1..2820d34f8 100644
--- a/airavata-django-portal/django_airavata/apps/api/output_views.py
+++ b/airavata-django-portal/django_airavata/apps/api/output_views.py
@@ -8,7 +8,9 @@ from functools import partial
import nbformat
import papermill as pm
-from airavata.model.application.io.ttypes import DataType
+from
airavata_sdk.generated.org.apache.airavata.model.application.io.application_io_pb2
import ( # noqa: E501
+ DataType,
+)
from django.conf import settings
from nbconvert import HTMLExporter
@@ -79,7 +81,7 @@ DEFAULT_VIEW_PROVIDERS = {
def get_output_views(request, experiment, application_interface=None):
output_views = {}
- for output in experiment.experimentOutputs:
+ for output in experiment.experiment_outputs:
output_views[output.name] = []
output_view_provider_ids = _get_output_view_providers(
output, application_interface)
@@ -123,9 +125,9 @@ def _get_output_view_provider(output_view_provider_id):
def _get_output_view_providers(experiment_output, application_interface):
output_view_providers = []
logger.debug("experiment_output={}".format(experiment_output))
- if experiment_output.metaData:
+ if experiment_output.meta_data:
try:
- output_metadata = json.loads(experiment_output.metaData)
+ output_metadata = json.loads(experiment_output.meta_data)
logger.debug("output_metadata={}".format(output_metadata))
if 'output-view-providers' in output_metadata:
output_view_providers.extend(
@@ -148,16 +150,16 @@ def _get_output_view_providers(experiment_output,
application_interface):
def _get_application_output_view_providers(application_interface, output_name):
app_output = [o
- for o in application_interface.applicationOutputs
+ for o in application_interface.application_outputs
if o.name == output_name]
if len(app_output) == 1:
logger.debug("{}: {}".format(output_name, app_output))
app_output = app_output[0]
else:
return []
- if app_output.metaData:
+ if app_output.meta_data:
try:
- output_metadata = json.loads(app_output.metaData)
+ output_metadata = json.loads(app_output.meta_data)
if 'output-view-providers' in output_metadata:
return output_metadata['output-view-providers']
except Exception:
@@ -175,10 +177,9 @@ def generate_data(request,
**kwargs):
output_view_provider = _get_output_view_provider(output_view_provider_id)
# TODO if output_view_provider is None, return 404
- experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(experiment_id))
+ experiment = request.airavata.research.get_experiment(experiment_id)
experiment_output = [o
- for o in experiment.experimentOutputs
+ for o in experiment.experiment_outputs
if o.name == experiment_output_name]
# TODO: handle experiment_output not found by name
experiment_output = experiment_output[0]
@@ -216,7 +217,7 @@ def _generate_data(request,
DataType.URI_COLLECTION,
DataType.STDOUT,
DataType.STDERR) and
- experiment_output.value.startswith("airavata-dp")):
+ experiment_output.value.startswith("airavata-dp")):
data_product_uris = experiment_output.value.split(",")
data_products = map(
lambda dpid: grpc_adapters.data_product(
diff --git a/airavata-django-portal/django_airavata/apps/api/serializers.py
b/airavata-django-portal/django_airavata/apps/api/serializers.py
index d65db1301..35476867c 100644
--- a/airavata-django-portal/django_airavata/apps/api/serializers.py
+++ b/airavata-django-portal/django_airavata/apps/api/serializers.py
@@ -3,13 +3,9 @@ import datetime
import json
import logging
import os
-from pathlib import Path
from urllib.parse import quote
from airavata.model.application.io.ttypes import DataType
-from airavata.model.appcatalog.appinterface.ttypes import (
- ApplicationInterfaceDescription
-)
from airavata.model.appcatalog.groupresourceprofile.ttypes import (
ComputeResourceReservation,
GroupComputeResourcePreference,
@@ -19,23 +15,13 @@ from airavata.model.appcatalog.groupresourceprofile.ttypes
import (
AwsComputeResourcePreference
)
from airavata.model.appcatalog.parser.ttypes import IOType as _ThriftIOType
-from airavata.model.application.io.ttypes import (
- InputDataObjectType,
- OutputDataObjectType
-)
from airavata.model.data.replica.ttypes import (
DataProductModel,
DataReplicaLocationModel
)
-from airavata.model.experiment.ttypes import (
- ExperimentModel
-)
from airavata.model.group.ttypes import GroupModel, ResourcePermissionType
-from airavata.model.job.ttypes import JobModel
from airavata.model.status.ttypes import (
- ExperimentState,
- ExperimentStatus,
- ProcessStatus
+ ExperimentState
)
from airavata.model.user.ttypes import UserProfile
from airavata_django_portal_sdk import (
@@ -394,6 +380,16 @@ class StoredJSONField(serializers.JSONField):
self.fail('invalid')
+class ProtoStoredJSONField(StoredJSONField):
+ """:class:`StoredJSONField` for a proto string field: the proto-empty
default
+ ``''`` renders ``null`` (the old adapters mapped ``pb.meta_data or
None``)."""
+
+ def to_representation(self, value):
+ if not value:
+ return None
+ return super().to_representation(value)
+
+
class OrderedListField(serializers.ListField):
def __init__(self, *args, **kwargs):
@@ -639,91 +635,155 @@ class EnumChoiceField(serializers.ChoiceField):
return value.name
+def _application_io_pb2():
+ from airavata_sdk.generated.org.apache.airavata.model.application.io
import (
+ application_io_pb2,
+ )
+ return application_io_pb2
+
+
+def _app_interface_pb2():
+ from
airavata_sdk.generated.org.apache.airavata.model.appcatalog.appinterface import
( # noqa: E501
+ app_interface_pb2,
+ )
+ return app_interface_pb2
+
+
+def _data_type_field(**kwargs):
+ # DataType renders as the member NAME (proto STRING/INTEGER/... == Thrift
names;
+ # proto prefixes only the zero DATA_TYPE_UNKNOWN sentinel).
+ return proto_enum_name_field(
+ _application_io_pb2().DataType.DESCRIPTOR,
+ proto_prefix='DATA_TYPE_', **kwargs)
+
+
+def _proto_input_data_object(d):
+ io = _application_io_pb2()
+ return io.InputDataObjectType(
+ name=d.get('name', '') or '',
+ value=d.get('value', '') or '',
+ type=d.get('type', 0) or 0,
+ application_argument=d.get('application_argument', '') or '',
+ standard_input=bool(d.get('standard_input', False)),
+ user_friendly_description=d.get('user_friendly_description', '') or '',
+ meta_data=d.get('meta_data', '') or '',
+ input_order=d.get('input_order', 0) or 0,
+ is_required=bool(d.get('is_required', False)),
+ required_to_added_to_command_line=bool(
+ d.get('required_to_added_to_command_line', False)),
+ data_staged=bool(d.get('data_staged', False)),
+ storage_resource_id=d.get('storage_resource_id', '') or '',
+ is_read_only=bool(d.get('is_read_only', False)),
+ override_filename=d.get('override_filename', '') or '',
+ )
+
+
+def _proto_output_data_object(d):
+ io = _application_io_pb2()
+ return io.OutputDataObjectType(
+ name=d.get('name', '') or '',
+ value=d.get('value', '') or '',
+ type=d.get('type', 0) or 0,
+ application_argument=d.get('application_argument', '') or '',
+ is_required=bool(d.get('is_required', False)),
+ required_to_added_to_command_line=bool(
+ d.get('required_to_added_to_command_line', False)),
+ data_movement=bool(d.get('data_movement', False)),
+ location=d.get('location', '') or '',
+ search_query=d.get('search_query', '') or '',
+ output_streaming=bool(d.get('output_streaming', False)),
+ storage_resource_id=d.get('storage_resource_id', '') or '',
+ meta_data=d.get('meta_data', '') or '',
+ )
+
+
class InputDataObjectTypeSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``InputDataObjectType`` message."""
+
name = serializers.CharField()
value = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
- type = EnumChoiceField(enum_class=DataType)
- applicationArgument = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- standardInput = serializers.BooleanField(default=False)
- userFriendlyDescription = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- metaData = StoredJSONField(allow_null=True, required=False)
- inputOrder = serializers.IntegerField(required=False, allow_null=True)
- isRequired = serializers.BooleanField(default=False)
- requiredToAddedToCommandLine = serializers.BooleanField(default=False)
- dataStaged = serializers.BooleanField(default=False)
- storageResourceId = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- isReadOnly = serializers.BooleanField(default=False)
- overrideFilename = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
+ type = _data_type_field(required=False)
+ applicationArgument = serializers.CharField(source='application_argument',
allow_blank=True, allow_null=True, required=False)
+ standardInput = serializers.BooleanField(source='standard_input',
default=False)
+ userFriendlyDescription =
serializers.CharField(source='user_friendly_description', allow_blank=True,
allow_null=True, required=False)
+ metaData = ProtoStoredJSONField(source='meta_data', allow_null=True,
required=False)
+ inputOrder = serializers.IntegerField(source='input_order',
required=False, allow_null=True)
+ isRequired = serializers.BooleanField(source='is_required', default=False)
+ requiredToAddedToCommandLine =
serializers.BooleanField(source='required_to_added_to_command_line',
default=False)
+ dataStaged = serializers.BooleanField(source='data_staged', default=False)
+ storageResourceId = serializers.CharField(source='storage_resource_id',
allow_blank=True, allow_null=True, required=False)
+ isReadOnly = serializers.BooleanField(source='is_read_only', default=False)
+ overrideFilename = serializers.CharField(source='override_filename',
allow_blank=True, allow_null=True, required=False)
def create(self, validated_data):
- return InputDataObjectType(**validated_data)
+ return _proto_input_data_object(validated_data)
- def update(self, instance, validated_data):
- for attr, value in validated_data.items():
- setattr(instance, attr, value)
- return instance
class OutputDataObjectTypeSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``OutputDataObjectType``
message."""
+
name = serializers.CharField()
value = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
- type = EnumChoiceField(enum_class=DataType)
- applicationArgument = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- isRequired = serializers.BooleanField(default=False)
- requiredToAddedToCommandLine = serializers.BooleanField(default=False)
- dataMovement = serializers.BooleanField(default=False)
+ type = _data_type_field(required=False)
+ applicationArgument = serializers.CharField(source='application_argument',
allow_blank=True, allow_null=True, required=False)
+ isRequired = serializers.BooleanField(source='is_required', default=False)
+ requiredToAddedToCommandLine =
serializers.BooleanField(source='required_to_added_to_command_line',
default=False)
+ dataMovement = serializers.BooleanField(source='data_movement',
default=False)
location = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
- searchQuery = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
- outputStreaming = serializers.BooleanField(default=False)
- storageResourceId = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- metaData = StoredJSONField(allow_null=True, required=False)
+ searchQuery = serializers.CharField(source='search_query',
allow_blank=True, allow_null=True, required=False)
+ outputStreaming = serializers.BooleanField(source='output_streaming',
default=False)
+ storageResourceId = serializers.CharField(source='storage_resource_id',
allow_blank=True, allow_null=True, required=False)
+ metaData = ProtoStoredJSONField(source='meta_data', allow_null=True,
required=False)
def create(self, validated_data):
- return OutputDataObjectType(**validated_data)
+ return _proto_output_data_object(validated_data)
- def update(self, instance, validated_data):
- for attr, value in validated_data.items():
- setattr(instance, attr, value)
- return instance
class ApplicationInterfaceDescriptionSerializer(serializers.Serializer):
- applicationInterfaceId = serializers.CharField(required=False,
allow_null=True, allow_blank=True)
- applicationName = serializers.CharField()
- applicationDescription = serializers.CharField(allow_blank=True,
allow_null=True, required=False)
- applicationModules = serializers.ListField(child=serializers.CharField(),
allow_null=True, required=False)
- applicationInputs = InputDataObjectTypeSerializer(many=True,
allow_null=True, required=False)
- applicationOutputs = OutputDataObjectTypeSerializer(many=True,
allow_null=True, required=False)
- archiveWorkingDirectory = serializers.BooleanField(default=False)
- hasOptionalFileInputs = serializers.BooleanField(default=False,
read_only=True)
+ """Proto-native serializer for the gRPC
``ApplicationInterfaceDescription``."""
+
+ applicationInterfaceId =
serializers.CharField(source='application_interface_id', required=False,
allow_null=True, allow_blank=True)
+ applicationName = serializers.CharField(source='application_name')
+ applicationDescription =
serializers.CharField(source='application_description', allow_blank=True,
allow_null=True, required=False)
+ applicationModules = serializers.ListField(source='application_modules',
child=serializers.CharField(), allow_null=True, required=False)
+ applicationInputs =
InputDataObjectTypeSerializer(source='application_inputs', many=True,
allow_null=True, required=False)
+ applicationOutputs =
OutputDataObjectTypeSerializer(source='application_outputs', many=True,
allow_null=True, required=False)
+ archiveWorkingDirectory =
serializers.BooleanField(source='archive_working_directory', default=False)
+ hasOptionalFileInputs =
serializers.BooleanField(source='has_optional_file_inputs', default=False,
read_only=True)
url = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:application-interface-detail',
- lookup_field='applicationInterfaceId',
+ lookup_field='application_interface_id',
lookup_url_kwarg='app_interface_id', read_only=True)
userHasWriteAccess = serializers.SerializerMethodField()
showQueueSettings = serializers.BooleanField(required=False)
queueSettingsCalculatorId = serializers.CharField(allow_null=True,
required=False)
def create(self, validated_data):
- # Convert inputs/outputs from dicts to Thrift objects
- inputs_data = validated_data.pop('applicationInputs', None)
- outputs_data = validated_data.pop('applicationOutputs', None)
-
- # Remove Django specific fields
+ io = _application_io_pb2() # noqa: F841 (imported for side-effect
symmetry)
+ inputs_data = validated_data.pop('application_inputs', None)
+ outputs_data = validated_data.pop('application_outputs', None)
+ # Remove Django-specific (non-proto) fields.
validated_data.pop('showQueueSettings', None)
validated_data.pop('queueSettingsCalculatorId', None)
- validated_data.pop('url', None)
- validated_data.pop('userHasWriteAccess', None)
-
- if 'applicationInterfaceId' in validated_data and
validated_data['applicationInterfaceId'] is None:
- validated_data['applicationInterfaceId'] = ""
-
- application_interface =
ApplicationInterfaceDescription(**validated_data)
-
+ ai = _app_interface_pb2()
+ application_interface = ai.ApplicationInterfaceDescription(
+ application_interface_id=validated_data.get(
+ 'application_interface_id', '') or '',
+ application_name=validated_data.get('application_name', '') or '',
+ application_description=validated_data.get(
+ 'application_description', '') or '',
+ application_modules=list(
+ validated_data.get('application_modules', []) or []),
+ archive_working_directory=bool(
+ validated_data.get('archive_working_directory', False)),
+ )
if inputs_data is not None:
- application_interface.applicationInputs =
[InputDataObjectType(**inp) for inp in inputs_data]
+ application_interface.application_inputs.extend(
+ _proto_input_data_object(inp) for inp in inputs_data)
if outputs_data is not None:
- application_interface.applicationOutputs =
[OutputDataObjectType(**out) for out in outputs_data]
-
+ application_interface.application_outputs.extend(
+ _proto_output_data_object(out) for out in outputs_data)
return application_interface
def update(self, instance, validated_data):
@@ -732,26 +792,40 @@ class
ApplicationInterfaceDescriptionSerializer(serializers.Serializer):
defaults["show_queue_settings"] =
validated_data.pop("showQueueSettings")
if "queueSettingsCalculatorId" in validated_data:
defaults["queue_settings_calculator_id"] =
validated_data.pop("queueSettingsCalculatorId")
- application_module_id = instance.applicationModules[0]
+ application_module_id = instance.application_modules[0]
if defaults:
models.ApplicationSettings.objects.update_or_create(
application_module_id=application_module_id, defaults=defaults
)
- inputs_data = validated_data.pop('applicationInputs', None)
- outputs_data = validated_data.pop('applicationOutputs', None)
+ inputs_data = validated_data.pop('application_inputs', None)
+ outputs_data = validated_data.pop('application_outputs', None)
- for attr, value in validated_data.items():
- setattr(instance, attr, value)
+ for proto_field in ('application_interface_id', 'application_name',
+ 'application_description',
'archive_working_directory'):
+ if proto_field in validated_data:
+ value = validated_data[proto_field]
+ if proto_field == 'archive_working_directory':
+ value = bool(value)
+ else:
+ value = value or ''
+ setattr(instance, proto_field, value)
+ if 'application_modules' in validated_data:
+ instance.application_modules[:] = list(
+ validated_data['application_modules'] or [])
if inputs_data is not None:
- instance.applicationInputs = [InputDataObjectType(**inp) for inp
in inputs_data]
+ del instance.application_inputs[:]
+ instance.application_inputs.extend(
+ _proto_input_data_object(inp) for inp in inputs_data)
if outputs_data is not None:
- instance.applicationOutputs = [OutputDataObjectType(**out) for out
in outputs_data]
+ del instance.application_outputs[:]
+ instance.application_outputs.extend(
+ _proto_output_data_object(out) for out in outputs_data)
return instance
- def get_userHasWriteAccess(self, appDeployment):
+ def get_userHasWriteAccess(self, appInterface):
request = self.context['request']
return request.is_gateway_admin
@@ -1216,68 +1290,341 @@ class
GridFtpDataMovementSerializer(serializers.Serializer):
required=False)
-class ExperimentStatusSerializer(
- thrift_utils.create_serializer_class(ExperimentStatus)):
- timeOfStateChange = UTCPosixTimestampDateTimeField()
+def _status_pb2():
+ from airavata_sdk.generated.org.apache.airavata.model.status import
status_pb2
+ return status_pb2
+
+
+def _experiment_state_field(**kwargs):
+ from airavata.model.status.ttypes import ExperimentState as _T
+ return proto_enum_int_field(
+ _status_pb2().ExperimentState.DESCRIPTOR, _T,
+ proto_prefix='EXPERIMENT_STATE_', **kwargs)
-class ProcessStatusSerializer(
- thrift_utils.create_serializer_class(ProcessStatus)):
- timeOfStateChange = UTCPosixTimestampDateTimeField()
+def _process_state_field(**kwargs):
+ from airavata.model.status.ttypes import ProcessState as _T
+ return proto_enum_int_field(
+ _status_pb2().ProcessState.DESCRIPTOR, _T,
+ proto_prefix='PROCESS_STATE_', **kwargs)
-class ExperimentSerializer(
- thrift_utils.create_serializer_class(ExperimentModel)):
- class Meta:
- required = ('projectId', 'experimentType', 'experimentName')
- read_only = ('userName', 'gatewayId')
+def _task_state_field(**kwargs):
+ from airavata.model.status.ttypes import TaskState as _T
+ return proto_enum_int_field(
+ _status_pb2().TaskState.DESCRIPTOR, _T,
+ proto_prefix='TASK_STATE_', **kwargs)
+
+
+def _job_state_field(**kwargs):
+ from airavata.model.status.ttypes import JobState as _T
+ return proto_enum_int_field(
+ _status_pb2().JobState.DESCRIPTOR, _T,
+ proto_prefix='JOB_STATE_', **kwargs)
+
+
+class ExperimentStatusSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``ExperimentStatus`` message."""
+
+ state = _experiment_state_field(required=False, allow_null=True)
+ timeOfStateChange =
UTCPosixTimestampDateTimeField(source='time_of_state_change')
+ reason = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ statusId = serializers.CharField(source='status_id', allow_blank=True,
allow_null=True, required=False)
+
+
+class ProcessStatusSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``ProcessStatus`` message."""
+
+ state = _process_state_field(required=False, allow_null=True)
+ timeOfStateChange =
UTCPosixTimestampDateTimeField(source='time_of_state_change')
+ reason = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ statusId = serializers.CharField(source='status_id', allow_blank=True,
allow_null=True, required=False)
+ processId = serializers.CharField(source='process_id', allow_blank=True,
allow_null=True, required=False)
+
+
+class _NestedProcessStatusSerializer(serializers.Serializer):
+ """``ProcessStatus`` nested in the experiment tree (timeOfStateChange is
the
+ raw epoch-millis int, matching the old auto-generated field)."""
+
+ state = _process_state_field(required=False, allow_null=True)
+ timeOfStateChange =
serializers.IntegerField(source='time_of_state_change', allow_null=True,
required=False)
+ reason = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ statusId = serializers.CharField(source='status_id', allow_blank=True,
allow_null=True, required=False)
+ processId = serializers.CharField(source='process_id', allow_blank=True,
allow_null=True, required=False)
+
+
+class _TaskStatusSerializer(serializers.Serializer):
+ state = _task_state_field(required=False, allow_null=True)
+ timeOfStateChange =
serializers.IntegerField(source='time_of_state_change', allow_null=True,
required=False)
+ reason = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ statusId = serializers.CharField(source='status_id', allow_blank=True,
allow_null=True, required=False)
+
+
+class _JobStatusSerializer(serializers.Serializer):
+ jobState = _job_state_field(source='job_state', required=False,
allow_null=True)
+ timeOfStateChange =
serializers.IntegerField(source='time_of_state_change', allow_null=True,
required=False)
+ reason = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ statusId = serializers.CharField(source='status_id', allow_blank=True,
allow_null=True, required=False)
+
+
+class _ErrorModelSerializer(serializers.Serializer):
+ errorId = serializers.CharField(source='error_id', allow_blank=True,
allow_null=True, required=False)
+ creationTime = ProtoIntOrNoneField(source='creation_time')
+ actualErrorMessage = serializers.CharField(source='actual_error_message',
allow_blank=True, allow_null=True, required=False)
+ userFriendlyMessage =
serializers.CharField(source='user_friendly_message', allow_blank=True,
allow_null=True, required=False)
+ transientOrPersistent =
serializers.BooleanField(source='transient_or_persistent', required=False,
default=False)
+ rootCauseErrorIdList =
serializers.ListField(source='root_cause_error_id_list',
child=serializers.CharField(), required=False)
+
+
+class _ComputationalResourceSchedulingSerializer(serializers.Serializer):
+ resourceHostId = serializers.CharField(source='resource_host_id',
allow_blank=True, allow_null=True, required=False)
+ totalCPUCount = serializers.IntegerField(source='total_cpu_count',
allow_null=True, required=False)
+ nodeCount = serializers.IntegerField(source='node_count', allow_null=True,
required=False)
+ numberOfThreads = serializers.IntegerField(source='number_of_threads',
allow_null=True, required=False)
+ queueName = serializers.CharField(source='queue_name', allow_blank=True,
allow_null=True, required=False)
+ wallTimeLimit = serializers.IntegerField(source='wall_time_limit',
allow_null=True, required=False)
+ totalPhysicalMemory =
serializers.IntegerField(source='total_physical_memory', allow_null=True,
required=False)
+ chessisNumber = serializers.CharField(source='chessis_number',
allow_blank=True, allow_null=True, required=False)
+ staticWorkingDir = serializers.CharField(source='static_working_dir',
allow_blank=True, allow_null=True, required=False)
+ overrideLoginUserName =
serializers.CharField(source='override_login_user_name', allow_blank=True,
allow_null=True, required=False)
+ overrideScratchLocation =
serializers.CharField(source='override_scratch_location', allow_blank=True,
allow_null=True, required=False)
+ overrideAllocationProjectNumber =
serializers.CharField(source='override_allocation_project_number',
allow_blank=True, allow_null=True, required=False)
+ mGroupCount = serializers.IntegerField(source='m_group_count',
allow_null=True, required=False)
+ def create(self, validated_data):
+ from airavata_sdk.generated.org.apache.airavata.model.scheduling
import (
+ scheduling_pb2,
+ )
+ d = validated_data
+ return scheduling_pb2.ComputationalResourceSchedulingModel(
+ resource_host_id=d.get('resource_host_id', '') or '',
+ total_cpu_count=d.get('total_cpu_count', 0) or 0,
+ node_count=d.get('node_count', 0) or 0,
+ number_of_threads=d.get('number_of_threads', 0) or 0,
+ queue_name=d.get('queue_name', '') or '',
+ wall_time_limit=d.get('wall_time_limit', 0) or 0,
+ total_physical_memory=d.get('total_physical_memory', 0) or 0,
+ chessis_number=d.get('chessis_number', '') or '',
+ static_working_dir=d.get('static_working_dir', '') or '',
+ override_login_user_name=d.get('override_login_user_name', '') or
'',
+ override_scratch_location=d.get('override_scratch_location', '')
or '',
+
override_allocation_project_number=d.get('override_allocation_project_number',
'') or '',
+ m_group_count=d.get('m_group_count', 0) or 0,
+ )
+
+
+class _UserConfigurationDataSerializer(serializers.Serializer):
+ airavataAutoSchedule =
serializers.BooleanField(source='airavata_auto_schedule', required=False,
default=False)
+ overrideManualScheduledParams =
serializers.BooleanField(source='override_manual_scheduled_params',
required=False, default=False)
+ shareExperimentPublicly =
serializers.BooleanField(source='share_experiment_publicly', required=False,
default=False)
+ computationalResourceScheduling =
_ComputationalResourceSchedulingSerializer(
+ source='computational_resource_scheduling', required=False)
+ throttleResources = serializers.BooleanField(source='throttle_resources',
required=False, default=False)
+ userDN = serializers.CharField(source='user_dn', allow_blank=True,
allow_null=True, required=False)
+ generateCert = serializers.BooleanField(source='generate_cert',
required=False, default=False)
+ inputStorageResourceId =
serializers.CharField(source='input_storage_resource_id', allow_blank=True,
allow_null=True, required=False)
+ outputStorageResourceId =
serializers.CharField(source='output_storage_resource_id', allow_blank=True,
allow_null=True, required=False)
+ experimentDataDir = serializers.CharField(source='experiment_data_dir',
allow_blank=True, allow_null=True, required=False)
+ useUserCRPref = serializers.BooleanField(source='use_user_cr_pref',
required=False, default=False)
+ groupResourceProfileId =
serializers.CharField(source='group_resource_profile_id', allow_blank=True,
allow_null=True, required=False)
+ autoScheduledCompResourceSchedulingList =
_ComputationalResourceSchedulingSerializer(source='auto_scheduled_comp_resource_scheduling_list',
many=True, required=False)
+
+ def to_representation(self, ucd):
+ ret = super().to_representation(ucd)
+ # proto3 singular sub-message is always present; the old adapter
rendered
+ # computationalResourceScheduling only when HasField, else None.
+ if not ucd.HasField('computational_resource_scheduling'):
+ ret['computationalResourceScheduling'] = None
+ return ret
+
+
+class JobSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``JobModel`` message."""
+
+ jobId = serializers.CharField(source='job_id', allow_blank=True,
allow_null=True, required=False)
+ taskId = serializers.CharField(source='task_id', allow_blank=True,
allow_null=True, required=False)
+ processId = serializers.CharField(source='process_id', allow_blank=True,
allow_null=True, required=False)
+ jobDescription = serializers.CharField(source='job_description',
allow_blank=True, allow_null=True, required=False)
+ creationTime = ProtoTimestampField(source='creation_time',
null_if_zero=True)
+ jobStatuses = _JobStatusSerializer(source='job_statuses', many=True,
required=False)
+ computeResourceConsumed =
serializers.CharField(source='compute_resource_consumed', allow_blank=True,
allow_null=True, required=False)
+ jobName = serializers.CharField(source='job_name', allow_blank=True,
allow_null=True, required=False)
+ workingDir = serializers.CharField(source='working_dir', allow_blank=True,
allow_null=True, required=False)
+ stdOut = serializers.CharField(source='std_out', allow_blank=True,
allow_null=True, required=False)
+ stdErr = serializers.CharField(source='std_err', allow_blank=True,
allow_null=True, required=False)
+ exitCode = serializers.IntegerField(source='exit_code', allow_null=True,
required=False)
+
+
+class _NestedJobSerializer(JobSerializer):
+ """``JobModel`` nested in the process tree: ``creationTime`` is the raw
+ epoch-millis int (the old auto-generated field), unlike the standalone
+ ``JobSerializer`` (jobs action) which renders ISO."""
+
+ creationTime = ProtoIntOrNoneField(source='creation_time')
+
+
+class _TaskModelSerializer(serializers.Serializer):
+ taskId = serializers.CharField(source='task_id', allow_blank=True,
allow_null=True, required=False)
+ taskType = serializers.SerializerMethodField()
+ parentProcessId = serializers.CharField(source='parent_process_id',
allow_blank=True, allow_null=True, required=False)
+ creationTime = ProtoIntOrNoneField(source='creation_time')
+ lastUpdateTime = ProtoIntOrNoneField(source='last_update_time')
+ taskStatuses = _TaskStatusSerializer(source='task_statuses', many=True,
required=False)
+ taskDetail = serializers.CharField(source='task_detail', allow_blank=True,
allow_null=True, required=False)
+ subTaskModel = serializers.CharField(source='sub_task_model',
allow_blank=True, allow_null=True, required=False)
+ taskErrors = _ErrorModelSerializer(source='task_errors', many=True,
required=False)
+ jobs = _NestedJobSerializer(many=True, required=False)
+ maxRetry = serializers.IntegerField(source='max_retry', allow_null=True,
required=False)
+ currentRetry = serializers.IntegerField(source='current_retry',
allow_null=True, required=False)
+
+ def get_taskType(self, task):
+ from airavata.model.task.ttypes import TaskTypes as _T
+ from airavata_sdk.generated.org.apache.airavata.model.task import
task_pb2
+ field = proto_enum_int_field(
+ task_pb2.TaskTypes.DESCRIPTOR, _T, proto_prefix='TASK_TYPES_')
+ return field.to_representation(task.task_type)
+
+
+class _ProcessModelSerializer(serializers.Serializer):
+ processId = serializers.CharField(source='process_id', allow_blank=True,
allow_null=True, required=False)
+ experimentId = serializers.CharField(source='experiment_id',
allow_blank=True, allow_null=True, required=False)
+ creationTime = ProtoIntOrNoneField(source='creation_time')
+ lastUpdateTime = ProtoIntOrNoneField(source='last_update_time')
+ processStatuses =
_NestedProcessStatusSerializer(source='process_statuses', many=True,
required=False)
+ processDetail = serializers.CharField(source='process_detail',
allow_blank=True, allow_null=True, required=False)
+ applicationInterfaceId =
serializers.CharField(source='application_interface_id', allow_blank=True,
allow_null=True, required=False)
+ applicationDeploymentId =
serializers.CharField(source='application_deployment_id', allow_blank=True,
allow_null=True, required=False)
+ computeResourceId = serializers.CharField(source='compute_resource_id',
allow_blank=True, allow_null=True, required=False)
+ processInputs = InputDataObjectTypeSerializer(source='process_inputs',
many=True, required=False)
+ processOutputs = OutputDataObjectTypeSerializer(source='process_outputs',
many=True, required=False)
+ processResourceSchedule = serializers.SerializerMethodField()
+ tasks = _TaskModelSerializer(many=True, required=False)
+ taskDag = serializers.CharField(source='task_dag', allow_blank=True,
allow_null=True, required=False)
+ processErrors = _ErrorModelSerializer(source='process_errors', many=True,
required=False)
+ gatewayExecutionId = serializers.CharField(source='gateway_execution_id',
allow_blank=True, allow_null=True, required=False)
+ enableEmailNotification =
serializers.BooleanField(source='enable_email_notification', required=False,
default=False)
+ emailAddresses = serializers.ListField(source='email_addresses',
child=serializers.CharField(), required=False)
+ inputStorageResourceId =
serializers.CharField(source='input_storage_resource_id', allow_blank=True,
allow_null=True, required=False)
+ outputStorageResourceId =
serializers.CharField(source='output_storage_resource_id', allow_blank=True,
allow_null=True, required=False)
+ userDn = serializers.CharField(source='user_dn', allow_blank=True,
allow_null=True, required=False)
+ generateCert = serializers.BooleanField(source='generate_cert',
required=False, default=False)
+ experimentDataDir = serializers.CharField(source='experiment_data_dir',
allow_blank=True, allow_null=True, required=False)
+ userName = serializers.CharField(source='user_name', allow_blank=True,
allow_null=True, required=False)
+ useUserCRPref = serializers.BooleanField(source='use_user_cr_pref',
required=False, default=False)
+ groupResourceProfileId =
serializers.CharField(source='group_resource_profile_id', allow_blank=True,
allow_null=True, required=False)
+ processWorkflows = serializers.SerializerMethodField()
+
+ def get_processResourceSchedule(self, proc):
+ if not proc.HasField('process_resource_schedule'):
+ return None
+ return _ComputationalResourceSchedulingSerializer(
+ proc.process_resource_schedule, context=self.context).data
+
+ def get_processWorkflows(self, proc):
+ # The legacy workflow-engine subsystem is not adapted (rarely
populated);
+ # an empty list matches the Thrift default for non-workflow processes.
+ return []
+
+
+def _experiment_type_field(**kwargs):
+ from airavata.model.experiment.ttypes import ExperimentType as _T
+ from airavata_sdk.generated.org.apache.airavata.model.experiment import (
+ experiment_pb2,
+ )
+ return proto_enum_int_field(
+ experiment_pb2.ExperimentType.DESCRIPTOR, _T,
+ proto_prefix='EXPERIMENT_TYPE_', **kwargs)
+
+
+def _experiment_pb2():
+ from airavata_sdk.generated.org.apache.airavata.model.experiment import (
+ experiment_pb2,
+ )
+ return experiment_pb2
+
+
+class ExperimentSerializer(serializers.Serializer):
+ """Proto-native serializer for the gRPC ``ExperimentModel`` message.
+
+ The deepest read model: the processes -> tasks -> jobs tree (each with its
+ status list), user configuration + scheduling, inputs/outputs, status, and
+ errors. ``save()`` returns a proto ``ExperimentModel`` for the facade.
+ """
+
+ experimentId = serializers.CharField(source='experiment_id',
read_only=True)
+ projectId = serializers.CharField(source='project_id')
+ gatewayId = serializers.CharField(source='gateway_id', read_only=True)
+ experimentType = _experiment_type_field(source='experiment_type',
required=False, allow_null=True)
+ userName = serializers.CharField(source='user_name', read_only=True)
+ experimentName = serializers.CharField(source='experiment_name')
+ creationTime = ProtoTimestampField(source='creation_time',
null_if_zero=True, read_only=True)
+ description = serializers.CharField(allow_blank=True, allow_null=True,
required=False)
+ executionId = serializers.CharField(source='execution_id',
allow_blank=True, allow_null=True, required=False)
+ gatewayExecutionId = serializers.CharField(source='gateway_execution_id',
allow_blank=True, allow_null=True, required=False)
+ gatewayInstanceId = serializers.CharField(source='gateway_instance_id',
allow_blank=True, allow_null=True, required=False)
+ enableEmailNotification =
serializers.BooleanField(source='enable_email_notification', required=False,
default=False)
+ emailAddresses = serializers.ListField(source='email_addresses',
child=serializers.CharField(), required=False)
+ userConfigurationData = _UserConfigurationDataSerializer(
+ source='user_configuration_data', required=False)
+ experimentInputs = OrderedListField(
+ source='experiment_inputs', order_by='inputOrder',
+ child=InputDataObjectTypeSerializer(), required=False)
+ experimentOutputs = OutputDataObjectTypeSerializer(
+ source='experiment_outputs', many=True, required=False)
+ experimentStatus = ExperimentStatusSerializer(
+ source='experiment_status', many=True, required=False)
+ errors = _ErrorModelSerializer(many=True, required=False)
+ processes = _ProcessModelSerializer(many=True, required=False)
+ workflow = serializers.SerializerMethodField()
url = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:experiment-detail',
- lookup_field='experimentId',
- lookup_url_kwarg='experiment_id')
+ lookup_field='experiment_id', lookup_url_kwarg='experiment_id')
full_experiment = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:full-experiment-detail',
- lookup_field='experimentId',
- lookup_url_kwarg='experiment_id')
+ lookup_field='experiment_id', lookup_url_kwarg='experiment_id')
project = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:project-detail',
- lookup_field='projectId',
- lookup_url_kwarg='project_id')
+ lookup_field='project_id', lookup_url_kwarg='project_id')
jobs = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:experiment-jobs',
- lookup_field='experimentId',
- lookup_url_kwarg='experiment_id')
+ lookup_field='experiment_id', lookup_url_kwarg='experiment_id')
shared_entity = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:shared-entity-detail',
- lookup_field='experimentId',
- lookup_url_kwarg='entity_id')
- experimentInputs = OrderedListField(
- order_by='inputOrder',
- child=InputDataObjectTypeSerializer(),
- allow_null=True)
- experimentOutputs = serializers.ListField(
- child=OutputDataObjectTypeSerializer(),
- allow_null=True)
- creationTime = UTCPosixTimestampDateTimeField(allow_null=True)
- experimentStatus = ExperimentStatusSerializer(many=True, allow_null=True)
+ lookup_field='experiment_id', lookup_url_kwarg='entity_id')
userHasWriteAccess = serializers.SerializerMethodField()
+ def get_workflow(self, experiment):
+ # The legacy workflow-engine subsystem is not adapted (rarely
populated).
+ return None
+
def get_userHasWriteAccess(self, experiment):
- return user_has_access(self.context['request'],
experiment.experimentId)
+ return user_has_access(self.context['request'],
experiment.experiment_id)
def to_representation(self, experiment):
result = super().to_representation(experiment)
+ # proto3 singular sub-message is always present; the old adapter
rendered
+ # userConfigurationData only when HasField, else None.
+ if not experiment.HasField('user_configuration_data'):
+ result['userConfigurationData'] = None
self._add_intermediate_output_information(experiment, result)
return result
+ def create(self, validated_data):
+ return _experiment_request(validated_data)
+
+ def update(self, instance, validated_data):
+ return _experiment_request(validated_data)
+
def _add_intermediate_output_information(self, experiment, representation):
request = self.context['request']
-
+ from airavata_sdk.generated.org.apache.airavata.model.status import (
+ status_pb2,
+ )
# If experiment is EXECUTING, add intermediateOutput information to
- # experiment outputs
- if (experiment.experimentStatus and
- experiment.experimentStatus[-1].state ==
ExperimentState.EXECUTING):
+ # experiment outputs.
+ if (experiment.experiment_status and
+ experiment.experiment_status[-1].state ==
+ status_pb2.ExperimentState.EXPERIMENT_STATE_EXECUTING):
for output in representation["experimentOutputs"]:
output["intermediateOutput"] = {"processStatus": None}
try:
@@ -1298,6 +1645,60 @@ class ExperimentSerializer(
log.debug("Failed to get intermediate output status",
exc_info=True)
+def _experiment_request(validated_data):
+ """Build a proto ``ExperimentModel`` from validated write data.
+
+ Only the user-submittable fields are carried; status / errors / processes /
+ workflow are server-managed (the old write adapter dropped them too).
+ """
+ e = _experiment_pb2()
+ d = validated_data
+ ucd = d.get('user_configuration_data')
+ kwargs = dict(
+ experiment_id=d.get('experiment_id', '') or '',
+ project_id=d.get('project_id', '') or '',
+ gateway_id=d.get('gateway_id', '') or '',
+ experiment_type=d.get('experiment_type', 0) or 0,
+ user_name=d.get('user_name', '') or '',
+ experiment_name=d.get('experiment_name', '') or '',
+ description=d.get('description', '') or '',
+ execution_id=d.get('execution_id', '') or '',
+ enable_email_notification=bool(d.get('enable_email_notification',
False)),
+ email_addresses=list(d.get('email_addresses', []) or []),
+ experiment_inputs=[
+ _proto_input_data_object(i)
+ for i in d.get('experiment_inputs', []) or []],
+ experiment_outputs=[
+ _proto_output_data_object(o)
+ for o in d.get('experiment_outputs', []) or []],
+ )
+ if ucd is not None:
+ kwargs['user_configuration_data'] = _user_configuration_request(ucd)
+ return e.ExperimentModel(**kwargs)
+
+
+def _user_configuration_request(d):
+ e = _experiment_pb2()
+ crs = d.get('computational_resource_scheduling')
+ kwargs = dict(
+ airavata_auto_schedule=bool(d.get('airavata_auto_schedule', False)),
+
override_manual_scheduled_params=bool(d.get('override_manual_scheduled_params',
False)),
+ share_experiment_publicly=bool(d.get('share_experiment_publicly',
False)),
+ throttle_resources=bool(d.get('throttle_resources', False)),
+ user_dn=d.get('user_dn', '') or '',
+ generate_cert=bool(d.get('generate_cert', False)),
+ input_storage_resource_id=d.get('input_storage_resource_id', '') or '',
+ output_storage_resource_id=d.get('output_storage_resource_id', '') or
'',
+ experiment_data_dir=d.get('experiment_data_dir', '') or '',
+ use_user_cr_pref=bool(d.get('use_user_cr_pref', False)),
+ group_resource_profile_id=d.get('group_resource_profile_id', '') or '',
+ )
+ if crs is not None:
+ kwargs['computational_resource_scheduling'] = (
+ _ComputationalResourceSchedulingSerializer().create(crs))
+ return e.UserConfigurationDataModel(**kwargs)
+
+
class DataReplicaLocationSerializer(
thrift_utils.create_serializer_class(DataReplicaLocationModel)):
creationTime = UTCPosixTimestampDateTimeField()
@@ -1373,7 +1774,7 @@ class FullExperiment:
inputDataProducts=None, applicationModule=None,
computeResource=None, jobDetails=None, outputViews=None):
self.experiment = experimentModel
- self.experimentId = experimentModel.experimentId
+ self.experimentId = experimentModel.experiment_id
self.project = project
self.outputDataProducts = outputDataProducts
self.inputDataProducts = inputDataProducts
@@ -1383,10 +1784,6 @@ class FullExperiment:
self.outputViews = outputViews
-class JobSerializer(thrift_utils.create_serializer_class(JobModel)):
- creationTime = UTCPosixTimestampDateTimeField()
-
-
class FullExperimentSerializer(serializers.Serializer):
url = FullyEncodedHyperlinkedIdentityField(
view_name='django_airavata_api:full-experiment-detail',
diff --git a/airavata-django-portal/django_airavata/apps/api/views.py
b/airavata-django-portal/django_airavata/apps/api/views.py
index 327caad6d..509a0617e 100644
--- a/airavata-django-portal/django_airavata/apps/api/views.py
+++ b/airavata-django-portal/django_airavata/apps/api/views.py
@@ -7,9 +7,7 @@ import warnings
from datetime import datetime, timedelta
from urllib.parse import quote
-from airavata.model.application.io.ttypes import DataType
from airavata.model.experiment.ttypes import (
- ExperimentModel,
ExperimentSearchFields
)
from airavata.model.appcatalog.groupresourceprofile.ttypes import (
@@ -72,6 +70,13 @@ TMP_INPUT_FILE_UPLOAD_DIR = "tmp"
log = logging.getLogger(__name__)
+def _data_type_pb2():
+ from airavata_sdk.generated.org.apache.airavata.model.application.io
import (
+ application_io_pb2,
+ )
+ return application_io_pb2
+
+
def _storage_upload_and_register(request, dir_path, uploaded_file, name=None,
content_type=None, experiment_id=None):
"""Upload a file to user storage and register a data product for it (gRPC).
@@ -217,11 +222,9 @@ class ProjectViewSet(APIBackedViewSet):
@action(detail=True)
def experiments(self, request, project_id=None):
- experiments = [
- grpc_adapters.experiment(e)
- for e in request.airavata.research.get_experiments_in_project(
- project_id, -1, 0)
- ]
+ experiments = list(
+ request.airavata.research.get_experiments_in_project(
+ project_id, -1, 0))
serializer = serializers.ExperimentSerializer(
experiments, many=True, context={'request': request})
return Response(serializer.data)
@@ -240,41 +243,39 @@ class ExperimentViewSet(mixins.CreateModelMixin,
lookup_field = 'experiment_id'
def get_instance(self, lookup_value):
- return grpc_adapters.experiment(
- self.request.airavata.research.get_experiment(lookup_value))
+ return self.request.airavata.research.get_experiment(lookup_value)
def perform_create(self, serializer):
experiment = serializer.save(
- gatewayId=self.gateway_id,
- userName=self.username)
+ gateway_id=self.gateway_id,
+ user_name=self.username)
experiment_id = self.request.airavata.research.create_experiment(
- self.gateway_id, grpc_requests.experiment(experiment))
+ self.gateway_id, experiment)
self._update_workspace_preferences(
- project_id=experiment.projectId,
-
group_resource_profile_id=experiment.userConfigurationData.groupResourceProfileId,
-
compute_resource_id=experiment.userConfigurationData.computationalResourceScheduling.resourceHostId)
- experiment.experimentId = experiment_id
+ project_id=experiment.project_id,
+
group_resource_profile_id=experiment.user_configuration_data.group_resource_profile_id,
+
compute_resource_id=experiment.user_configuration_data.computational_resource_scheduling.resource_host_id)
+ experiment.experiment_id = experiment_id
def perform_update(self, serializer):
experiment = serializer.save(
- gatewayId=self.gateway_id,
- userName=self.username)
+ gateway_id=self.gateway_id,
+ user_name=self.username)
self.request.airavata.research.update_experiment(
- experiment.experimentId, grpc_requests.experiment(experiment))
+ experiment.experiment_id, experiment)
self._update_workspace_preferences(
- project_id=experiment.projectId,
-
group_resource_profile_id=experiment.userConfigurationData.groupResourceProfileId,
-
compute_resource_id=experiment.userConfigurationData.computationalResourceScheduling.resourceHostId)
+ project_id=experiment.project_id,
+
group_resource_profile_id=experiment.user_configuration_data.group_resource_profile_id,
+
compute_resource_id=experiment.user_configuration_data.computational_resource_scheduling.resource_host_id)
@action(methods=['post'], detail=True)
def launch(self, request, experiment_id=None):
try:
- experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(experiment_id))
- if (experiment.enableEmailNotification):
- experiment.emailAddresses = [request.user.email]
+ experiment =
request.airavata.research.get_experiment(experiment_id)
+ if experiment.enable_email_notification:
+ experiment.email_addresses[:] = [request.user.email]
request.airavata.research.update_experiment(
- experiment_id, grpc_requests.experiment(experiment))
+ experiment_id, experiment)
experiment_util.launch(request, experiment_id)
return Response({'success': True})
except Exception as e:
@@ -283,21 +284,18 @@ class ExperimentViewSet(mixins.CreateModelMixin,
@action(methods=['get'], detail=True)
def jobs(self, request, experiment_id=None):
- jobs = [
- grpc_adapters.job_model(j)
- for j in request.airavata.research.get_job_details(experiment_id)
- ]
+ jobs = list(request.airavata.research.get_job_details(experiment_id))
serializer = serializers.JobSerializer(
jobs, many=True, context={'request': request})
return Response(serializer.data)
@action(methods=['post'], detail=True)
def clone(self, request, experiment_id=None):
- # experiment_util.clone is the launch/clone orchestration (still
Thrift);
- # re-fetch the cloned experiment via gRPC.
+ # experiment_util.clone is the launch/clone orchestration (still on the
+ # legacy SDK); re-fetch the cloned experiment via gRPC.
cloned_experiment_id = experiment_util.clone(request, experiment_id)
- cloned_experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(cloned_experiment_id))
+ cloned_experiment = request.airavata.research.get_experiment(
+ cloned_experiment_id)
serializer = self.serializer_class(
cloned_experiment, context={'request': request})
return Response(serializer.data)
@@ -370,28 +368,27 @@ class FullExperimentViewSet(mixins.RetrieveModelMixin,
def get_instance(self, lookup_value):
"""Get FullExperiment instance with resolved references."""
# TODO: move loading experiment and references to airavata_sdk?
- experimentModel = grpc_adapters.experiment(
- self.request.airavata.research.get_experiment(lookup_value))
+ experimentModel = self.request.airavata.research.get_experiment(
+ lookup_value)
+ DT = _data_type_pb2().DataType
outputDataProducts = [
grpc_adapters.data_product(
self.request.airavata.research.get_data_product(output.value))
- for output in experimentModel.experimentOutputs
+ for output in experimentModel.experiment_outputs
if (output.value and
output.value.startswith('airavata-dp') and
- output.type in (DataType.URI,
- DataType.STDOUT,
- DataType.STDERR))]
+ output.type in (DT.URI, DT.STDOUT, DT.STDERR))]
outputDataProducts += [
grpc_adapters.data_product(
self.request.airavata.research.get_data_product(dp))
- for output in experimentModel.experimentOutputs
+ for output in experimentModel.experiment_outputs
if (output.value and
- output.type == DataType.URI_COLLECTION)
+ output.type == DT.URI_COLLECTION)
for dp in output.value.split(',')
if output.value.startswith('airavata-dp')]
- appInterfaceId = experimentModel.executionId
+ appInterfaceId = experimentModel.execution_id
try:
- applicationInterface = grpc_adapters.application_interface(
+ applicationInterface = (
self.request.airavata.research.get_application_interface(
appInterfaceId))
except Exception as e:
@@ -402,24 +399,22 @@ class FullExperimentViewSet(mixins.RetrieveModelMixin,
inputDataProducts = [
grpc_adapters.data_product(
self.request.airavata.research.get_data_product(inp.value))
- for inp in experimentModel.experimentInputs
+ for inp in experimentModel.experiment_inputs
if (inp.value and
inp.value.startswith('airavata-dp') and
- inp.type in (DataType.URI,
- DataType.STDOUT,
- DataType.STDERR))]
+ inp.type in (DT.URI, DT.STDOUT, DT.STDERR))]
inputDataProducts += [
grpc_adapters.data_product(
self.request.airavata.research.get_data_product(dp))
- for inp in experimentModel.experimentInputs
+ for inp in experimentModel.experiment_inputs
if (inp.value and
- inp.type == DataType.URI_COLLECTION)
+ inp.type == DT.URI_COLLECTION)
for dp in inp.value.split(',')
if inp.value.startswith('airavata-dp')]
applicationModule = None
try:
if applicationInterface is not None:
- appModuleId = applicationInterface.applicationModules[0]
+ appModuleId = applicationInterface.application_modules[0]
applicationModule = (
self.request.airavata.research.get_application_module(
appModuleId))
@@ -430,10 +425,11 @@ class FullExperimentViewSet(mixins.RetrieveModelMixin,
log.exception("Failed to load app interface/module",
extra={'request': self.request})
compute_resource_id = None
- user_conf = experimentModel.userConfigurationData
- if user_conf and user_conf.computationalResourceScheduling:
- comp_res_sched = user_conf.computationalResourceScheduling
- compute_resource_id = comp_res_sched.resourceHostId
+ if experimentModel.HasField('user_configuration_data'):
+ user_conf = experimentModel.user_configuration_data
+ if user_conf.HasField('computational_resource_scheduling'):
+ compute_resource_id = (
+
user_conf.computational_resource_scheduling.resource_host_id)
try:
compute_resource = (
self.request.airavata.compute.get_compute_resource(
@@ -444,15 +440,14 @@ class FullExperimentViewSet(mixins.RetrieveModelMixin,
compute_resource_id), extra={'request': self.request})
compute_resource = None
if serializers.user_has_access(
- self.request, experimentModel.projectId, 'READ'):
+ self.request, experimentModel.project_id, 'READ'):
project = self.request.airavata.research.get_project(
- experimentModel.projectId)
+ experimentModel.project_id)
else:
# User may not have access to project, only experiment
project = None
- job_details = [
- grpc_adapters.job_model(j)
- for j in
self.request.airavata.research.get_job_details(lookup_value)]
+ job_details = list(
+ self.request.airavata.research.get_job_details(lookup_value))
full_experiment = serializers.FullExperiment(
experimentModel,
project=project,
@@ -494,15 +489,14 @@ class ApplicationModuleViewSet(APIBackedViewSet):
@action(detail=True)
def application_interface(self, request, app_module_id):
- all_app_interfaces = [
- grpc_adapters.application_interface(i)
- for i in request.airavata.research.get_all_application_interfaces(
- self.gateway_id)]
+ all_app_interfaces = list(
+ request.airavata.research.get_all_application_interfaces(
+ self.gateway_id))
app_interfaces = []
for app_interface in all_app_interfaces:
- if not app_interface.applicationModules:
+ if not app_interface.application_modules:
continue
- if app_module_id in app_interface.applicationModules:
+ if app_module_id in app_interface.application_modules:
app_interfaces.append(app_interface)
if len(app_interfaces) == 1:
serializer = serializers.ApplicationInterfaceDescriptionSerializer(
@@ -581,17 +575,14 @@ class ApplicationInterfaceViewSet(APIBackedViewSet):
lookup_field = 'app_interface_id'
def get_list(self):
- return [
- grpc_adapters.application_interface(i)
- for i in
self.request.airavata.research.get_all_application_interfaces(
- self.gateway_id)
- ]
+ return list(
+ self.request.airavata.research.get_all_application_interfaces(
+ self.gateway_id))
def get_instance(self, lookup_value):
try:
- return grpc_adapters.application_interface(
- self.request.airavata.research.get_application_interface(
- lookup_value))
+ return self.request.airavata.research.get_application_interface(
+ lookup_value)
except Exception:
# If it failed to load, check to see if it exists at all
all_interfaces =
self.request.airavata.research.get_all_application_interfaces(
@@ -607,24 +598,24 @@ class ApplicationInterfaceViewSet(APIBackedViewSet):
self._update_input_metadata(application_interface)
log.debug("application_interface: {}".format(application_interface))
app_interface_id =
self.request.airavata.research.register_application_interface(
- self.gateway_id,
grpc_requests.application_interface(application_interface))
- application_interface.applicationInterfaceId = app_interface_id
+ self.gateway_id, application_interface)
+ application_interface.application_interface_id = app_interface_id
def perform_update(self, serializer):
application_interface = serializer.save()
self._update_input_metadata(application_interface)
self.request.airavata.research.update_application_interface(
- application_interface.applicationInterfaceId,
- grpc_requests.application_interface(application_interface))
+ application_interface.application_interface_id,
+ application_interface)
def perform_destroy(self, instance):
self.request.airavata.research.delete_application_interface(
- instance.applicationInterfaceId)
+ instance.application_interface_id)
def _update_input_metadata(self, app_interface):
- for app_input in app_interface.applicationInputs:
- if app_input.metaData:
- metadata = json.loads(app_input.metaData)
+ for app_input in app_interface.application_inputs:
+ if app_input.meta_data:
+ metadata = json.loads(app_input.meta_data)
# Automatically add {showOptions: {isRequired: true/false}} to
# toggle isRequired on hidden/shown inputs
if ("editor" in metadata and
@@ -633,8 +624,8 @@ class ApplicationInterfaceViewSet(APIBackedViewSet):
if "showOptions" not in metadata["editor"]["dependencies"]:
metadata["editor"]["dependencies"]["showOptions"] = {}
o = metadata["editor"]["dependencies"]["showOptions"]
- o["isRequired"] = app_input.isRequired
- app_input.metaData = json.dumps(metadata)
+ o["isRequired"] = app_input.is_required
+ app_input.meta_data = json.dumps(metadata)
@action(detail=True)
def compute_resources(self, request, app_interface_id):
@@ -1492,13 +1483,12 @@ class CurrentGatewayResourceProfile(APIView):
class ExperimentArchiveView(APIView):
def get(self, request, experiment_id=None, format=None):
- experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(experiment_id))
+ experiment = request.airavata.research.get_experiment(experiment_id)
result = dict(archived=False, archive_name=None, created_date=None,
max_age=settings.GATEWAY_USER_DATA_ARCHIVE_MAX_AGE_DAYS)
try:
archive_entry = UserDataArchiveEntry.objects.get(
- entry_path=experiment.userConfigurationData.experimentDataDir,
+
entry_path=experiment.user_configuration_data.experiment_data_dir,
user_data_archive__rolled_back=False)
result["archived"] = True
result["archive_name"] =
archive_entry.user_data_archive.archive_name
@@ -1590,10 +1580,10 @@ def _user_storage_path(path, experiment_id=None,
request=None):
"""
rel = (path or "").lstrip("/")
if experiment_id:
- experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(experiment_id))
- data_dir = (experiment.userConfigurationData.experimentDataDir
- if experiment.userConfigurationData else None) or ""
+ experiment = request.airavata.research.get_experiment(experiment_id)
+ data_dir = (experiment.user_configuration_data.experiment_data_dir
+ if experiment.HasField('user_configuration_data')
+ else None) or ""
base = data_dir.rstrip("/")
full = base + ("/" + rel if rel else "")
return full if (full.startswith("/") or full.startswith("~/")) else
"~/" + full
diff --git a/airavata-django-portal/django_airavata/apps/workspace/views.py
b/airavata-django-portal/django_airavata/apps/workspace/views.py
index a53711ff5..64120977d 100644
--- a/airavata-django-portal/django_airavata/apps/workspace/views.py
+++ b/airavata-django-portal/django_airavata/apps/workspace/views.py
@@ -132,11 +132,10 @@ def create_experiment(request, app_module_id):
def edit_experiment(request, experiment_id):
request.active_nav_item = 'experiments'
- experiment = grpc_adapters.experiment(
- request.airavata.research.get_experiment(experiment_id))
- applicationInterface = grpc_adapters.application_interface(
-
request.airavata.research.get_application_interface(experiment.executionId))
- app_module_id = applicationInterface.applicationModules[0]
+ experiment = request.airavata.research.get_experiment(experiment_id)
+ applicationInterface = request.airavata.research.get_application_interface(
+ experiment.execution_id)
+ app_module_id = applicationInterface.application_modules[0]
context = {
'bundle_name': 'edit-experiment',
'experiment_id': experiment_id,
