This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 08ef4ae050a59ac8137c25f88ea3ad2d778dbf50 Author: Marcus Christie <[email protected]> AuthorDate: Tue Jan 9 09:49:26 2018 -0500 AIRAVATA-2598 Implement saving experiment thrift_utils updated to support * required fields * read_only fields * nested structs and lists of structs --- django_airavata/apps/api/serializers.py | 32 ++++++------- .../api/static/django_airavata_api/js/index.js | 2 + .../models/ComputationalResourceSchedulingModel.js | 30 ++++++++++-- .../django_airavata_api/js/models/Experiment.js | 37 ++++++++++----- .../js/models/InputDataObjectType.js | 20 +++++++- .../django_airavata_api/js/models/ProcessModel.js | 13 ++++++ .../js/models/UserConfigurationData.js | 54 ++++++++++++++++++---- .../js/services/ExperimentService.js | 30 ++++++++++++ django_airavata/apps/api/thrift_utils.py | 51 ++++++++++++++++---- django_airavata/apps/api/urls.py | 1 + django_airavata/apps/api/views.py | 32 +++++++++---- .../js/components/experiment/ExperimentEditor.vue | 27 ++++++++--- .../js/containers/CreateExperimentContainer.vue | 1 + 13 files changed, 261 insertions(+), 69 deletions(-) diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py index 28b0124..511d080 100644 --- a/django_airavata/apps/api/serializers.py +++ b/django_airavata/apps/api/serializers.py @@ -103,23 +103,6 @@ class ProjectSerializer(serializers.Serializer): return instance -class ExperimentSerializer(serializers.Serializer): - - experimentId = serializers.CharField(read_only=True) - projectId = serializers.CharField(required=True) - project = FullyEncodedHyperlinkedIdentityField(view_name='django_airavata_api:project-detail', lookup_field='projectId', lookup_url_kwarg='project_id') - gatewayId = GatewayIdDefaultField() - experimentType = serializers.CharField(required=True) - userName = GatewayUsernameDefaultField() - experimentName = serializers.CharField(required=True) - - def create(self, validated_data): - return ExperimentModel(**validated_data) - - def update(self, instance, validated_data): - raise Exception("Not implemented") - - class ApplicationModuleSerializer(serializers.Serializer): url = FullyEncodedHyperlinkedIdentityField(view_name='django_airavata_api:application-detail', lookup_field='appModuleId', lookup_url_kwarg='app_module_id') appModuleId = serializers.CharField(required=True) @@ -243,5 +226,20 @@ class ComputeResourceDescriptionSerializer(CustomSerializer): resourceDescription=serializers.CharField() enabled=serializers.BooleanField() + class BatchQueueSerializer(thrift_utils.create_serializer_class(BatchQueue)): pass + + +class ExperimentSerializer( + thrift_utils.create_serializer_class(ExperimentModel)): + + class Meta: + required = ('projectId', 'experimentType', 'experimentName') + read_only = ('experimentId',) + + url = FullyEncodedHyperlinkedIdentityField(view_name='django_airavata_api:experiment-detail', lookup_field='experimentId', lookup_url_kwarg='experiment_id') + project = FullyEncodedHyperlinkedIdentityField(view_name='django_airavata_api:project-detail', lookup_field='projectId', lookup_url_kwarg='project_id') + userName = GatewayUsernameDefaultField() + gatewayId = GatewayIdDefaultField() + creationTime = UTCPosixTimestampDateTimeField(allow_null=True) diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js index fc3b6bc..8860efb 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/index.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js @@ -9,6 +9,7 @@ import Project from './models/Project' import ApplicationDeploymentService from './services/ApplicationDeploymentService' import ApplicationInterfaceService from './services/ApplicationInterfaceService' import ApplicationModuleService from './services/ApplicationModuleService' +import ExperimentService from './services/ExperimentService' import ProjectService from './services/ProjectService' import FetchUtils from './utils/FetchUtils' @@ -27,6 +28,7 @@ exports.services = { ApplicationDeploymentService, ApplicationInterfaceService, ApplicationModuleService, + ExperimentService, ProjectService, } diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js index f9cc61f..03c4782 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js @@ -8,11 +8,31 @@ const FIELDS = [ 'queueName', 'wallTimeLimit', 'totalPhysicalMemory', - 'chessisNumber', - 'staticWorkingDir', - 'overrideLoginUserName', - 'overrideScratchLocation', - 'overrideAllocationProjectNumber', + { + name: 'chessisNumber', + type: 'string', + default: '', + }, + { + name: 'staticWorkingDir', + type: 'string', + default: '', + }, + { + name: 'overrideLoginUserName', + type: 'string', + default: '', + }, + { + name: 'overrideScratchLocation', + type: 'string', + default: '', + }, + { + name: 'overrideAllocationProjectNumber', + type: 'string', + default: '', + }, ]; export default class ComputationalResourceSchedulingModel extends BaseModel { diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/Experiment.js b/django_airavata/apps/api/static/django_airavata_api/js/models/Experiment.js index 53aa4d2..89404c8 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/Experiment.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/Experiment.js @@ -1,24 +1,38 @@ + import BaseModel from './BaseModel'; -import UserConfigurationData from './UserConfigurationData' +import ErrorModel from './ErrorModel' +import ExperimentStatus from './ExperimentStatus' import InputDataObjectType from './InputDataObjectType' import OutputDataObjectType from './OutputDataObjectType' -import ExperimentStatus from './ExperimentStatus' -import ErrorModel from './ErrorModel' +import ProcessModel from './ProcessModel' +import UserConfigurationData from './UserConfigurationData' const FIELDS = [ 'experimentId', 'projectId', 'gatewayId', - 'experimentType', + { + name: 'experimentType', + type: 'number', + default: 0, + }, 'userName', 'experimentName', { name: 'creationTime', type: 'date' }, - 'description', + { + name: 'description', + type: 'string', + default: '', + }, 'executionId', - 'enableEmailNotification', + { + name: 'enableEmailNotification', + type: 'boolean', + default: false, + }, { name: 'emailAddresses', type: 'string', @@ -49,12 +63,11 @@ const FIELDS = [ type: ErrorModel, list: true, }, - // TODO: map the ProcessModel - // { - // name: 'processes', - // type: ProcessModel, - // list: true, - // }, + { + name: 'processes', + type: ProcessModel, + list: true, + }, ]; export default class Experiment extends BaseModel { diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js index 601fdd0..cf3e349 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js @@ -8,17 +8,33 @@ const FIELDS = [ 'applicationArgument', 'standardInput', 'userFriendlyDescription', - 'metaData', + { + name: 'metaData', + type: 'string', + default: '', + }, 'inputOrder', 'isRequired', 'requiredToAddedToCommandLine', 'dataStaged', - 'storageResourceId', + { + name: 'storageResourceId', + type: 'string', + default: '', + }, 'isReadOnly', ]; export default class InputDataObjectType extends BaseModel { constructor(data = {}) { super(FIELDS, data); + // TODO: move into BaseModel + // Convert null strings into empty strings + if ('metaData' in this && this.metaData === null) { + this.metaData = ''; + } + if ('storageResourceId' in this && this.storageResourceId === null) { + this.storageResourceId = ''; + } } } diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessModel.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessModel.js new file mode 100644 index 0000000..ee37636 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ProcessModel.js @@ -0,0 +1,13 @@ +import BaseModel from './BaseModel'; + +const FIELDS = [ + 'processId', + 'experimentId', + // TODO: finish mapping fields +]; + +export default class ProcessModel extends BaseModel { + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/UserConfigurationData.js b/django_airavata/apps/api/static/django_airavata_api/js/models/UserConfigurationData.js index e1a5a47..f70ffdf 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/UserConfigurationData.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/UserConfigurationData.js @@ -2,20 +2,56 @@ import BaseModel from './BaseModel'; import ComputationalResourceSchedulingModel from './ComputationalResourceSchedulingModel' const FIELDS = [ - 'airavataAutoSchedule', - 'overrideManualScheduledParams', - 'shareExperimentPublicly', + { + name: 'airavataAutoSchedule', + type: 'boolean', + default: false, + }, + { + name: 'overrideManualScheduledParams', + type: 'boolean', + default: false, + }, + { + name: 'shareExperimentPublicly', + type: 'boolean', + default: false, + }, { name: 'computationalResourceScheduling', type: ComputationalResourceSchedulingModel, default: BaseModel.defaultNewInstance(ComputationalResourceSchedulingModel), }, - 'throttleResources', - 'userDN', - 'generateCert', - 'storageId', - 'experimentDataDir', - 'useUserCRPref', + { + name: 'throttleResources', + type: 'boolean', + default: false, + }, + { + name: 'userDN', + type: 'string', + default: '', + }, + { + name: 'generateCert', + type: 'boolean', + default: false, + }, + { + name: 'storageId', + type: 'string', + default: '', + }, + { + name: 'experimentDataDir', + type: 'string', + default: '', + }, + { + name: 'useUserCRPref', + type: 'boolean', + default: false, + } ]; export default class UserConfigurationData extends BaseModel { diff --git a/django_airavata/apps/api/static/django_airavata_api/js/services/ExperimentService.js b/django_airavata/apps/api/static/django_airavata_api/js/services/ExperimentService.js new file mode 100644 index 0000000..b840b86 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/services/ExperimentService.js @@ -0,0 +1,30 @@ + +import Experiment from '../models/Experiment' +import FetchUtils from '../utils/FetchUtils' + +class ExperimentService { + list(data = null) { + if (data) { + return Promise.resolve(data.map(result => new Experiment(result))); + } else { + return FetchUtils.get('/api/experiments/') + .then(results => results.map(result => new Experiment(result))); + } + } + + create(experiment) { + return FetchUtils.post('/api/experiments/', JSON.stringify(experiment)) + .then(result => new Experiment(result)); + } + + update() { + // TODO + } + + get() { + // TODO + } +} + +// Export as a singleton +export default new ExperimentService(); \ No newline at end of file diff --git a/django_airavata/apps/api/thrift_utils.py b/django_airavata/apps/api/thrift_utils.py index 5bccf0e..476cfbf 100644 --- a/django_airavata/apps/api/thrift_utils.py +++ b/django_airavata/apps/api/thrift_utils.py @@ -33,11 +33,17 @@ def create_serializer_class(thrift_data_type): class CustomSerializerMeta(SerializerMetaclass): def __new__(cls, name, bases, attrs): + meta = attrs.get('Meta', None) thrift_spec = thrift_data_type.thrift_spec for field in thrift_spec: # Don't replace existing attrs to allow subclasses to override if field and field[2] not in attrs: - field_serializer = process_field(field) + required = field[2] in meta.required if meta else False + read_only = field[2] in meta.read_only if meta else False + allow_null = not required + field_serializer = process_field( + field, required=required, read_only=read_only, + allow_null=allow_null) attrs[field[2]] = field_serializer return super().__new__(cls, name, bases, attrs) @@ -47,16 +53,22 @@ def create_serializer_class(thrift_data_type): Custom Serializer which handle the list fields which holds custom class objects """ - def process_list_fields(self, validated_data): + def process_nested_fields(self, validated_data): fields = self.fields params = copy.deepcopy(validated_data) for field_name, serializer in fields.items(): if isinstance(serializer, ListField): - params[field_name] = serializer.to_representation(params[field_name]) + if (params[field_name] is not None or not serializer.allow_null): + if isinstance(serializer.child, Serializer): + params[field_name] = [serializer.child.create(item) for item in params[field_name]] + else: + params[field_name] = serializer.to_representation(params[field_name]) + elif isinstance(serializer, Serializer): + params[field_name] = serializer.create(params[field_name]) return params def create(self, validated_data): - params = self.process_list_fields(validated_data) + params = self.process_nested_fields(validated_data) return thrift_data_type(**params) def update(self, instance, validated_data): @@ -65,22 +77,40 @@ def create_serializer_class(thrift_data_type): return CustomSerializer -def process_field(field): +def process_field(field, required=False, read_only=False, allow_null=False): """ Used to process a thrift data type field :param field: + :param required: + :param read_only: + :param allow_null: :return: """ if field[1] in mapping: - # handling scenarios when the thrift field type is present in the mapping - return mapping[field[1]](required=False) + # handling scenarios when the thrift field type is present in the + # mapping + field_class = mapping[field[1]] + kwargs = dict(required=required, read_only=read_only) + # allow_null isn't allowed for BooleanField and we'll use allow_blank + # for CharField + if field_class not in (BooleanField, CharField): + kwargs['allow_null'] = allow_null + if field_class == CharField: + kwargs['allow_blank'] = allow_null + return mapping[field[1]](**kwargs) elif field[1] == TType.LIST: # handling scenario when the thrift field type is list list_field_serializer = process_list_field(field) - return ListField(child=list_field_serializer, required=False) + return ListField(child=list_field_serializer, + required=required, + read_only=read_only, + allow_null=allow_null) elif field[1] == TType.STRUCT: # handling scenario when the thrift field type is struct - return create_serializer(field[3][0]) + return create_serializer(field[3][0], + required=required, + read_only=read_only, + allow_null=allow_null) def process_list_field(field): @@ -91,7 +121,8 @@ def process_list_field(field): """ list_details = field[3] if list_details[0] in mapping: - # handling scenario when the data type hold by the list is in the mapping + # handling scenario when the data type hold by the list is in the + # mapping return mapping[list_details[0]]() elif list_details[0] == TType.STRUCT: # handling scenario when the data type hold by the list is a struct diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py index f9c6e4d..a9c059a 100644 --- a/django_airavata/apps/api/urls.py +++ b/django_airavata/apps/api/urls.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) router = routers.DefaultRouter() router.register(r'projects', views.ProjectViewSet, base_name='project') +router.register(r'experiments', views.ExperimentViewSet, base_name='experiment') router.register(r'new/application/module', views.RegisterApplicationModule, base_name='register_app_module') router.register(r'application-interfaces', views.ApplicationInterfaceViewSet, base_name='application-interface') router.register(r'applications', views.ApplicationModuleViewSet, base_name='application') diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py index 9751a1b..dcada13 100644 --- a/django_airavata/apps/api/views.py +++ b/django_airavata/apps/api/views.py @@ -225,15 +225,31 @@ class ProjectViewSet(APIBackedViewSet): serializer = serializers.ExperimentSerializer(experiments, many=True, context={'request': request}) return Response(serializer.data) -# TODO: convert to ViewSet -class ExperimentList(APIView): - def get(self, request, format=None): - gateway_id = settings.GATEWAY_ID - username = request.user.username - experiments = request.airavata_client.getUserExperiments(request.authz_token, gateway_id, username, -1, 0) - serializer = serializers.ExperimentSerializer(experiments, many=True, context={'request': request}) - return Response(serializer.data) +class ExperimentViewSet(APIBackedViewSet): + + serializer_class = serializers.ExperimentSerializer + lookup_field = 'experiment_id' + + def get_list(self): + return self.request.airavata_client.getUserExperiments(self.authz_token, self.gateway_id, self.username, 1, 0) + + def get_instance(self, lookup_value): + return self.request.airavata_client.getExperiment(self.authz_token, lookup_value) + + def perform_create(self, serializer): + experiment = serializer.save() + experiment_id = self.request.airavata_client.createExperiment(self.authz_token, self.gateway_id, experiment) + experiment.experimentId = experiment_id + + def perform_update(self, serializer): + experiment = serializer.save() + self.request.airavata_client.updateExperiment(self.authz_token, experiment.experimentId, experiment) + + @detail_route(methods=['post']) + def launch(self, request, experiment_id=None): + request.airavata_client.launchExperiment(request.authz_token, experiment_id, self.gateway_id) + return Response({'success': True}) class ApplicationModuleViewSet(APIBackedViewSet): diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue index 5e18b38..8ec6543 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue @@ -87,7 +87,7 @@ export default { // TODO: clone experiment instead of editing it directly props: { experiment: { - type: models.ExperimentModel, + type: models.Experiment, required: true }, appModule: { @@ -125,9 +125,21 @@ export default { saveExperiment: function() { console.log(JSON.stringify(this.localExperiment)); // TODO: validate experiment - // TODO: save experiment - // TODO: set the experiment ID on the new experiment - // TODO: dispatch save event with updated experiment + // save experiment + if (this.localExperiment.experimentId) { + services.ExperimentService.update(this.localExperiment) + .then(experiment => { + console.log(experiment); + this.$emit('saved', experiment); + }); + } else { + services.ExperimentService.create(this.localExperiment) + .then(experiment => { + this.localExperiment.experimentId = experiment.experimentId; + console.log(experiment); + this.$emit('saved', experiment); + }); + } }, saveAndLaunchExperiment: function() { console.log(JSON.stringify(this.localExperiment)); @@ -135,9 +147,12 @@ export default { // TODO: save experiment // TODO: set the experiment ID on the new experiment // TODO: dispatch save event with updated experiment - } + }, }, watch: { + experiment: function(newValue) { + this.localExperiment = newValue.clone(); + }, } } </script> @@ -149,4 +164,4 @@ export default { #col-exp-buttons { text-align: right; } -</style> \ No newline at end of file +</style> diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/CreateExperimentContainer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/CreateExperimentContainer.vue index 9ffdb94..e78cc8b 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/CreateExperimentContainer.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/containers/CreateExperimentContainer.vue @@ -41,6 +41,7 @@ export default { .then(appInterface => { this.experiment.experimentInputs = appInterface.getOrderedApplicationInputs().map(input => input.clone()); this.appInterface = appInterface; + this.experiment.executionId = this.appInterface.applicationInterfaceId; }); } } -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
