This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 3ae702a708a0115871eac3a775b842af7b1e847d Author: Marcus Christie <machr...@iu.edu> AuthorDate: Thu Feb 8 10:43:40 2018 -0500 AIRAVATA-2599 Upload input files before saving/launching --- .../api/static/django_airavata_api/js/index.js | 2 + .../django_airavata_api/js/models/DataType.js | 13 ++++ .../js/models/InputDataObjectType.js | 6 +- .../django_airavata_api/js/utils/FetchUtils.js | 8 ++- .../js/components/experiment/ExperimentEditor.vue | 79 +++++++++++++++------- django_airavata/apps/workspace/urls.py | 6 +- django_airavata/apps/workspace/views.py | 62 +++++++++++++++++ django_airavata/settings_local.py.sample | 1 + 8 files changed, 149 insertions(+), 28 deletions(-) diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js index c3015df..352dec1 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/index.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js @@ -1,6 +1,7 @@ import ApplicationInterfaceDefinition from './models/ApplicationInterfaceDefinition' import ApplicationModule from './models/ApplicationModule' +import DataType from './models/DataType' import Experiment from './models/Experiment' import InputDataObjectType from './models/InputDataObjectType' import OutputDataObjectType from './models/OutputDataObjectType' @@ -23,6 +24,7 @@ import PaginationIterator from './utils/PaginationIterator' exports.models = { ApplicationInterfaceDefinition, ApplicationModule, + DataType, Experiment, FullExperiment, InputDataObjectType, diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js new file mode 100644 index 0000000..f8c229d --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/DataType.js @@ -0,0 +1,13 @@ +import BaseEnum from './BaseEnum' + +export default class DataType extends BaseEnum { +} +DataType.init([ + "STRING", + "INTEGER", + "FLOAT", + "URI", + "URI_COLLECTION", + "STDOUT", + "STDERR", +]); diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js index 5969555..dfe2846 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/InputDataObjectType.js @@ -1,10 +1,14 @@ import BaseModel from './BaseModel'; +import DataType from './DataType' const FIELDS = [ 'name', 'value', - 'type', + { + name: 'type', + type: DataType, + }, 'applicationArgument', 'standardInput', 'userFriendlyDescription', diff --git a/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js b/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js index 074e4ea..58027f0 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/utils/FetchUtils.js @@ -21,9 +21,13 @@ export default { }, post: function (url, body, mediaType = "application/json") { var headers = this.createHeaders(mediaType) + // Browsers automatically handle content type for FormData request bodies + if (body instanceof FormData) { + headers.delete("Content-Type"); + } return fetch(url, { method: 'post', - body: typeof body !== 'string' ? JSON.stringify(body) : body, + body: body, headers: headers, credentials: "same-origin" }).then((response) => { @@ -42,7 +46,7 @@ export default { var headers = this.createHeaders(mediaType) return fetch(url, { method: 'put', - body: typeof body !== 'string' ? JSON.stringify(body) : body, + body: body, headers: headers, credentials: "same-origin" }).then((response) => { diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue index d0f21ad..954781b 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ExperimentEditor.vue @@ -54,9 +54,12 @@ :label="experimentInput.name" :label-for="experimentInput.name" :key="experimentInput.name" :feedback="getValidationFeedback(['experimentInputs', experimentInput.name, 'value'])" :state="getValidationState(['experimentInputs', experimentInput.name, 'value'])"> - <b-form-input :id="experimentInput.name" type="text" v-model="experimentInput.value" required + <b-form-input v-if="isSimpleInput(experimentInput)" :id="experimentInput.name" type="text" v-model="experimentInput.value" required :placeholder="experimentInput.userFriendlyDescription" :state="getValidationState(['experimentInputs', experimentInput.name, 'value'])"></b-form-input> + <b-form-file v-if="isFileInput(experimentInput)" :id="experimentInput.name" type="text" v-model="experimentInput.value" required + :placeholder="experimentInput.userFriendlyDescription" + :state="getValidationState(['experimentInputs', experimentInput.name, 'value'])"></b-form-file> </b-form-group> </div> </div> @@ -94,7 +97,7 @@ <script> import ComputationalResourceSchedulingEditor from './ComputationalResourceSchedulingEditor.vue' -import {models, services} from 'django-airavata-api' +import {models, services, utils as apiUtils} from 'django-airavata-api' import {utils} from 'django-airavata-common-ui' export default { @@ -140,34 +143,52 @@ export default { }, methods: { saveExperiment: function() { - console.log(JSON.stringify(this.localExperiment)); - // TODO: validate experiment - // save experiment - services.ExperimentService.save(this.localExperiment) - .then(experiment => { - this.localExperiment = experiment; - console.log(experiment); - alert('Experiment saved!'); - this.$emit('saved', experiment); + return this.uploadInputFiles() + .then(uploadResults => { + return services.ExperimentService.save(this.localExperiment) + .then(experiment => { + this.localExperiment = experiment; + console.log(experiment); + alert('Experiment saved!'); + this.$emit('saved', experiment); + }); + }) + .catch(result => { + console.log("Save failed!", result); }); }, saveAndLaunchExperiment: function() { - console.log(JSON.stringify(this.localExperiment)); - // TODO: validate experiment - let savedExperiment = null; - services.ExperimentService.save(this.localExperiment) - .then(experiment => { - this.localExperiment = experiment; - return services.ExperimentService.launch(experiment.experimentId) - .then(result => { - alert('Experiment launched!'); - this.$emit('savedAndLaunched', experiment); - }); - }) + return this.uploadInputFiles() + .then(uploadResults => { + return services.ExperimentService.save(this.localExperiment) + .then(experiment => { + this.localExperiment = experiment; + return services.ExperimentService.launch(experiment.experimentId) + .then(result => { + alert('Experiment launched!'); + this.$emit('savedAndLaunched', experiment); + }); + }) + }) .catch(result => { console.log("Launch failed!", result); }); }, + uploadInputFiles: function() { + let uploads = []; + this.localExperiment.experimentInputs.forEach(input => { + if (input.type === models.DataType.URI && input.value) { + let data = new FormData(); + data.append('file', input.value); + data.append('project-id', this.localExperiment.projectId); + data.append('experiment-name', this.localExperiment.experimentName); + let uploadRequest = apiUtils.FetchUtils.post('/workspace/upload', data) + .then(result => input.value = result['data-product-uri']) + uploads.push(uploadRequest); + } + }); + return Promise.all(uploads); + }, getApplicationInputState: function(applicationInput) { const validation = this.getApplicationInputValidation(applicationInput); return validation !== null ? 'invalid' : null; @@ -189,6 +210,18 @@ export default { getValidationState: function(properties) { return this.getValidationFeedback(properties) ? 'invalid' : null; }, + isSimpleInput: function(experimentInput) { + return [ + models.DataType.STRING, + models.DataType.FLOAT, + models.DataType.INTEGER, + ].indexOf(experimentInput.type) >= 0; + }, + isFileInput: function(experimentInput) { + return [ + models.DataType.URI, + ].indexOf(experimentInput.type) >= 0; + }, }, watch: { experiment: function(newValue) { diff --git a/django_airavata/apps/workspace/urls.py b/django_airavata/apps/workspace/urls.py index a3aabc1..b6a0f31 100644 --- a/django_airavata/apps/workspace/urls.py +++ b/django_airavata/apps/workspace/urls.py @@ -9,6 +9,8 @@ urlpatterns = [ url(r'^experiments/(?P<experiment_id>[^/]+)/$', views.view_experiment, name='view_experiment'), url(r'^experiments$', views.experiments_list, name='experiments'), - url(r'^applications/(?P<app_module_id>[^/]+)/create_experiment$', views.create_experiment, name='create_experiment'), + url(r'^applications/(?P<app_module_id>[^/]+)/create_experiment$', + views.create_experiment, name='create_experiment'), url(r'^dashboard$', views.dashboard, name='dashboard'), -] \ No newline at end of file + url(r'^upload$', views.upload_input_file, name='upload_input_file'), +] diff --git a/django_airavata/apps/workspace/views.py b/django_airavata/apps/workspace/views.py index fd79970..c121a0f 100644 --- a/django_airavata/apps/workspace/views.py +++ b/django_airavata/apps/workspace/views.py @@ -1,11 +1,21 @@ import json import logging +import os +from urllib.parse import urlparse +from django.conf import settings from django.contrib.auth.decorators import login_required +from django.core.files.storage import FileSystemStorage +from django.http import JsonResponse from django.shortcuts import render from rest_framework.renderers import JSONRenderer +from airavata.model.data.replica.ttypes import (DataProductModel, + DataProductType, + DataReplicaLocationModel, + ReplicaLocationCategory, + ReplicaPersistentType) from django_airavata.apps.api.views import (ExperimentSearchViewSet, FullExperimentViewSet, ProjectViewSet) @@ -64,3 +74,55 @@ def view_experiment(request, experiment_id): 'full_experiment_data': full_experiment_json, 'launching': json.dumps(launching), }) + + +experiment_data_storage = FileSystemStorage( + location=settings.GATEWAY_DATA_STORE_DIR) + + +@login_required +def upload_input_file(request): + try: + # Save input file to username/project name/exp name/filename + username = request.user.username + project_id = request.POST['project-id'] + project = request.airavata_client.getProject( + request.authz_token, project_id) + exp_name = request.POST['experiment-name'] + input_file = request.FILES['file'] + exp_dir = os.path.join( + experiment_data_storage.get_valid_name(username), + experiment_data_storage.get_valid_name(project.name), + experiment_data_storage.get_valid_name(exp_name)) + file_path = os.path.join( + exp_dir, + experiment_data_storage.get_valid_name(input_file.name)) + input_file_name = experiment_data_storage.save(file_path, input_file) + input_file_fullpath = experiment_data_storage.path(input_file_name) + # Register DataProductModel with DataReplicaLocationModel + data_product = DataProductModel() + data_product.gatewayId = settings.GATEWAY_ID + data_product.ownerName = username + data_product.productName = input_file.name + data_product.dataProductType = DataProductType.FILE + data_replica_location = DataReplicaLocationModel() + data_replica_location.storageResourceId = \ + settings.GATEWAY_DATA_STORE_RESOURCE_ID + data_replica_location.replicaName = \ + "{} gateway data store copy".format(input_file.name) + data_replica_location.replicaLocationCategory = \ + ReplicaLocationCategory.GATEWAY_DATA_STORE + data_replica_location.replicaPersistentType = \ + ReplicaPersistentType.TRANSIENT + hostname = urlparse(request.build_absolute_uri()).hostname + data_replica_location.filePath = \ + "file://{}:{}".format(hostname, input_file_fullpath) + data_product.replicaLocations = [data_replica_location] + data_product_uri = request.airavata_client.registerDataProduct( + request.authz_token, data_product) + return JsonResponse({'uploaded': True, + 'data-product-uri': data_product_uri}) + except Exception as e: + resp = JsonResponse({'uploaded': False, 'error': str(e)}) + resp.status_code = 500 + return resp diff --git a/django_airavata/settings_local.py.sample b/django_airavata/settings_local.py.sample index e40eb04..1a1a0a2 100644 --- a/django_airavata/settings_local.py.sample +++ b/django_airavata/settings_local.py.sample @@ -33,6 +33,7 @@ AIRAVATA_API_HOST = 'localhost' AIRAVATA_API_PORT = 8930 AIRAVATA_API_SECURE = False GATEWAY_DATA_STORE_RESOURCE_ID = '...' +GATEWAY_DATA_STORE_DIR = '...' # Profile Service Configuration PROFILE_SERVICE_HOST = AIRAVATA_API_HOST -- To stop receiving notification emails like this one, please contact machris...@apache.org.