This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 0e90f71541097bf37daa015e6789b6fc5ab4210e Author: Marcus Christie <machr...@iu.edu> AuthorDate: Fri Feb 23 05:50:19 2018 -0500 Load and apply GroupResourceProfile --- django_airavata/apps/api/serializers.py | 2 +- .../api/static/django_airavata_api/js/index.js | 4 + .../js/models/BatchQueueResourcePolicy.js | 19 ++++ .../models/ComputationalResourceSchedulingModel.js | 8 +- .../js/models/ComputeResourcePolicy.js | 20 ++++ .../js/models/GroupComputeResourcePreference.js | 30 ++++++ .../js/models/GroupResourceProfile.js | 40 ++++++++ .../js/services/GroupResourceProfileService.js | 17 ++++ .../ComputationalResourceSchedulingEditor.vue | 45 ++++++++- .../components/experiment/QueueSettingsEditor.vue | 108 ++++++++++++++++----- 10 files changed, 268 insertions(+), 25 deletions(-) diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py index ca9207b..3d64f7d 100644 --- a/django_airavata/apps/api/serializers.py +++ b/django_airavata/apps/api/serializers.py @@ -117,7 +117,7 @@ class GroupSerializer(serializers.Serializer): url = FullyEncodedHyperlinkedIdentityField(view_name='django_airavata_api:group-detail', lookup_field='id', lookup_url_kwarg='group_id') id = serializers.CharField(default=GroupModel.thrift_spec[1][4], read_only=True) name = serializers.CharField(required=True) - description = serializers.CharField(allow_null=True) + description = serializers.CharField(allow_null=True, allow_blank=True) ownerId = serializers.CharField(read_only=True) members = serializers.ListSerializer(child=serializers.CharField()) isAdmin = serializers.SerializerMethodField() diff --git a/django_airavata/apps/api/static/django_airavata_api/js/index.js b/django_airavata/apps/api/static/django_airavata_api/js/index.js index b9f4b42..36b9725 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/index.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/index.js @@ -8,6 +8,7 @@ import OutputDataObjectType from './models/OutputDataObjectType' import Project from './models/Project' import FullExperiment from './models/FullExperiment' import Group from './models/Group' +import GroupResourceProfile from './models/GroupResourceProfile' import ApplicationDeploymentService from './services/ApplicationDeploymentService' import ApplicationInterfaceService from './services/ApplicationInterfaceService' @@ -17,6 +18,7 @@ import ExperimentSearchService from './services/ExperimentSearchService' import FullExperimentService from './services/FullExperimentService' import ProjectService from './services/ProjectService' import GroupService from './services/GroupService' +import GroupResourceProfileService from './services/GroupResourceProfileService' import UserProfileService from './services/UserProfileService' import FetchUtils from './utils/FetchUtils' @@ -32,6 +34,7 @@ exports.models = { OutputDataObjectType, Project, Group, + GroupResourceProfile, } exports.services = { @@ -43,6 +46,7 @@ exports.services = { FullExperimentService, ProjectService, GroupService, + GroupResourceProfileService, UserProfileService, } diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/BatchQueueResourcePolicy.js b/django_airavata/apps/api/static/django_airavata_api/js/models/BatchQueueResourcePolicy.js new file mode 100644 index 0000000..9f4b0d2 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/BatchQueueResourcePolicy.js @@ -0,0 +1,19 @@ +import BaseModel from './BaseModel' + + +const FIELDS = [ + 'resourcePolicyId', + 'computeResourceId', + 'groupResourceProfileId', + 'queuename', + 'maxAllowedNodes', + 'maxAllowedCores', + 'maxAllowedWalltime', +]; + +export default class BatchQueueResourcePolicy extends BaseModel { + + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js index 6fda294..67bb119 100644 --- a/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputationalResourceSchedulingModel.js @@ -20,23 +20,29 @@ export default class ComputationalResourceSchedulingModel extends BaseModel { super(FIELDS, data); } - validate(queueInfo = {}) { + validate(queueInfo = {}, batchQueueResourcePolicy = null) { const validationResults = {}; if (this.isEmpty(this.resourceHostId)) { validationResults['resourceHostId'] = "Please select a compute resource."; } if (!(this.nodeCount > 0)) { validationResults['nodeCount'] = "Enter a node count greater than 0."; + } else if (batchQueueResourcePolicy && this.nodeCount > batchQueueResourcePolicy.maxAllowedNodes) { + validationResults['nodeCount'] = `Enter a node count no greater than ${batchQueueResourcePolicy.maxAllowedNodes}.`; } else if (queueInfo.maxNodes && this.nodeCount > queueInfo.maxNodes) { validationResults['nodeCount'] = `Enter a node count no greater than ${queueInfo.maxNodes}.`; } if (!(this.totalCPUCount > 0)) { validationResults['totalCPUCount'] = "Enter a core count greater than 0."; + } else if (batchQueueResourcePolicy && this.totalCPUCount > batchQueueResourcePolicy.maxAllowedCores) { + validationResults['totalCPUCount'] = `Enter a core count no greater than ${batchQueueResourcePolicy.maxAllowedCores}.`; } else if (queueInfo.maxProcessors && this.totalCPUCount > queueInfo.maxProcessors) { validationResults['totalCPUCount'] = `Enter a core count no greater than ${queueInfo.maxProcessors}.`; } if (!(this.wallTimeLimit > 0)) { validationResults['wallTimeLimit'] = "Enter a wall time limit greater than 0."; + } else if (batchQueueResourcePolicy && this.wallTimeLimit > batchQueueResourcePolicy.maxAllowedWalltime) { + validationResults['wallTimeLimit'] = `Enter a wall time limit no greater than ${batchQueueResourcePolicy.maxAllowedWalltime}.`; } else if (queueInfo.maxRunTime && this.wallTimeLimit > queueInfo.maxRunTime) { validationResults['wallTimeLimit'] = `Enter a wall time limit no greater than ${queueInfo.maxRunTime}.`; } diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/ComputeResourcePolicy.js b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputeResourcePolicy.js new file mode 100644 index 0000000..82b97d5 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/ComputeResourcePolicy.js @@ -0,0 +1,20 @@ +import BaseModel from './BaseModel' + + +const FIELDS = [ + 'resourcePolicyId', + 'computeResourceId', + 'groupResourceProfileId', + { + name: 'allowedBatchQueues', + type: 'string', + list: true + } +]; + +export default class ComputeResourcePolicy extends BaseModel { + + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/GroupComputeResourcePreference.js b/django_airavata/apps/api/static/django_airavata_api/js/models/GroupComputeResourcePreference.js new file mode 100644 index 0000000..3a69495 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/GroupComputeResourcePreference.js @@ -0,0 +1,30 @@ +import BaseModel from './BaseModel' + + +const FIELDS = [ + 'computeResourceId', + 'groupResourceProfileId', + 'overridebyAiravata', + 'loginUserName', + 'preferredJobSubmissionProtocol', + 'preferredDataMovementProtocol', + 'preferredBatchQueue', + 'scratchLocation', + 'allocationProjectNumber', + 'resourceSpecificCredentialStoreToken', + 'usageReportingGatewayId', + 'qualityOfService', + 'reservation', + 'reservationStartTime', + 'reservationEndTime', + 'sshAccountProvisioner', + 'groupSSHAccountProvisionerConfigs', + 'sshAccountProvisionerAdditionalInfo', +]; + +export default class GroupComputeResourcePreference extends BaseModel { + + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/GroupResourceProfile.js b/django_airavata/apps/api/static/django_airavata_api/js/models/GroupResourceProfile.js new file mode 100644 index 0000000..1f66a38 --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/models/GroupResourceProfile.js @@ -0,0 +1,40 @@ +import BaseModel from './BaseModel' +import BatchQueueResourcePolicy from './BatchQueueResourcePolicy' +import ComputeResourcePolicy from './ComputeResourcePolicy' +import GroupComputeResourcePreference from './GroupComputeResourcePreference' + +const FIELDS = [ + 'gatewayId', + 'groupResourceProfileId', + 'groupResourceProfileName', + { + name: 'computePreferences', + type: GroupComputeResourcePreference, + list: true, + }, + { + name: 'computeResourcePolicies', + type: ComputeResourcePolicy, + list: true, + }, + { + name: 'batchQueueResourcePolicies', + type: BatchQueueResourcePolicy, + list: true + }, + { + name: 'creationTime', + type: 'date', + }, + { + name: 'updatedTime', + type: 'date', + }, +]; + +export default class GroupResourceProfile extends BaseModel { + + constructor(data = {}) { + super(FIELDS, data); + } +} diff --git a/django_airavata/apps/api/static/django_airavata_api/js/services/GroupResourceProfileService.js b/django_airavata/apps/api/static/django_airavata_api/js/services/GroupResourceProfileService.js new file mode 100644 index 0000000..b572bbd --- /dev/null +++ b/django_airavata/apps/api/static/django_airavata_api/js/services/GroupResourceProfileService.js @@ -0,0 +1,17 @@ + +import FetchUtils from '../utils/FetchUtils' +import GroupResourceProfile from '../models/GroupResourceProfile' + +class GroupResourceProfileService { + list(data = null) { + if (data) { + return Promise.resolve(data.map(result => new GroupResourceProfile(result))); + } else { + return FetchUtils.get('/api/group-resource-profiles/') + .then(results => results.map(result => new GroupResourceProfile(result))); + } + } +} + +// Export as a singleton +export default new GroupResourceProfileService(); \ No newline at end of file diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ComputationalResourceSchedulingEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ComputationalResourceSchedulingEditor.vue index 38ee6dd..c25cad3 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ComputationalResourceSchedulingEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/ComputationalResourceSchedulingEditor.vue @@ -24,6 +24,8 @@ v-model="localComputationalResourceScheduling" v-if="appDeploymentId" :app-deployment-id="appDeploymentId" + :compute-resource-policy="selectedComputeResourcePolicy" + :batch-queue-resource-policies="batchQueueResourcePolicies" @input="queueSettingsChanged"> </queue-settings-editor> </div> @@ -57,6 +59,8 @@ export default { localComputationalResourceScheduling: this.value.clone(), computeResources: {}, applicationDeployments: [], + groupResourceProfiles: [], + selectedGroupResourceProfile: null, appDeploymentId: null, resourceHostId: null, // TODO: replace this with Loading spinner, better mechanism @@ -69,12 +73,14 @@ export default { mounted: function () { this.loadApplicationDeployments(this.appModuleId); this.loadComputeResourcesForApplicationInterface(this.appInterfaceId); + this.loadGroupResourceProfiles(); }, computed: { computeResourceOptions: function() { const computeResourceOptions = []; for (let computeResourceId in this.computeResources) { - if (this.computeResources.hasOwnProperty(computeResourceId)) { + if (this.computeResources.hasOwnProperty(computeResourceId) + && this.isComputeHostInGroupResourceProfile(computeResourceId)) { computeResourceOptions.push({ value: computeResourceId, text: this.computeResources[computeResourceId], @@ -86,6 +92,22 @@ export default { }, loading: function() { return this.loadingCount > 0; + }, + selectedComputeResourcePolicy: function() { + if (this.groupResourceProfile === null) { + return null; + } + return this.groupResourceProfile.computeResourcePolicies.find(crp => { + return crp.computeResourceId === this.localComputationalResourceScheduling.resourceHostId; + }); + }, + batchQueueResourcePolicies: function() { + if (this.groupResourceProfile === null) { + return null; + } + return this.groupResourceProfile.batchQueueResourcePolicies.filter(bqrp => { + return bqrp.computeResourceId === this.localComputationalResourceScheduling.resourceHostId; + }); } }, methods: { @@ -113,6 +135,27 @@ export default { .then(computeResources => this.computeResources = computeResources) .then(()=> {this.loadingCount--;}, () => {this.loadingCount--;}); }, + loadGroupResourceProfiles: function() { + this.loadingCount++; + services.GroupResourceProfileService.list() + .then(groupResourceProfiles => { + this.groupResourceProfiles = groupResourceProfiles; + if (this.groupResourceProfiles && this.groupResourceProfiles.length > 0) { + // Just pick the first one for now + this.groupResourceProfile = this.groupResourceProfiles[0]; + } + }) + .then(()=> {this.loadingCount--;}, () => {this.loadingCount--;}); + }, + isComputeHostInGroupResourceProfile: function(computeHostId) { + // TODO: for now don't require a GroupResourceProfile + if (this.groupResourceProfile === null) { + return true; + } + return this.groupResourceProfile.computePreferences.some(cp => { + return cp.computeResourceId === computeHostId; + }) + }, queueSettingsChanged: function() { // QueueSettingsEditor updates the full // ComputationalResourceSchedulingModel instance but doesn't know diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/QueueSettingsEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/QueueSettingsEditor.vue index 355ec4b..5dbe3f9 100644 --- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/QueueSettingsEditor.vue +++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/QueueSettingsEditor.vue @@ -52,28 +52,28 @@ :feedback="getValidationFeedback('nodeCount')" :state="getValidationState('nodeCount')"> <b-form-input id="node-count" type="number" min="1" - :max="selectedQueueDefault.maxNodes" + :max="maxNodes" v-model="localComputationalResourceScheduling.nodeCount" required @input="emitValueChanged" :state="getValidationState('nodeCount')"> </b-form-input> <div slot="description"> <i class="fa fa-info-circle" aria-hidden="true"></i> - Max Allowed Nodes = {{ selectedQueueDefault.maxNodes }} + Max Allowed Nodes = {{ maxNodes }} </div> </b-form-group> <b-form-group label="Total Core Count" label-for="core-count" :feedback="getValidationFeedback('totalCPUCount')" :state="getValidationState('totalCPUCount')"> <b-form-input id="core-count" type="number" min="1" - :max="selectedQueueDefault.maxProcessors" + :max="maxCPUCount" v-model="localComputationalResourceScheduling.totalCPUCount" required @input="emitValueChanged" :state="getValidationState('totalCPUCount')"> </b-form-input> <div slot="description"> <i class="fa fa-info-circle" aria-hidden="true"></i> - Max Allowed Cores = {{ selectedQueueDefault.maxProcessors }} + Max Allowed Cores = {{ maxCPUCount }} </div> </b-form-group> <b-form-group label="Wall Time Limit" label-for="walltime-limit" @@ -81,7 +81,7 @@ :state="getValidationState('wallTimeLimit')"> <b-input-group right="minutes"> <b-form-input id="walltime-limit" type="number" min="1" - :max="selectedQueueDefault.maxRunTime" + :max="maxWalltime" v-model="localComputationalResourceScheduling.wallTimeLimit" required @input="emitValueChanged" :state="getValidationState('wallTimeLimit')"> @@ -89,7 +89,7 @@ </b-input-group> <div slot="description"> <i class="fa fa-info-circle" aria-hidden="true"></i> - Max Allowed Wall Time = {{ selectedQueueDefault.maxRunTime }} + Max Allowed Wall Time = {{ maxWalltime }} </div> </b-form-group> <div> @@ -117,6 +117,14 @@ export default { type: String, required: true }, + computeResourcePolicy: { + type: models.ComputeResourcePolicy, + required: false, + }, + batchQueueResourcePolicies: { + type: models.BatchQueueResourcePolicy, + required: false, + } }, data () { return { @@ -138,14 +146,35 @@ export default { selectedQueueDefault: function() { return this.queueDefaults.find(queue => queue.queueName === this.localComputationalResourceScheduling.queueName); }, + maxCPUCount: function() { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(this.selectedQueueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedCores, this.selectedQueueDefault.maxProcessors); + } + return this.selectedQueueDefault.maxProcessors; + }, + maxNodes: function() { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(this.selectedQueueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedNodes, this.selectedQueueDefault.maxNodes); + } + return this.selectedQueueDefault.maxNodes; + }, + maxWalltime: function() { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(this.selectedQueueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedWalltime, this.selectedQueueDefault.maxRunTime); + } + return this.selectedQueueDefault.maxRunTime; + } }, methods: { queueChanged: function(queueName) { const queueDefault = this.queueDefaults.find(queue => queue.queueName === queueName); - this.localComputationalResourceScheduling.totalCPUCount = queueDefault.defaultCPUCount; - this.localComputationalResourceScheduling.nodeCount = queueDefault.defaultNodeCount; - this.localComputationalResourceScheduling.wallTimeLimit = queueDefault.defaultWalltime; + this.localComputationalResourceScheduling.totalCPUCount = this.getDefaultCPUCount(defaultQueue); + this.localComputationalResourceScheduling.nodeCount = this.getDefaultNodeCount(defaultQueue); + this.localComputationalResourceScheduling.wallTimeLimit = this.getDefaultWalltime(defaultQueue); this.emitValueChanged(); }, emitValueChanged: function() { @@ -155,28 +184,63 @@ export default { services.ApplicationDeploymentService.getQueues(this.appDeploymentId) .then(queueDefaults => { // Sort queue defaults - this.queueDefaults = queueDefaults.sort((a, b) => { - // Sort default first, then by alphabetically by name - if (a.isDefaultQueue) { - return -1; - } else if (b.isDefaultQueue) { - return 1; - } else { - return a.queueName.localeCompare(b.queueName); - } + this.queueDefaults = queueDefaults + .filter(q => this.isQueueInComputeResourcePolicy(q.queueName)) + .sort((a, b) => { + // Sort default first, then by alphabetically by name + if (a.isDefaultQueue) { + return -1; + } else if (b.isDefaultQueue) { + return 1; + } else { + return a.queueName.localeCompare(b.queueName); + } }); // Find the default queue and apply it's settings const defaultQueue = this.queueDefaults[0]; this.localComputationalResourceScheduling.queueName = defaultQueue.queueName; - this.localComputationalResourceScheduling.totalCPUCount = defaultQueue.defaultCPUCount; - this.localComputationalResourceScheduling.nodeCount = defaultQueue.defaultNodeCount; - this.localComputationalResourceScheduling.wallTimeLimit = defaultQueue.defaultWalltime; + this.localComputationalResourceScheduling.totalCPUCount = this.getDefaultCPUCount(defaultQueue); + this.localComputationalResourceScheduling.nodeCount = this.getDefaultNodeCount(defaultQueue); + this.localComputationalResourceScheduling.wallTimeLimit = this.getDefaultWalltime(defaultQueue); this.emitValueChanged(); }); }, + isQueueInComputeResourcePolicy: function(queueName) { + if (!this.computeResourcePolicy) { + return true; + } + return this.computeResourcePolicy.allowedBatchQueues.includes(queueName); + }, + getBatchQueueResourcePolicy: function(queueName) { + if (!this.batchQueueResourcePolicies || this.batchQueueResourcePolicies.length === 0) { + return null; + } + return this.batchQueueResourcePolicies.find(bqrp => bqrp.queuename === queueName); + }, + getDefaultCPUCount: function(queueDefault) { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(queueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedCores, queueDefault.defaultCPUCount); + } + return queueDefault.defaultCPUCount; + }, + getDefaultNodeCount: function(queueDefault) { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(queueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedNodes, queueDefault.defaultNodeCount); + } + return queueDefault.defaultNodeCount; + }, + getDefaultWalltime: function(queueDefault) { + const batchQueueResourcePolicy = this.getBatchQueueResourcePolicy(queueDefault.queueName); + if (batchQueueResourcePolicy) { + return Math.min(batchQueueResourcePolicy.maxAllowedWalltime, queueDefault.defaultWalltime); + } + return queueDefault.defaultWalltime; + }, getValidationFeedback: function(properties) { - return utils.getProperty(this.localComputationalResourceScheduling.validate(this.selectedQueueDefault), properties); + return utils.getProperty(this.localComputationalResourceScheduling.validate(this.selectedQueueDefault, this.getBatchQueueResourcePolicy(this.selectedQueueDefault.queueName)), properties); }, getValidationState: function(properties) { return this.getValidationFeedback(properties) ? 'invalid' : null; -- To stop receiving notification emails like this one, please contact machris...@apache.org.