Repository: airavata Updated Branches: refs/heads/master 9ef67cdf3 -> 6c5e9954a
http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift new file mode 100644 index 0000000..efacd0b --- /dev/null +++ b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +include "../../airavata-apis/airavata_commons.thrift" +include "data_movement_models.thrift" + +namespace java org.apache.airavata.model.appcatalog.computeresource +namespace php Airavata.Model.AppCatalog.ComputeResource +namespace cpp apache.airavata.model.appcatalog.computeresource +namespace py apache.airavata.model.appcatalog.computeresource + + +/** + * Enumeration of local resource job manager types supported by Airavata + * + * FORK: + * Forking of commands without any job manager + * + * PBS: + * Job manager supporting the Portal Batch System (PBS) protocol. Some examples include TORQUE, PBSPro, Grid Engine. + * + * SLURM: + * The Simple Linux Utility for Resource Management is a open source workload manager. + * + * UGE: + * Univa Grid Engine, a variation of PBS implementation. + * + * LSF: + * IBM Platform Load Sharing Facility is dominantly installed on IBM clusters. + * +*/ +enum ResourceJobManagerType { + FORK, + PBS, + SLURM, + LSF, + UGE +} + +/** + * Enumeration of resource job manager commands + * + * SUBMISSION: + * Ex: qsub, sbatch + * + * JOBMONITORING: + * Ex: qstat, squeue + * + * DELETION: + * Ex: qdel, scancel + * + * CHECK_JOB: + * Detailed Status about the Job. Ex: checkjob + * + * SHOW_QUEUE: + * List of Queued Job by the schedular. Ex: showq + * + * SHOW_RESERVATION: + * List all reservations. Ex:showres, show_res + * + * SHOW_START: + * Display the start time of the specified job. Ex: showstart + * +*/ +enum JobManagerCommand { + SUBMISSION, + JOB_MONITORING, + DELETION, + CHECK_JOB, + SHOW_QUEUE, + SHOW_RESERVATION, + SHOW_START +} + + +/** + * Resource Job Manager Information + * + * resourceJobManagerType: + * A typical HPC cluster has a single Job Manager to manage the resources. + * + * pushMonitoringEndpoint: + * If the job manager pushes out state changes to a database or bus, specify the service endpoint. + * Ex: Moab Web Service, Moab MongoDB URL, AMQP (GLUE2) Broker + * + * jobManagerBinPath: + * Path to the Job Manager Installation Binary directory. + * + * jobManagerCommands: + * An enumeration of commonly used manager commands. + * +*/ +struct ResourceJobManager { + 1: required string resourceJobManagerId = airavata_commons.DEFAULT_ID, + 2: required ResourceJobManagerType resourceJobManagerType, + 3: optional string pushMonitoringEndpoint, + 4: optional string jobManagerBinPath, + 5: optional map<JobManagerCommand, string> jobManagerCommands +} + +/** + * Enumeration of File Systems on the resource + * + * FORK: + * Forking of commands without any job manager + * + * PBS: + * Job manager supporting the Portal Batch System (PBS) protocol. Some examples include TORQUE, PBSPro, Grid Engine. + * + * UGE: + * Univa Grid Engine, a variation of PBS implementation. + * + * SLURM: + * The Simple Linux Utility for Resource Management is a open source workload manager. + * +*/ +enum FileSystems { + HOME, + WORK, + LOCALTMP, + SCRATCH, + ARCHIVE +} + +/** + * Batch Queue Information on SuperComputers + * + * maxRunTime: + * Maximum allowed run time in hours. +*/ +struct BatchQueue { + 1: required string queueName, + 2: optional string queueDescription, + 3: optional i32 maxRunTime, + 4: optional i32 maxNodes, + 5: optional i32 maxProcessors, + 6: optional i32 maxJobsInQueue, + 7: optional i32 maxMemory +} + + +/** + * Enumeration of Airavata supported Job Submission Mechanisms for High Performance Computing Clusters. + * + * SSH: + * Execute remote job submission commands using via secure shell protocol. + * + * GRAM: + * Execute remote jobs via Globus GRAM service. + * + * UNICORE: + * Execute remote jobs via Unicore services + * +*/ +enum JobSubmissionProtocol { + LOCAL, + SSH, + GLOBUS, + UNICORE, + CLOUD, + SSH_FORK, + LOCAL_FORK +} + +/** +* Monitoring modes +* +* POLL_JOB_MANAGER: +* GFac need to pull job status changes. +* +* XSEDE_AMQP_SUBSCRIBE: +* Server will publish job status changes to amqp servert. +* +**/ +enum MonitorMode { + POLL_JOB_MANAGER, + JOB_EMAIL_NOTIFICATION_MONITOR, + XSEDE_AMQP_SUBSCRIBE, + FORK +} + +/** + * Locally Fork Jobs as OS processes + * + * alternativeSSHHostName: + * If the login to ssh is different than the hostname itself, specify it here + * + * sshPort: + * If a non-default port needs to used, specify it. +*/ +struct LOCALSubmission { + 1: required string jobSubmissionInterfaceId = airavata_commons.DEFAULT_ID, + 2: required data_movement_models.SecurityProtocol securityProtocol, + 3: required ResourceJobManager resourceJobManager +} + +/** + * Authenticate using Secured Shell + * + * alternativeSSHHostName: + * If the login to ssh is different than the hostname itself, specify it here + * + * sshPort: + * If a non-default port needs to used, specify it. + * + * batchQueueEmailSenders: + * If a resource always sends the monitoring from a specific address, specify the + * full email address. If a resource sends emails from multiple addresses ( + * example: based on the submitted login node) then use the wildchar * to indicate + * the same. Example: *@*.example.com or *@example.com + * +*/ +struct SSHJobSubmission { + 1: required string jobSubmissionInterfaceId = airavata_commons.DEFAULT_ID, + 2: required data_movement_models.SecurityProtocol securityProtocol, + 3: required ResourceJobManager resourceJobManager, + 4: optional string alternativeSSHHostName, + 5: optional i32 sshPort = 22, + 6: optional MonitorMode monitorMode, + 7: optional list<string> batchQueueEmailSenders +} + +struct GlobusJobSubmission { + 1: required string jobSubmissionInterfaceId = airavata_commons.DEFAULT_ID, + 2: required data_movement_models.SecurityProtocol securityProtocol, + 3: optional list<string> globusGateKeeperEndPoint +} + +/** + * Unicore Job Submission + * + * unicoreEndPointURL: + * unicoreGateway End Point. The provider will query this service to fetch required service end points. + * authenticationMode + * The authenticationMode defines the way certificate is fetched. +*/ +struct UnicoreJobSubmission { + 1: required string jobSubmissionInterfaceId = airavata_commons.DEFAULT_ID, + 2: required data_movement_models.SecurityProtocol securityProtocol, + 3: required string unicoreEndPointURL, +} + + + +/** +* Provider name +**/ +enum ProviderName { + EC2, + AWSEC2, + RACKSPACE +} + +/** + * Cloud Job Submission + * + * +*/ +struct CloudJobSubmission { + 1: required string jobSubmissionInterfaceId = airavata_commons.DEFAULT_ID, + 2: required data_movement_models.SecurityProtocol securityProtocol, + 3: required string nodeId, + 4: required string executableType, + 5: required ProviderName providerName, + 6: required string userAccountName +} + +/** + * Job Submission Interfaces + * + * jobSubmissionInterfaceId: The Job Submission Interface has to be previously registered and referenced here. + * + * priorityOrder: + * For resources with multiple interfaces, the priority order should be selected. + * Lower the numerical number, higher the priority + * +*/ +struct JobSubmissionInterface { + 1: required string jobSubmissionInterfaceId, + 2: required JobSubmissionProtocol jobSubmissionProtocol + 3: required i32 priorityOrder = 0, +} + + + +/** + * Computational Resource Description + * + * computeResourceId: Airavata Internal Unique Identifier to distinguish Compute Resource. + * + * hostName: + * Fully Qualified Host Name. + * + * hostAliases: + * Aliases if any. + * + * ipAddress: + * IP Addresses of the Resource. + * + * resourceDescription: + * A user friendly description of the resource. + * + * JobSubmissionProtocols: + * A computational resources may have one or more ways of submitting Jobs. This structure + * will hold all available mechanisms to interact with the resource. + * The key is the priority + * + * DataMovementProtocol: + * Option to specify a prefered data movement mechanism of the available options. + * + * fileSystems: + * Map of file systems type and the path. + * +*/ +struct ComputeResourceDescription { +// 1: required bool isEmpty = 0, + 1: required string computeResourceId = airavata_commons.DEFAULT_ID, + 2: required string hostName, + 3: optional list<string> hostAliases, + 4: optional list<string> ipAddresses, + 5: optional string resourceDescription, + 6: optional bool enabled, + 7: optional list<BatchQueue> batchQueues, + 8: optional map<FileSystems, string> fileSystems, + 9: optional list<JobSubmissionInterface> jobSubmissionInterfaces, + 10: optional list<data_movement_models.DataMovementInterface> dataMovementInterfaces, + 11: optional i32 maxMemoryPerNode +} http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/resource-catalog-models/data_movement_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/data_movement_models.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/data_movement_models.thrift new file mode 100644 index 0000000..ed950f2 --- /dev/null +++ b/thrift-interface-descriptions/data-models/resource-catalog-models/data_movement_models.thrift @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +include "../../airavata-apis/airavata_commons.thrift" + +namespace java org.apache.airavata.model.data.movement +namespace php Airavata.Model.Data.Movement +namespace cpp apache.airavata.model.data.movement +namespace py apache.airavata.model.data.movement + +enum DMType { + COMPUTE_RESOURCE, + STORAGE_RESOURCE +} + +/** + * Enumeration of security sshKeyAuthentication and authorization mechanisms supported by Airavata. This enumeration just + * describes the supported mechanism. The corresponding security credentials are registered with Airavata Credential + * store. + * + * USERNAME_PASSWORD: + * A User Name. + * + * SSH_KEYS: + * SSH Keys + * + * FIXME: Change GSI to a more precise generic security protocol - X509 + * +*/ +enum SecurityProtocol { + USERNAME_PASSWORD, + SSH_KEYS, + GSI, + KERBEROS, + OAUTH, + LOCAL +} + + +/** + * Enumeration of data movement supported by Airavata + * + * SCP: + * Job manager supporting the Portal Batch System (PBS) protocol. Some examples include TORQUE, PBSPro, Grid Engine. + * + * SFTP: + * The Simple Linux Utility for Resource Management is a open source workload manager. + * + * GridFTP: + * Globus File Transfer Protocol + * + * UNICORE_STORAGE_SERVICE: + * Storage Service Provided by Unicore + * +*/ +enum DataMovementProtocol { + LOCAL, + SCP, + SFTP, + GridFTP, + UNICORE_STORAGE_SERVICE +} + +/** + * Data Movement through Secured Copy + * + * alternativeSCPHostName: + * If the login to scp is different than the hostname itself, specify it here + * + * sshPort: + * If a non-default port needs to used, specify it. +*/ +struct SCPDataMovement { + 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, + 2: required SecurityProtocol securityProtocol, + 3: optional string alternativeSCPHostName, + 4: optional i32 sshPort = 22 +} + +/** + * Data Movement through GridFTP + * + * alternativeSCPHostName: + * If the login to scp is different than the hostname itself, specify it here + * + * sshPort: + * If a non-default port needs to used, specify it. +*/ +struct GridFTPDataMovement { + 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, + 2: required SecurityProtocol securityProtocol, + 3: required list<string> gridFTPEndPoints +} + +/** + * Data Movement through UnicoreStorage + * + * unicoreEndPointURL: + * unicoreGateway End Point. The provider will query this service to fetch required service end points. +*/ +struct UnicoreDataMovement { + 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, + 2: required SecurityProtocol securityProtocol, + 3: required string unicoreEndPointURL +} + +/** + * LOCAL + * + * alternativeSCPHostName: + * If the login to scp is different than the hostname itself, specify it here + * + * sshPort: + * If a non-defualt port needs to used, specify it. +*/ +struct LOCALDataMovement { + 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, +} + +/** + * Data Movement Interfaces + * + * dataMovementInterfaceId: The Data Movement Interface has to be previously registered and referenced here. + * + * priorityOrder: + * For resources with multiple interfaces, the priority order should be selected. + * Lower the numerical number, higher the priority + * +*/ +struct DataMovementInterface { + 1: required string dataMovementInterfaceId, + 2: required DataMovementProtocol dataMovementProtocol, + 3: required i32 priorityOrder = 0, +} + + + http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/resource-catalog-models/gateway_resource_profile_model.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/gateway_resource_profile_model.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/gateway_resource_profile_model.thrift new file mode 100644 index 0000000..de76c7d --- /dev/null +++ b/thrift-interface-descriptions/data-models/resource-catalog-models/gateway_resource_profile_model.thrift @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +namespace java org.apache.airavata.model.appcatalog.gatewayprofile +namespace php Airavata.Model.AppCatalog.GatewayProfile +namespace cpp apache.airavata.model.appcatalog.gatewayprofile +namespace py apache.airavata.model.appcatalog.gatewayprofile + +include "compute_resource_model.thrift" +include "data_movement_models.thrift" + +/** + * Gateway specific preferences for a Computer Resource + * + * computeResourceId: + * Corelate the preference to a compute resource. + * + * overridebyAiravata: + * If turned true, Airavata will override the preferences of better alternatives exist. + * + * loginUserName: + * If turned true, Airavata will override the preferences of better alternatives exist. + * + * preferredJobSubmissionProtocol: + * For resources with multiple job submission protocols, the gateway can pick a preferred option. + * + * preferredDataMovementProtocol: + * For resources with multiple data movement protocols, the gateway can pick a preferred option. + * + * preferredBatchQueue: + * Gateways can choose a defualt batch queue based on average job dimention, reservations or other metrics. + * + * scratchLocation: + * Path to the local scratch space on a HPC cluster. Typically used to create working directory for job execution. + * + * allocationProjectNumber: + * Typically used on HPC machines to charge computing usage to a account number. For instance, on XSEDE once an + * allocation is approved, an allocation number is assigned. Before passing this number with job submittions, the + * account to be used has to be added to the allocation. + * + * resourceSpecificCredentialStoreToken: + * Resource specific credential store token. If this token is specified, then it is superceeded by the gateway's + * default credential store. + * +*/ +struct ComputeResourcePreference { + 1: required string computeResourceId, + 2: required bool overridebyAiravata = 1, + 3: optional string loginUserName, + 4: optional compute_resource_model.JobSubmissionProtocol preferredJobSubmissionProtocol, + 5: optional data_movement_models.DataMovementProtocol preferredDataMovementProtocol, + 6: optional string preferredBatchQueue, + 7: optional string scratchLocation, + 8: optional string allocationProjectNumber, + 9: optional string resourceSpecificCredentialStoreToken +} + +struct StoragePreference { + 1: required string storageResourceId, + 2: optional string loginUserName, + 3: optional string fileSystemRootLocation, + 4: optional string resourceSpecificCredentialStoreToken +} + +/** + * Gateway Resource Profile + * + * gatewayID: + * Unique identifier for the gateway assigned by Airavata. Corelate this to Airavata Admin API Gateway Registration. + * + * credentialStoreToken: + * Gateway's defualt credential store token. + * + * computeResourcePreferences: + * List of resource preferences for each of the registered compute resources. + * +*/ +struct GatewayResourceProfile { + 1: required string gatewayID, + 2: optional string credentialStoreToken, + 3: optional list<ComputeResourcePreference> computeResourcePreferences + 4: optional list<StoragePreference> storagePreferences +} http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/resource-catalog-models/resource_catalog_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/resource_catalog_models.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/resource_catalog_models.thrift new file mode 100644 index 0000000..8f67576 --- /dev/null +++ b/thrift-interface-descriptions/data-models/resource-catalog-models/resource_catalog_models.thrift @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +include "compute_resource_model.thrift" +include "storage_resource_model.thrift" +include "gateway_resource_profile_model.thrift" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/resource-catalog-models/storage_resource_model.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/storage_resource_model.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/storage_resource_model.thrift new file mode 100644 index 0000000..315c388 --- /dev/null +++ b/thrift-interface-descriptions/data-models/resource-catalog-models/storage_resource_model.thrift @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +include "../../airavata-apis/airavata_commons.thrift" +include "data_movement_models.thrift" + +namespace java org.apache.airavata.model.appcatalog.storageresource +namespace php Airavata.Model.AppCatalog.StorageResource +namespace cpp apache.airavata.model.appcatalog.storageresource +namespace py apache.airavata.model.appcatalog.storageresource + +/** + * Storage Resource Description + * + * storageResourceId: Airavata Internal Unique Identifier to distinguish Compute Resource. + * + * hostName: + * Fully Qualified Host Name. + * + * storageResourceDescription: + * A user friendly description of the resource. + * + * + * DataMovementProtocol: + * Option to specify a prefered data movement mechanism of the available options. + * + * +*/ +struct StorageResourceDescription { + 1: required string storageResourceId = airavata_commons.DEFAULT_ID, + 2: required string hostName, + 3: optional string storageResourceDescription, + 4: optional bool enabled, + 5: optional list<data_movement_models.DataMovementInterface> dataMovementInterfaces, +} http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/data-models/workflow-models/workflow_data_model.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/data-models/workflow-models/workflow_data_model.thrift b/thrift-interface-descriptions/data-models/workflow-models/workflow_data_model.thrift new file mode 100644 index 0000000..6b3dfad --- /dev/null +++ b/thrift-interface-descriptions/data-models/workflow-models/workflow_data_model.thrift @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + + +namespace java org.apache.airavata.model +namespace php Airavata.Model +namespace py apache.airavata.model.workflow + +include "../app-catalog-models/application_io_models.thrift" +include "../../airavata-apis/airavata_commons.thrift" + +/* + * This file describes the definitions of the Airavata Execution Data Structures. Each of the + * language specific Airavata Client SDK's will translate this neutral data model into an + * appropriate form for passing to the Airavata Server Execution API Calls. +*/ + + +struct Workflow { + 1: required string templateId = airavata_commons.DEFAULT_ID, + 2: required string name, + 3: optional string graph, + 4: optional binary image, + 5: optional list<application_io_models.InputDataObjectType> workflowInputs, + 6: optional list<application_io_models.OutputDataObjectType> workflowOutputs +} http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/generate-thrift-stubs.sh ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/generate-thrift-stubs.sh b/thrift-interface-descriptions/generate-thrift-stubs.sh index e3aad14..18c9a93 100755 --- a/thrift-interface-descriptions/generate-thrift-stubs.sh +++ b/thrift-interface-descriptions/generate-thrift-stubs.sh @@ -60,8 +60,16 @@ if [ "$VERSION" -ne 1 ] ; then fi # Global Constants used across the script -AIRAVATA_API_IDL_DIR='airavata-api' +AIRAVATA_API_IDL_DIR='airavata-apis' BASE_TARGET_DIR='target' + +# Thrift files +AIRAVATA_API_THRIFT_FILE='airavata-apis/airavata_api.thrift' +DATAMODEL_THRIFT_FILE='data-models/airavata_data_models.thrift' +APP_CATALOG_THRIFT_FILE='data-models/app-catalog-models/app_catalog_models.thrift' +RESOURCE_CATALOG_THRIFT_FILE='data-models/resource-catalog-models/resource_catalog_models.thrift' +WORKFLOW_THRIFT_FILE='data-models/workflow-models/workflow_data_model.thrift' + DATAMODEL_SRC_DIR='../airavata-api/airavata-data-models/src/main/java' JAVA_API_SDK_DIR='../airavata-api/airavata-api-stubs/src/main/java' PHP_SDK_DIR='../airavata-api/airavata-client-sdks/airavata-php-sdk/src/main/resources/lib' @@ -151,11 +159,13 @@ generate_java_stubs() { # Generate the Airavata Data Model using thrift Java Beans generator. This will take generate the classes in bean style # with members being private and setters returning voids. # The airavata_data_models.thrift includes rest of data models. - $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${AIRAVATA_API_IDL_DIR}/airavata_data_models.thrift || fail unable to generate java bean thrift classes on base data model + $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${DATAMODEL_THRIFT_FILE} || fail unable to generate java bean thrift classes on base data model + + $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${APP_CATALOG_THRIFT_FILE} || fail unable to generate java bean thrift classes on app catalog data models - $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${AIRAVATA_API_IDL_DIR}/app_catalog_models.thrift || fail unable to generate java bean thrift classes on app catalog data models + $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${RESOURCE_CATALOG_THRIFT_FILE} || fail unable to generate java bean thrift classes on app catalog data models - $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${AIRAVATA_API_IDL_DIR}/workflow_data_model.thrift || fail unable to generate java bean thrift classes on app workflow data models + $THRIFT_EXEC ${THRIFT_ARGS} --gen java:beans ${WORKFLOW_THRIFT_FILE} || fail unable to generate java bean thrift classes on app workflow data models # For the generated java beans add the ASF V2 License header add_license_header $JAVA_BEAN_GEN_DIR @@ -176,9 +186,7 @@ generate_java_stubs() { # Using thrift Java generator, generate the java classes based on Airavata API. This # The airavata_api.thrift includes rest of data models. - $THRIFT_EXEC ${THRIFT_ARGS} --gen java ${AIRAVATA_API_IDL_DIR}/airavata_api.thrift || fail unable to generate java thrift classes on AiravataAPI - - #$THRIFT_EXEC ${THRIFT_ARGS} --gen java ${AIRAVATA_API_IDL_DIR}/workflow_api.thrift || fail unable to generate java thrift classes on WorkflowAPI + $THRIFT_EXEC ${THRIFT_ARGS} --gen java ${AIRAVATA_API_THRIFT_FILE} || fail unable to generate java thrift classes on AiravataAPI # For the generated java classes add the ASF V2 License header add_license_header $JAVA_GEN_DIR @@ -202,13 +210,13 @@ generate_php_stubs() { # As a precaution remove and previously generated files if exists rm -rf ${PHP_GEN_DIR} - # Using thrift Java generator, generate the java classes based on Airavata API. This + # Using thrift Java generator, generate the PHP classes based on Airavata API. This # The airavata_api.thrift includes rest of data models. - $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${AIRAVATA_API_IDL_DIR}/airavata_data_models.thrift || fail unable to generate PHP thrift classes - $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${AIRAVATA_API_IDL_DIR}/app_catalog_models.thrift || fail unable to generate PHP thrift classes - $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${AIRAVATA_API_IDL_DIR}/airavata_api.thrift || fail unable to generate PHP thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${DATAMODEL_THRIFT_FILE} || fail unable to generate PHP thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${APP_CATALOG_THRIFT_FILE} || fail unable to generate PHP thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${RESOURCE_CATALOG_THRIFT_FILE} || fail unable to generate PHP thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${AIRAVATA_API_THRIFT_FILE} || fail unable to generate PHP thrift classes - #$THRIFT_EXEC ${THRIFT_ARGS} --gen php:autoload ${AIRAVATA_API_IDL_DIR}/workflow_api.thrift || fail unable to generate PHP thrift classes for WorkflowAPI # For the generated java classes add the ASF V2 License header ## TODO Write PHP license parser @@ -232,9 +240,8 @@ generate_cpp_stubs() { # Using thrift Java generator, generate the java classes based on Airavata API. This # The airavata_api.thrift includes rest of data models. - $THRIFT_EXEC ${THRIFT_ARGS} --gen cpp ${AIRAVATA_API_IDL_DIR}/airavata_api.thrift || fail unable to generate C++ thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen cpp ${AIRAVATA_API_THRIFT_FILE} || fail unable to generate C++ thrift classes - #$THRIFT_EXEC ${THRIFT_ARGS} --gen cpp ${AIRAVATA_API_IDL_DIR}/workflow_api.thrift || fail unable to generate C++ thrift classes for WorkflowAPI # For the generated CPP classes add the ASF V2 License header add_license_header $CPP_GEN_DIR @@ -258,7 +265,7 @@ generate_python_stubs() { # Using thrift Python generator, generate the python classes based on Airavata API. This # The airavata_api.thrift includes rest of data models. - $THRIFT_EXEC ${THRIFT_ARGS} --gen py ${AIRAVATA_API_IDL_DIR}/airavata_api.thrift || fail unable to generate Python thrift classes + $THRIFT_EXEC ${THRIFT_ARGS} --gen py ${AIRAVATA_API_THRIFT_FILE} || fail unable to generate Python thrift classes # For the generated CPP classes add the ASF V2 License header #add_license_header #PYTHON_GEN_DIR http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/gfac-cpi/generate-gfac-stubs.sh ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/gfac-cpi/generate-gfac-stubs.sh b/thrift-interface-descriptions/gfac-cpi/generate-gfac-stubs.sh deleted file mode 100755 index 34cb4f2..0000000 --- a/thrift-interface-descriptions/gfac-cpi/generate-gfac-stubs.sh +++ /dev/null @@ -1,127 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will regenerate the thrift code for Airavata GFAC Server Skeltons and Client Stubs. - - -# Global Constants used across the script -REQUIRED_THRIFT_VERSION='0.9.2' -BASE_TARGET_DIR='target' -GFAC_SERVICE_DIR='../../modules/gfac/gfac-client/src/main/java/' - -# The Funcation fail prints error messages on failure and quits the script. -fail() { - echo $@ - exit 1 -} - -# The funcation add_license_header adds the ASF V2 license header to all java files within the specified generated -# directory. The funcation also adds suppress all warnings annotation to all public classes and enum's -# To Call: -# add_license_header $generated_code_directory -add_license_header() { - - # Fetch the generated code directory passed as the argument - GENERATED_CODE_DIR=$1 - - # For each java file within the genrated directory, add the ASF V2 LICENSE header - for f in $(find ${GENERATED_CODE_DIR} -name '*.java'); do - cat - ${f} >${f}-with-license <<EOF - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -EOF - mv ${f}-with-license ${f} - done -} - -# The funcation compares every generated java file with the one in specified existing source location. If the comparision -# shows a difference, then it replaces with the newly generated file (with added license header). -# To Call: -# copy_changed_files $generated_code_directory $existing_source_directory -copy_changed_files() { - - # Read all the funcation arguments - GENERATED_CODE_DIR=$1 - WORKSPACE_SRC_DIR=$2 - - echo "Generated sources are in ${GENERATED_CODE_DIR}" - echo "Destination workspace is in ${WORKSPACE_SRC_DIR}" - - # Check if the newly generated files exist in the targetted workspace, if not copy. Only changed files will be synced. - # the extra slash to GENERATED_CODE_DIR is needed to ensure the parent directory itself is not copied. - rsync -auv ${GENERATED_CODE_DIR}/ ${WORKSPACE_SRC_DIR} -} - -# Generation of thrift files will require installing Apache Thrift. Please add thrift to your path. -# Verify is thrift is installed, is in the path is at a specified version. -VERSION=$(thrift -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l) -if [ "$VERSION" -ne 1 ] ; then - echo "****************************************************" - echo "*** thrift is not installed or is not in the path" - echo "*** expecting 'thrift -version' to return ${REQUIRED_THRIFT_VERSION}" - echo "*** generated code will not be updated" - fail "****************************************************" -fi - -# Initialize the thrift arguements. -# Since most of the Airavata API and Data Models have includes, use recursive option by defualt. -# Generate all the files in target directory -THRIFT_ARGS="-r -o ${BASE_TARGET_DIR}" -# Ensure the required target directories exists, if not create. -mkdir -p ${BASE_TARGET_DIR} - -####################################################################### -# Generate/Update the GFAC CPI service stubs -# To start with both the servicer and client are in same package, but -# needs to be split using a common generated api-boilerplate-code -####################################################################### - -#Java generation directory -JAVA_GEN_DIR=${BASE_TARGET_DIR}/gen-java - -# As a precausion remove and previously generated files if exists -rm -rf ${JAVA_GEN_DIR} - -# Using thrify Java generator, generate the java classes based on Airavata API. This -# The airavata_api.thrift includes rest of data models. -thrift ${THRIFT_ARGS} --gen java gfac.cpi.service.thrift || fail unable to generate java thrift classes - -# For the generated java classes add the ASF V2 License header -add_license_header $JAVA_GEN_DIR - -# Compare the newly generated classes with existing java generated skelton/stub sources and replace the changed ones. -copy_changed_files ${JAVA_GEN_DIR} ${GFAC_SERVICE_DIR} - -# CleanUp: Delete the base target build directory -#rm -rf ${BASE_TARGET_DIR} - -echo "Successfully generated new sources, compared against exiting code and replaced the changed files" -exit 0 http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/gfac-cpi/gfac.cpi.service.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/gfac-cpi/gfac.cpi.service.thrift b/thrift-interface-descriptions/gfac-cpi/gfac.cpi.service.thrift deleted file mode 100644 index d051514..0000000 --- a/thrift-interface-descriptions/gfac-cpi/gfac.cpi.service.thrift +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -/* - * Component Programming Interface definition for Apache Airavata GFac Service. - * -*/ - -namespace java org.apache.airavata.gfac.cpi - -const string GFAC_CPI_VERSION = "0.16.0" - -service GfacService { - - /** Query gfac server to fetch the CPI version */ - string getGFACServiceVersion(), - - /** - * @param processId - * @param gatewayId: The GatewayId is inferred from security context and passed onto gfac. - * @param tokenId - * @return sucess/failure - * - **/ - bool submitProcess (1: required string processId, - 2: required string gatewayId, - 3: required string tokenId) - - /** - * - * @param processId - * @param gatewayId: The GatewayId is inferred from security context and passed onto gfac. - * @param tokenId - * @return sucess/failure - * - **/ - bool cancelProcess (1: required string processId, - 2: required string gatewayId, - 3: required string tokenId) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/orchestrator-cpi/generate-orchestrator-stubs.sh ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/orchestrator-cpi/generate-orchestrator-stubs.sh b/thrift-interface-descriptions/orchestrator-cpi/generate-orchestrator-stubs.sh deleted file mode 100755 index 4b83cef..0000000 --- a/thrift-interface-descriptions/orchestrator-cpi/generate-orchestrator-stubs.sh +++ /dev/null @@ -1,131 +0,0 @@ -#! /usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will regenerate the thrift code for Airavata Orchestrator Server Skeltons and Client Stubs. - - -# Global Constants used across the script -REQUIRED_THRIFT_VERSION='0.9.2' -BASE_TARGET_DIR='target' -ORCHESTRATOR_SERVICE_DIR='../../modules/orchestrator/orchestrator-client/src/main/java' - -# The Funcation fail prints error messages on failure and quits the script. -fail() { - echo $@ - exit 1 -} - -# The funcation add_license_header adds the ASF V2 license header to all java files within the specified generated -# directory. The funcation also adds suppress all warnings annotation to all public classes and enum's -# To Call: -# add_license_header $generated_code_directory -add_license_header() { - - # Fetch the generated code directory passed as the argument - GENERATED_CODE_DIR=$1 - - # For each java file within the genrated directory, add the ASF V2 LICENSE header - for f in $(find ${GENERATED_CODE_DIR} -name '*.java'); do - cat - ${f} >${f}-with-license <<EOF - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -EOF - mv ${f}-with-license ${f} - done -} - -# The funcation compares every generated java file with the one in specified existing source location. If the comparision -# shows a difference, then it replaces with the newly generated file (with added license header). -# To Call: -# copy_changed_files $generated_code_directory $existing_source_directory -copy_changed_files() { - - # Read all the funcation arguments - GENERATED_CODE_DIR=$1 - WORKSPACE_SRC_DIR=$2 - - echo "Generated sources are in ${GENERATED_CODE_DIR}" - echo "Destination workspace is in ${WORKSPACE_SRC_DIR}" - - # Check if the newly generated files exist in the targetted workspace, if not copy. Only changed files will be synced. - # the extra slash to GENERATED_CODE_DIR is needed to ensure the parent directory itself is not copied. - rsync -auv ${GENERATED_CODE_DIR}/ ${WORKSPACE_SRC_DIR} -} - -# Generation of thrift files will require installing Apache Thrift. Please add thrift to your path. -# Verify is thrift is installed, is in the path is at a specified version. -VERSION=$(thrift -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l) -if [ "$VERSION" -ne 1 ] ; then - echo "****************************************************" - echo "*** thrift is not installed or is not in the path" - echo "*** expecting 'thrift -version' to return ${REQUIRED_THRIFT_VERSION}" - echo "*** generated code will not be updated" - fail "****************************************************" -fi - -# Initialize the thrift arguements. -# Since most of the Airavata API and Data Models have includes, use recursive option by defualt. -# Generate all the files in target directory -THRIFT_ARGS="-r -o ${BASE_TARGET_DIR}" -# Ensure the required target directories exists, if not create. -mkdir -p ${BASE_TARGET_DIR} - -####################################################################### -# Generate/Update the orchestrator CPI service stubs -# To start with both the servicer and client are in same package, but -# needs to be split using a common generated api-boilerplate-code -####################################################################### - -#Java generation directory -JAVA_GEN_DIR=${BASE_TARGET_DIR}/gen-java - -# As a precausion remove and previously generated files if exists -rm -rf ${JAVA_GEN_DIR} - -# Using thrify Java generator, generate the java classes based on Airavata API. This -# The airavata_api.thrift includes rest of data models. -thrift ${THRIFT_ARGS} --gen java orchestrator.cpi.service.thrift || fail unable to generate java thrift classes - -# Remove generated model classes, airavata api thrift file will generate those. -echo "Remove generated model classes ${JAVA_GEN_DIR}/org/airavata/apache/model" -rm -rf ${JAVA_GEN_DIR}/org/apache/airavata/model - -# For the generated java classes add the ASF V2 License header -add_license_header $JAVA_GEN_DIR - -# Compare the newly generated classes with existing java generated skelton/stub sources and replace the changed ones. -copy_changed_files ${JAVA_GEN_DIR} ${ORCHESTRATOR_SERVICE_DIR} - -# CleanUp: Delete the base target build directory -#rm -rf ${BASE_TARGET_DIR} - -echo "Successfully generated new sources, compared against exiting code and replaced the changed files" -exit 0 http://git-wip-us.apache.org/repos/asf/airavata/blob/6c5e9954/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift b/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift deleted file mode 100644 index 2212b4d..0000000 --- a/thrift-interface-descriptions/orchestrator-cpi/orchestrator.cpi.service.thrift +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -/* - * Component Programming Interface definition for Apache Airavata Orchestration Service. - * -*/ - -include "../airavata-api/airavata_errors.thrift" -include "../airavata-api/process_model.thrift" -namespace java org.apache.airavata.orchestrator.cpi - -const string ORCHESTRATOR_CPI_VERSION = "0.16.0" - -service OrchestratorService { - - /** Query orchestrator server to fetch the CPI version */ - string getOrchestratorCPIVersion(), - - /** - * After creating the experiment Data user have the - * experimentID as the handler to the experiment, during the launchExperiment - * We just have to give the experimentID - * - * @param experimentID - * @return sucess/failure - * - **/ - bool launchExperiment (1: required string experimentId, 2: required string gatewayId), - - /** - * In order to run single applications users should create an associating - * process and hand it over for execution - * along with a credential store token for sshKeyAuthentication - * - * @param processId - * @param airavataCredStoreToken - * @return sucess/failure - * - **/ - bool launchProcess (1: required string processId, 2: required string airavataCredStoreToken, 3: required string gatewayId), - - /** - * - * Validate funcations which can verify if the experiment is ready to be launced. - * - * @param experimentID - * @return sucess/failure - * - **/ - bool validateExperiment(1: required string experimentId) throws (1: airavata_errors.LaunchValidationException lve) - - bool validateProcess(1: required string experimentId, 2: required list<process_model.ProcessModel> processes) throws (1: airavata_errors.LaunchValidationException lve) - /** - * - * Terminate the running experiment. - * - * @param experimentID - * @return sucess/failure - * - **/ - bool terminateExperiment (1: required string experimentId, 2: required string gatewayId) -}
