Move aurora-scheduler properties to airavata-server.properties, AND detect leading aurora scheduler
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/01f80443 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/01f80443 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/01f80443 Branch: refs/heads/develop Commit: 01f804433e0745aca8c5e9f8fd48fb3efd33b25e Parents: c6e96ca Author: Gourav Shenoy <[email protected]> Authored: Wed Nov 2 17:50:48 2016 -0400 Committer: Gourav Shenoy <[email protected]> Committed: Wed Nov 2 17:50:48 2016 -0400 ---------------------------------------------------------------------- modules/cloud/aurora-client/pom.xml | 9 +-- .../client/AuroraSchedulerClientFactory.java | 45 +++++++++-- .../cloud/aurora/client/AuroraThriftClient.java | 28 +++---- .../aurora/client/bean/GetJobsResponseBean.java | 10 +++ .../cloud/aurora/sample/AuroraClientSample.java | 80 ++++++++------------ .../aurora/util/AuroraThriftClientUtil.java | 69 +++++++++++++---- .../airavata/cloud/aurora/util/Constants.java | 5 +- .../main/resources/aurora-scheduler.properties | 4 +- .../airavata/common/utils/ServerSettings.java | 26 +++++++ .../main/resources/airavata-server.properties | 9 +++ .../gfac/impl/task/AuroraJobSubmission.java | 14 ++-- .../gfac/monitor/cloud/AuroraJobMonitor.java | 2 +- 12 files changed, 203 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/pom.xml ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/pom.xml b/modules/cloud/aurora-client/pom.xml index 20f8f7c..f8c41a4 100644 --- a/modules/cloud/aurora-client/pom.xml +++ b/modules/cloud/aurora-client/pom.xml @@ -32,7 +32,6 @@ <artifactId>aurora-client</artifactId> <packaging>jar</packaging> <name>Thrift client for Apache Aurora</name> - <version>0.17-SNAPSHOT</version> <url>http://airavata.apache.org/</url> <dependencies> @@ -47,10 +46,10 @@ <version>${json.version}</version> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>${commons.io.version}</version> - </dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-commons</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraSchedulerClientFactory.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraSchedulerClientFactory.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraSchedulerClientFactory.java index 643b8d7..1a5d3fa 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraSchedulerClientFactory.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraSchedulerClientFactory.java @@ -36,31 +36,60 @@ public class AuroraSchedulerClientFactory { /** The Constant logger. */ private final static Logger logger = LoggerFactory.getLogger(AuroraSchedulerClientFactory.class); + /** * Creates a new AuroraSchedulerClient object. * * @param connectionUrl the connection url + * @param connectionTimeout the connection timeout * @return the client * @throws Exception the exception */ - public static ReadOnlyScheduler.Client createReadOnlySchedulerClient(String connectionUrl) throws Exception { + public static ReadOnlyScheduler.Client createReadOnlySchedulerClient(String connectionUrl, int connectionTimeout) throws Exception { try { - TTransport transport = new THttpClient(connectionUrl); - transport.open(); - TProtocol protocol = new TJSONProtocol(transport); - return new ReadOnlyScheduler.Client(protocol); + return new ReadOnlyScheduler.Client( + getTProtocol(connectionUrl, connectionTimeout)); + } catch(Exception ex) { + logger.error(ex.getMessage(), ex); + throw ex; + } + } + + /** + * Creates a new AuroraSchedulerClient object. + * + * @param connectionUrl the connection url + * @param connectionTimeout the connection timeout + * @return the client + * @throws Exception the exception + */ + public static AuroraSchedulerManager.Client createSchedulerManagerClient(String connectionUrl, int connectionTimeout) throws Exception { + try { + return new AuroraSchedulerManager.Client( + getTProtocol(connectionUrl, connectionTimeout)); } catch(Exception ex) { logger.error(ex.getMessage(), ex); throw ex; } } - public static AuroraSchedulerManager.Client createSchedulerManagerClient(String connectionUrl) throws Exception { + /** + * Gets the t protocol. + * + * @param connectionUrl the connection url + * @param connectionTimeout the connection timeout + * @return the t protocol + * @throws Exception the exception + */ + private static TProtocol getTProtocol(String connectionUrl, int connectionTimeout) throws Exception { try { - TTransport transport = new THttpClient(connectionUrl); + THttpClient client = new THttpClient(connectionUrl); + client.setConnectTimeout(connectionTimeout); + + TTransport transport = client; transport.open(); TProtocol protocol = new TJSONProtocol(transport); - return new AuroraSchedulerManager.Client(protocol); + return protocol; } catch(Exception ex) { logger.error(ex.getMessage(), ex); throw ex; http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java index 0e0b36e..977479b 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java @@ -21,7 +21,6 @@ package org.apache.airavata.cloud.aurora.client; import java.text.MessageFormat; import java.util.HashSet; -import java.util.Properties; import java.util.Set; import org.apache.airavata.cloud.aurora.client.bean.GetJobsResponseBean; @@ -36,10 +35,10 @@ import org.apache.airavata.cloud.aurora.client.sdk.JobKey; import org.apache.airavata.cloud.aurora.client.sdk.ReadOnlyScheduler; import org.apache.airavata.cloud.aurora.client.sdk.Response; import org.apache.airavata.cloud.aurora.client.sdk.TaskQuery; -import org.apache.airavata.cloud.aurora.sample.AuroraClientSample; import org.apache.airavata.cloud.aurora.util.AuroraThriftClientUtil; import org.apache.airavata.cloud.aurora.util.Constants; import org.apache.airavata.cloud.aurora.util.ResponseResultType; +import org.apache.airavata.common.utils.ServerSettings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,9 +50,6 @@ public class AuroraThriftClient { /** The Constant logger. */ private final static Logger logger = LoggerFactory.getLogger(AuroraThriftClient.class); - /** The properties. */ - private static Properties properties = new Properties(); - /** The read only scheduler client. */ private ReadOnlyScheduler.Client readOnlySchedulerClient = null; @@ -71,23 +67,29 @@ public class AuroraThriftClient { /** * Gets the aurora thrift client. * - * @param auroraSchedulerPropFile the aurora scheduler prop file * @return the aurora thrift client * @throws Exception the exception */ - public static AuroraThriftClient getAuroraThriftClient(String auroraSchedulerPropFile) throws Exception { + public static AuroraThriftClient getAuroraThriftClient() throws Exception { try { if(thriftClient == null) { thriftClient = new AuroraThriftClient(); // construct connection url for scheduler - properties.load(AuroraClientSample.class.getClassLoader().getResourceAsStream(auroraSchedulerPropFile)); - String auroraHost = properties.getProperty(Constants.AURORA_SCHEDULER_HOST); - String auroraPort = properties.getProperty(Constants.AURORA_SCHEDULER_PORT); - String connectionUrl = MessageFormat.format(Constants.AURORA_SCHEDULER_CONNECTION_URL, auroraHost, auroraPort); + String auroraHosts = ServerSettings.getAuroraSchedulerHosts(); + Integer connectTimeout = ServerSettings.getAuroraSchedulerTimeout(); - thriftClient.readOnlySchedulerClient = AuroraSchedulerClientFactory.createReadOnlySchedulerClient(connectionUrl); - thriftClient.auroraSchedulerManagerClient = AuroraSchedulerClientFactory.createSchedulerManagerClient(connectionUrl); + // check reachable scheduler host + for(String auroraHost : auroraHosts.split(",")) { + String hostname = auroraHost.split(":")[0]; + String port = auroraHost.split(":")[1]; + String connectionUrl = MessageFormat.format(Constants.AURORA_SCHEDULER_CONNECTION_URL, hostname, port); + + if(AuroraThriftClientUtil.isSchedulerHostReachable(connectionUrl, connectTimeout)) { + thriftClient.readOnlySchedulerClient = AuroraSchedulerClientFactory.createReadOnlySchedulerClient(connectionUrl, connectTimeout); + thriftClient.auroraSchedulerManagerClient = AuroraSchedulerClientFactory.createSchedulerManagerClient(connectionUrl, connectTimeout); + } + } } } catch(Exception ex) { logger.error(ex.getMessage(), ex); http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/bean/GetJobsResponseBean.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/bean/GetJobsResponseBean.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/bean/GetJobsResponseBean.java index 6ab822b..71786a0 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/bean/GetJobsResponseBean.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/bean/GetJobsResponseBean.java @@ -29,6 +29,16 @@ public class GetJobsResponseBean extends ResponseBean { /** The job configs. */ private Set<JobConfigBean> jobConfigs; + + /** + * Instantiates a new gets the jobs response bean. + * + * @param responseBean the response bean + */ + public GetJobsResponseBean(ResponseBean responseBean) { + this.setResponseCode(responseBean.getResponseCode()); + this.setServerInfo(responseBean.getServerInfo()); + } /** * Gets the job configs. http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java index 3f29969..10cac08 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java @@ -19,15 +19,13 @@ */ package org.apache.airavata.cloud.aurora.sample; -import java.text.MessageFormat; import java.util.HashSet; import java.util.LinkedHashSet; -import java.util.Properties; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; -import org.apache.airavata.cloud.aurora.client.AuroraSchedulerClientFactory; import org.apache.airavata.cloud.aurora.client.AuroraThriftClient; +import org.apache.airavata.cloud.aurora.client.bean.GetJobsResponseBean; import org.apache.airavata.cloud.aurora.client.bean.IdentityBean; import org.apache.airavata.cloud.aurora.client.bean.JobConfigBean; import org.apache.airavata.cloud.aurora.client.bean.JobKeyBean; @@ -35,16 +33,7 @@ import org.apache.airavata.cloud.aurora.client.bean.ProcessBean; import org.apache.airavata.cloud.aurora.client.bean.ResourceBean; import org.apache.airavata.cloud.aurora.client.bean.ResponseBean; import org.apache.airavata.cloud.aurora.client.bean.TaskConfigBean; -import org.apache.airavata.cloud.aurora.client.sdk.ExecutorConfig; -import org.apache.airavata.cloud.aurora.client.sdk.GetJobsResult; -import org.apache.airavata.cloud.aurora.client.sdk.Identity; -import org.apache.airavata.cloud.aurora.client.sdk.JobConfiguration; -import org.apache.airavata.cloud.aurora.client.sdk.JobKey; -import org.apache.airavata.cloud.aurora.client.sdk.ReadOnlyScheduler; -import org.apache.airavata.cloud.aurora.client.sdk.Response; -import org.apache.airavata.cloud.aurora.client.sdk.TaskConfig; import org.apache.airavata.cloud.aurora.util.AuroraThriftClientUtil; -import org.apache.airavata.cloud.aurora.util.Constants; import org.apache.thrift.TException; /** @@ -52,35 +41,35 @@ import org.apache.thrift.TException; */ public class AuroraClientSample { - /** The aurora scheduler client. */ - private static ReadOnlyScheduler.Client auroraSchedulerClient; - - /** The properties. */ - private static Properties properties = new Properties(); - /** * Gets the job summary. * * @param client the client * @return the job summary + * @throws Exception */ - public static void getJobSummary(ReadOnlyScheduler.Client client) { + public static void getRunningJobsList(String ownerRole) throws Exception { try { - Response response = client.getJobs("centos"); + AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(); + ResponseBean response = client.getJobList(ownerRole); System.out.println("Response status: " + response.getResponseCode().name()); - if(response.getResult().isSetGetJobsResult()) { - GetJobsResult result = response.getResult().getGetJobsResult(); + if(response instanceof GetJobsResponseBean) { + GetJobsResponseBean result = (GetJobsResponseBean) response; System.out.println(result); - Set<JobConfiguration> jobConfigs = result.getConfigs(); - for(JobConfiguration jobConfig : jobConfigs) { + + Set<JobConfigBean> jobConfigs = result.getJobConfigs(); + for(JobConfigBean jobConfig : jobConfigs) { System.out.println(jobConfig); - JobKey jobKey = jobConfig.getKey(); - Identity owner = jobConfig.getOwner(); - TaskConfig taskConfig = jobConfig.getTaskConfig(); - ExecutorConfig exeConfig = taskConfig.getExecutorConfig(); + JobKeyBean jobKey = jobConfig.getJob(); + IdentityBean owner = jobConfig.getOwner(); + TaskConfigBean taskConfig = jobConfig.getTaskConfig(); + Set<ProcessBean> processes = taskConfig.getProcesses(); System.out.println("\n**** JOB CONFIG ****"); - System.out.println("\t # instanceCount: " + jobConfig.getInstanceCount()); + System.out.println("\t # cluster: " + jobConfig.getCluster()); + System.out.println("\t # instanceCount: " + jobConfig.getInstances()); + System.out.println("\t # isService: " + jobConfig.isService()); + System.out.println("\t\t # priority: " + jobConfig.getPriority()); System.out.println("\t >> Job Key <<"); System.out.println("\t\t # name: " + jobKey.getName()); @@ -91,15 +80,17 @@ public class AuroraClientSample { System.out.println("\t\t # owner: " + owner.getUser()); System.out.println("\t >> Task Config <<"); - System.out.println("\t\t # numCPUs: " + taskConfig.getNumCpus()); - System.out.println("\t\t # diskMb: " + taskConfig.getDiskMb()); - System.out.println("\t\t # ramMb: " + taskConfig.getRamMb()); - System.out.println("\t\t # priority: " + taskConfig.getPriority()); + System.out.println("\t\t >> Resources <<"); + System.out.println("\t\t\t # numCPUs: " + taskConfig.getResources().getNumCpus()); + System.out.println("\t\t\t # diskMb: " + taskConfig.getResources().getDiskMb()); + System.out.println("\t\t\t # ramMb: " + taskConfig.getResources().getRamMb()); - - System.out.println("\t >> Executor Config <<"); - System.out.println("\t\t # name: " + exeConfig.getName()); - System.out.println("\t\t # data: " + exeConfig.getData()); + System.out.println("\t\t >> Processes <<"); + for(ProcessBean process : processes) { + System.out.println("\t\t\t ***** PROCESS *****"); + System.out.println("\t\t\t # name: " + process.getName()); + System.out.println("\t\t\t # cmdline: " + process.getCmdLine()); + } } } @@ -126,7 +117,7 @@ public class AuroraClientSample { String executorConfigJson = AuroraThriftClientUtil.getExecutorConfigJson(jobConfig); System.out.println(executorConfigJson); - AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE); + AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(); ResponseBean response = client.createJob(jobConfig); System.out.println(response); } @@ -156,14 +147,14 @@ public class AuroraClientSample { String executorConfigJson = AuroraThriftClientUtil.getExecutorConfigJson(jobConfig); System.out.println(executorConfigJson); - AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE); + AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(); ResponseBean response = client.createJob(jobConfig); System.out.println(response); } public static void killTasks(String jobName) throws Exception { JobKeyBean jobKey = new JobKeyBean("devel", "centos", jobName); - AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE); + AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(); ResponseBean response = client.killTasks(jobKey, new HashSet<>()); System.out.println(response); } @@ -175,20 +166,15 @@ public class AuroraClientSample { */ public static void main(String[] args) { try { - properties.load(AuroraClientSample.class.getClassLoader().getResourceAsStream(Constants.AURORA_SCHEDULER_PROP_FILE)); - String auroraHost = properties.getProperty(Constants.AURORA_SCHEDULER_HOST); - String auroraPort = properties.getProperty(Constants.AURORA_SCHEDULER_PORT); - auroraSchedulerClient = AuroraSchedulerClientFactory.createReadOnlySchedulerClient(MessageFormat.format(Constants.AURORA_SCHEDULER_CONNECTION_URL, auroraHost, auroraPort)); - // create sample job // AuroraClientSample.createJob(); - AuroraClientSample.createAutoDockJob(); +// AuroraClientSample.createAutoDockJob(); // kill pending job // AuroraClientSample.killTasks("test_autodock"); // get jobs summary - AuroraClientSample.getJobSummary(auroraSchedulerClient); + AuroraClientSample.getRunningJobsList("centos"); // AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE); // ResponseBean response = client.getPendingReasonForJob(new JobKeyBean("devel", "centos", "hello_pending")); http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/AuroraThriftClientUtil.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/AuroraThriftClientUtil.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/AuroraThriftClientUtil.java index 7bf3fbb..f66c739 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/AuroraThriftClientUtil.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/AuroraThriftClientUtil.java @@ -19,11 +19,11 @@ */ package org.apache.airavata.cloud.aurora.util; -import java.nio.charset.Charset; import java.util.HashSet; -import java.util.Properties; import java.util.Set; +import org.apache.airavata.cloud.aurora.client.AuroraSchedulerClientFactory; +import org.apache.airavata.cloud.aurora.client.bean.GetJobsResponseBean; import org.apache.airavata.cloud.aurora.client.bean.IdentityBean; import org.apache.airavata.cloud.aurora.client.bean.JobConfigBean; import org.apache.airavata.cloud.aurora.client.bean.JobDetailsResponseBean; @@ -37,11 +37,11 @@ import org.apache.airavata.cloud.aurora.client.sdk.ExecutorConfig; import org.apache.airavata.cloud.aurora.client.sdk.Identity; import org.apache.airavata.cloud.aurora.client.sdk.JobConfiguration; import org.apache.airavata.cloud.aurora.client.sdk.JobKey; +import org.apache.airavata.cloud.aurora.client.sdk.ReadOnlyScheduler; import org.apache.airavata.cloud.aurora.client.sdk.Resource; import org.apache.airavata.cloud.aurora.client.sdk.Response; import org.apache.airavata.cloud.aurora.client.sdk.TaskConfig; -import org.apache.airavata.cloud.aurora.sample.AuroraClientSample; -import org.apache.commons.io.IOUtils; +import org.apache.airavata.common.utils.ServerSettings; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,9 +54,6 @@ public class AuroraThriftClientUtil { /** The Constant logger. */ private final static Logger logger = LoggerFactory.getLogger(AuroraThriftClientUtil.class); - /** The properties. */ - private static Properties properties = new Properties(); - /** * Gets the executor config json. * @@ -67,9 +64,11 @@ public class AuroraThriftClientUtil { public static String getExecutorConfigJson(JobConfigBean jobConfig) throws Exception { String exeConfigJson = null; try { - String template = IOUtils.toString(AuroraThriftClientUtil.class.getClassLoader() - .getResourceAsStream("executor-config-template.json"), Charset.defaultCharset()); - + // read the executor config json template + java.net.URL url = AuroraThriftClientUtil.class.getClassLoader().getResource(ServerSettings.getAuroraExecutorConfigTemplateFileName()); + java.nio.file.Path resPath = java.nio.file.Paths.get(url.toURI()); + String template = new String(java.nio.file.Files.readAllBytes(resPath), "UTF8"); + JSONObject exeConfig = new JSONObject(template); if(exeConfig != null) { exeConfig.put("environment", jobConfig.getJob().getEnvironment()); @@ -172,8 +171,7 @@ public class AuroraThriftClientUtil { ExecutorConfig exeConfig = null; try { - properties.load(AuroraClientSample.class.getClassLoader().getResourceAsStream(Constants.AURORA_SCHEDULER_PROP_FILE)); - String executorName = properties.getProperty(Constants.AURORA_EXECUTOR_NAME); + String executorName = ServerSettings.getAuroraExecutorName(); // create the executor config if(exeConfigJson != null) { @@ -294,6 +292,8 @@ public class AuroraThriftClientUtil { */ public static ResponseBean getResponseBean(Response response, ResponseResultType resultType) { switch (resultType) { + case GET_JOBS: + return getJobsResponseBean(response); case GET_JOB_DETAILS: return getJobDetailsResponseBean(response); case GET_PENDING_JOB_REASON: @@ -336,6 +336,22 @@ public class AuroraThriftClientUtil { } /** + * Gets the jobs response bean. + * + * @param response the response + * @return the jobs response bean + */ + private static GetJobsResponseBean getJobsResponseBean(Response response) { + GetJobsResponseBean responseBean = null; + if(response != null) { + responseBean = new GetJobsResponseBean(getJobResponse(response)); + //TODO: set jobconfig list in response + } + + return responseBean; + } + + /** * Gets the job response. * * @param response the response @@ -357,12 +373,35 @@ public class AuroraThriftClientUtil { } /** + * Checks if is scheduler host reachable. + * + * @param connectionUrl the connection url + * @param connectionTimeout the connection timeout + * @return true, if is scheduler host reachable + */ + public static boolean isSchedulerHostReachable(String connectionUrl, int connectionTimeout) { + boolean isReachable = false; + ReadOnlyScheduler.Client auroraSchedulerClient = null; + try { + // connect to scheduler & run dummy command + auroraSchedulerClient = AuroraSchedulerClientFactory.createReadOnlySchedulerClient(connectionUrl, connectionTimeout); + auroraSchedulerClient.getTierConfigs(); + + // host is reachable + isReachable = true; + } catch(Exception ex) { + logger.error("Timed-out connecting to URL: " + connectionUrl, ex); + } + return isReachable; + } + + /** * The main method. * * @param args the arguments * @throws Exception the exception */ -// public static void main(String[] args) throws Exception { + public static void main(String[] args) throws Exception { // JobKeyBean jobKey = new JobKeyBean("devel", "centos", "test_job"); // IdentityBean owner = new IdentityBean("centos"); // @@ -379,5 +418,7 @@ public class AuroraThriftClientUtil { // // String executorConfigJson = getExecutorConfigJson(jobConfig); // System.out.println(executorConfigJson); -// } + +// System.out.println(new Scanner(AuroraThriftClientUtil.class.getClassLoader().getResourceAsStream("executor-config-template.json"), "UTF-8").useDelimiter("\\A").next()); + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/Constants.java ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/Constants.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/Constants.java index a3e862a..2796541 100644 --- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/Constants.java +++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/util/Constants.java @@ -28,7 +28,7 @@ public class Constants { public static final String AURORA_SCHEDULER_PROP_FILE = "aurora-scheduler.properties"; /** The Constant AURORA_SCHEDULER_HOST. */ - public static final String AURORA_SCHEDULER_HOST = "aurora.scheduler.host"; + public static final String AURORA_SCHEDULER_HOSTS = "aurora.scheduler.hosts"; /** The Constant AURORA_SCHEDULER_PORT. */ public static final String AURORA_SCHEDULER_PORT = "aurora.scheduler.port"; @@ -41,4 +41,7 @@ public class Constants { /** The Constant AURORA_SCHEDULER_CONNECTION_URL. */ public static final String AURORA_SCHEDULER_CONNECTION_URL = "http://{0}:{1}/api";; + + /** The Constant AURORA_SCHEDULER_CONNECT_TIMEOUT_MS. */ + public static final String AURORA_SCHEDULER_CONNECT_TIMEOUT_MS = "aurora.scheduler.timeoutms"; } http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties ---------------------------------------------------------------------- diff --git a/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties b/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties index cf21b8f..8011de9 100644 --- a/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties +++ b/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties @@ -1,9 +1,11 @@ #Aurora scheduler properties -aurora.scheduler.host=52.15.103.85 +aurora.scheduler.hosts=sgg-mesos-master-1,sgg-mesos-master-2,sgg-mesos-master-3 aurora.scheduler.port=8081 +aurora.scheduler.timeoutms=1000 + aurora.executor.name=AuroraExecutor mesos.cluster.name=example \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java index 3ac2a6e..46b1ebb 100644 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java @@ -64,6 +64,12 @@ public class ServerSettings extends ApplicationSettings { public static final String ZOOKEEPER_SERVER_CONNECTION = "zookeeper.server.connection"; public static final String ZOOKEEPER_TIMEOUT = "zookeeper.timeout"; + // Aurora Scheduler Constants + public static final String AURORA_SCHEDULER_HOSTS = "aurora.scheduler.hosts"; + public static final String AURORA_EXECUTOR_NAME = "aurora.executor.name"; + public static final String MESOS_CLUSTER_NAME = "mesos.cluster.name"; + public static final String AURORA_SCHEDULER_CONNECT_TIMEOUT_MS = "aurora.scheduler.timeoutms"; + public static final String AURORA_EXECUTOR_CONFIG_TEMPLATE_FILE = "aurora.executor.config.template.filename"; private static final String CREDENTIAL_STORE_DB_URL = "credential.store.jdbc.url"; private static final String CREDENTIAL_STORE_DB_USER = "credential.store.jdbc.user"; @@ -439,4 +445,24 @@ public class ServerSettings extends ApplicationSettings { public static String[] getServerRoles() { return listConfigurations.get(SERVER_ROLES); } + + public static String getAuroraSchedulerHosts() throws ApplicationSettingsException { + return getSetting(AURORA_SCHEDULER_HOSTS); + } + + public static String getMesosClusterName() throws ApplicationSettingsException { + return getSetting(MESOS_CLUSTER_NAME); + } + + public static String getAuroraExecutorName() throws ApplicationSettingsException { + return getSetting(AURORA_EXECUTOR_NAME); + } + + public static String getAuroraExecutorConfigTemplateFileName() throws ApplicationSettingsException { + return getSetting(AURORA_EXECUTOR_CONFIG_TEMPLATE_FILE); + } + + public static int getAuroraSchedulerTimeout() throws ApplicationSettingsException { + return Integer.valueOf(getSetting(AURORA_SCHEDULER_CONNECT_TIMEOUT_MS)); + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/configuration/server/src/main/resources/airavata-server.properties ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties index c823d89..e5b6b33 100644 --- a/modules/configuration/server/src/main/resources/airavata-server.properties +++ b/modules/configuration/server/src/main/resources/airavata-server.properties @@ -282,6 +282,15 @@ embedded.zk=true zookeeper.server.connection=localhost:2181 zookeeper.timeout=30000 +########################################################################### +# Aurora Scheduler Configuration +########################################################################### +aurora.scheduler.hosts=sgg-mesos-master-1:8081,sgg-mesos-master-2:8081,sgg-mesos-master-3:8081 +aurora.scheduler.timeoutms=1000 +aurora.executor.name=AuroraExecutor +mesos.cluster.name=example +aurora.executor.config.template.filename=executor-config-template.json + ######################################################################## ## API Security Configuration ######################################################################## http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java index 6035cec..f51503d 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java @@ -20,6 +20,11 @@ */ package org.apache.airavata.gfac.impl.task; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + import org.apache.airavata.cloud.aurora.client.AuroraThriftClient; import org.apache.airavata.cloud.aurora.client.bean.IdentityBean; import org.apache.airavata.cloud.aurora.client.bean.JobConfigBean; @@ -29,7 +34,6 @@ import org.apache.airavata.cloud.aurora.client.bean.ResourceBean; import org.apache.airavata.cloud.aurora.client.bean.ResponseBean; import org.apache.airavata.cloud.aurora.client.bean.TaskConfigBean; import org.apache.airavata.cloud.aurora.util.AuroraThriftClientUtil; -import org.apache.airavata.cloud.aurora.util.Constants; import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.gfac.core.GFacException; import org.apache.airavata.gfac.core.GFacUtils; @@ -48,12 +52,6 @@ import org.apache.airavata.model.task.TaskTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; - public class AuroraJobSubmission implements JobSubmissionTask{ private static final Logger log = LoggerFactory.getLogger(AuroraJobSubmission.class); @@ -102,7 +100,7 @@ public class AuroraJobSubmission implements JobSubmissionTask{ String executorConfigJson = AuroraThriftClientUtil.getExecutorConfigJson(jobConfig); log.info("Executor Config for Job {} , {}", jobIdAndName, executorConfigJson); - AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE); + AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(); ResponseBean response = client.createJob(jobConfig); log.info("Response for job {}, {}", jobIdAndName, response); jobModel.setJobDescription(resources.toString()); http://git-wip-us.apache.org/repos/asf/airavata/blob/01f80443/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java index 9b94623..65eb221 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java @@ -118,7 +118,7 @@ public class AuroraJobMonitor implements JobMonitor, Runnable { AuroraThriftClient client; public AuroraTimer() throws Exception { - client = AuroraThriftClient.getAuroraThriftClient(AuroraUtils.AURORA_SCHEDULER_PROP_FILE); + client = AuroraThriftClient.getAuroraThriftClient(); }
