This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch staging
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/staging by this push:
new fbabb5a Adding status monitor for load client
fbabb5a is described below
commit fbabb5a32057bea892733eb921fb138a02cc55b2
Author: Dimuthu Wannipurage <[email protected]>
AuthorDate: Sat Mar 30 10:29:57 2019 -0400
Adding status monitor for load client
---
.../org/apache/airavata/tools/load/LoadClient.java | 29 +++++++--
.../apache/airavata/tools/load/StatusMonitor.java | 68 ++++++++++++++++++++++
.../org/apache/airavata/tools/load/UnitLoad.java | 31 ++++++----
.../src/main/resources/conf/load-config.yml | 10 ++--
4 files changed, 117 insertions(+), 21 deletions(-)
diff --git
a/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
index 16d96d0..64b22c6 100644
---
a/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
+++
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/LoadClient.java
@@ -2,6 +2,7 @@ package org.apache.airavata.tools.load;
import org.apache.airavata.api.Airavata;
import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.model.appcatalog.gatewayprofile.StoragePreference;
import
org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
import org.apache.airavata.model.security.AuthzToken;
@@ -10,9 +11,12 @@ import org.yaml.snakeyaml.Yaml;
import java.io.FileInputStream;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletionService;
+import java.util.concurrent.Future;
public class LoadClient {
@@ -45,17 +49,34 @@ public class LoadClient {
public void start() throws Exception {
for (Configuration configuration : configurations.getConfigurations())
{
+
+ AuthzToken mockAuthzToken = new AuthzToken("");
+ Map<String, String> claimsMap = new HashMap<>();
+ claimsMap.put(Constants.USER_NAME, configuration.getUserId());
+ claimsMap.put(Constants.GATEWAY_ID, configuration.getGatewayId());
+ mockAuthzToken.setClaimsMap(claimsMap);
+
UnitLoad unitLoad = new UnitLoad(configurations.getApiHost(),
configurations.getApiPort(),
securityManager.getTrustStorePath(),
securityManager.getTrustStorePassword(),
-
storageResourceManagerStore.get(configuration.getStorageResourceId()));
- CompletionService<Boolean> completion =
unitLoad.execute(configuration);
+
storageResourceManagerStore.get(configuration.getStorageResourceId()),
mockAuthzToken);
+
+ StatusMonitor statusMonitor = new
StatusMonitor(configurations.getApiHost(), configurations.getApiPort(),
+ securityManager.getTrustStorePath(),
securityManager.getTrustStorePassword(), mockAuthzToken);
+
+ CompletionService<List<String>> completion =
unitLoad.execute(configuration);
+
+ List<String> allExperiments = new ArrayList<>();
for (int i = 0; i < configuration.getConcurrentUsers(); i++) {
- completion.take();
+ Future<List<String>> experimentsPerUser = completion.take();
+ allExperiments.addAll(experimentsPerUser.get());
}
+ System.out.println("All experiments " );
+ System.out.println(allExperiments);
+ statusMonitor.monitorExperiments(allExperiments);
}
destroyStorageResourceManagers();
- System.out.println("Finished load");
+ System.out.println("Finished load ");
System.exit(0);
}
diff --git
a/tools/load-client/src/main/java/org/apache/airavata/tools/load/StatusMonitor.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StatusMonitor.java
new file mode 100644
index 0000000..9246df3
--- /dev/null
+++
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/StatusMonitor.java
@@ -0,0 +1,68 @@
+package org.apache.airavata.tools.load;
+
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.error.AiravataClientException;
+import org.apache.airavata.model.experiment.ExperimentModel;
+import org.apache.airavata.model.job.JobModel;
+import org.apache.airavata.model.security.AuthzToken;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.thrift.TException;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class StatusMonitor {
+
+ private String apiHost;
+ private int apiPort;
+ private String trustStorePath;
+ private String trustStorePassword;
+ private AuthzToken authzToken;
+
+ public StatusMonitor(String apiHost, int apiPort, String trustStorePath,
String trustStorePassword, AuthzToken authzToken) throws
AiravataClientException {
+ this.apiHost = apiHost;
+ this.apiPort = apiPort;
+ this.trustStorePath = trustStorePath;
+ this.trustStorePassword = trustStorePassword;
+ this.authzToken = authzToken;
+ }
+
+ public void monitorExperiments(List<String> experiments) throws TException
{
+
+ Airavata.Client airavataClient =
AiravataClientFactory.createAiravataSecureClient(apiHost, apiPort,
trustStorePath, trustStorePassword, 100000);
+
+ Map<String, JobModel> jobModelMap = new HashMap<>();
+ Map<String, ExperimentModel> experimentModelMap = new HashMap<>();
+
+ for (String experiment : experiments) {
+ experimentModelMap.put(experiment,
airavataClient.getExperiment(authzToken, experiment));
+ }
+
+ while(experiments.size() > jobModelMap.size()) {
+ for (String experiment : experiments) {
+ if (jobModelMap.containsKey(experiment)) {
+ continue;
+ }
+ List<JobModel> jobDetails =
airavataClient.getJobDetails(authzToken, experiment);
+ if (jobDetails.size() > 0) {
+ jobModelMap.put(experiment, jobDetails.get(0));
+ }
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ long totalTime = 0;
+
+ for (String experiment : experiments) {
+ totalTime += jobModelMap.get(experiment).getCreationTime() -
experimentModelMap.get(experiment).getCreationTime();
+ }
+ System.out.println("All jobs created");
+ System.out.println("Average time " + (totalTime
*1.0/experiments.size())/1000 + " s");
+ }
+}
diff --git
a/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
index 97e7496..72a9f57 100644
---
a/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
+++
b/tools/load-client/src/main/java/org/apache/airavata/tools/load/UnitLoad.java
@@ -28,19 +28,22 @@ public class UnitLoad {
private String trustStorePath;
private String trustStorePassword;
private StorageResourceManager storageResourceManager;
+ private AuthzToken authzToken;
- public UnitLoad(String apiHost, int apiPort, String trustStorePath, String
trustStorePassword, StorageResourceManager storageResourceManager) {
+ public UnitLoad(String apiHost, int apiPort, String trustStorePath, String
trustStorePassword,
+ StorageResourceManager storageResourceManager, AuthzToken
authzToken) {
this.apiHost = apiHost;
this.apiPort = apiPort;
this.trustStorePath = trustStorePath;
this.trustStorePassword = trustStorePassword;
this.storageResourceManager = storageResourceManager;
+ this.authzToken = authzToken;
}
- public CompletionService<Boolean> execute(Configuration config) {
+ public CompletionService<List<String>> execute(Configuration config) {
String randomUUID = UUID.randomUUID().toString();
ExecutorService executorService =
Executors.newFixedThreadPool(config.getConcurrentUsers());
- CompletionService<Boolean> completionService = new
ExecutorCompletionService<>(executorService);
+ CompletionService<List<String>> completionService = new
ExecutorCompletionService<>(executorService);
for (int i = 0; i < config.getConcurrentUsers(); i++) {
completionService.submit(new Worker(config, randomUUID + "-" + i,
config.getIterationsPerUser(), config.getRandomMSDelayWithinSubmissions()));
@@ -48,7 +51,7 @@ public class UnitLoad {
return completionService;
}
- public class Worker implements Callable<Boolean> {
+ public class Worker implements Callable<List<String>> {
private final String id;
private final int iterations;
@@ -63,10 +66,11 @@ public class UnitLoad {
}
@Override
- public Boolean call() {
+ public List<String> call() {
+ List<String> experiments = new ArrayList<>();
for (int i = 0; i < iterations; i++) {
try {
- submitExperiment(config,id + "-" + i);
+ experiments.add(submitExperiment(config,id + "-" + i));
Thread.sleep(delay);
} catch (TException e) {
e.printStackTrace();
@@ -76,11 +80,11 @@ public class UnitLoad {
e.printStackTrace();
}
}
- return true;
+ return experiments;
}
}
- private void submitExperiment(Configuration config, String suffix) throws
TException, AgentException {
+ private String submitExperiment(Configuration config, String suffix)
throws TException, AgentException {
String experimentName = config.getExperimentBaseName() + suffix;
@@ -114,7 +118,7 @@ public class UnitLoad {
Airavata.Client airavataClient =
AiravataClientFactory.createAiravataSecureClient(apiHost, apiPort,
trustStorePath, trustStorePassword, 100000);
- List<InputDataObjectType> applicationInputs =
airavataClient.getApplicationInputs(new AuthzToken(""),
+ List<InputDataObjectType> applicationInputs =
airavataClient.getApplicationInputs(authzToken,
config.getApplicationInterfaceId());
List<InputDataObjectType> experimentInputs = new ArrayList<>();
@@ -138,13 +142,16 @@ public class UnitLoad {
}
experimentModel.setExperimentInputs(experimentInputs);
-
experimentModel.setExperimentOutputs(airavataClient.getApplicationOutputs(new
AuthzToken(""), config.getApplicationInterfaceId()));
+
experimentModel.setExperimentOutputs(airavataClient.getApplicationOutputs(authzToken,
config.getApplicationInterfaceId()));
experimentModel.setExperimentType(ExperimentType.SINGLE_APPLICATION);
- String experimentId = airavataClient.createExperiment(new
AuthzToken(""), config.getGatewayId(), experimentModel);
+ String experimentId = airavataClient.createExperiment(authzToken,
config.getGatewayId(), experimentModel);
- airavataClient.launchExperiment(new AuthzToken(""), experimentId,
config.getGatewayId());
+ airavataClient.launchExperiment(authzToken, experimentId,
config.getGatewayId());
System.out.println(experimentId);
+ ExperimentModel experiment = airavataClient.getExperiment(authzToken,
experimentId);
+ return experimentId;
+
}
}
diff --git a/tools/load-client/src/main/resources/conf/load-config.yml
b/tools/load-client/src/main/resources/conf/load-config.yml
index 16654a2..8ca0dd8 100644
--- a/tools/load-client/src/main/resources/conf/load-config.yml
+++ b/tools/load-client/src/main/resources/conf/load-config.yml
@@ -1,13 +1,13 @@
apiHost: 149.165.170.103
apiPort: 9930
configurations:
- - experimentBaseName: "LoadTest"
+ - experimentBaseName: "TestEcho"
userId: "dimuthu"
gatewayId: "seagrid"
- projectId: "DefaultProject_7ac38275-0ca1-433a-ab6a-630c8c1df2ef"
+ projectId: "Dimuthu-Perc18_7651cf55-187c-4dd1-96ed-6c611ef095aa"
applicationInterfaceId: "Echo_3f480d1f-ea86-4018-94bb-015423d66a1c"
- computeResourceId:
"js-156-93.jetstream-cloud.org_696c097d-a138-4445-b254-cd7e55c84fad"
- storageResourceId:
"testing.seagrid.org_20dbec4c-223e-4568-a267-63d7efc6267e"
+ computeResourceId:
"js-156-93.jetstream-cloud.org_dcdf7bfc-fdfa-4b83-ad8b-5f23e32b54ae"
+ storageResourceId:
"testing.seagrid.org_8a52ed94-b990-42d2-8714-f3d0abde1022"
inputs:
- name: "Input-to-Echo"
@@ -19,6 +19,6 @@ configurations:
nodeCount: 1
physicalMemory: 512
- concurrentUsers: 10
+ concurrentUsers: 1
iterationsPerUser: 2
randomMSDelayWithinSubmissions: 100
