Repository: airavata
Updated Branches:
  refs/heads/develop e04785a4c -> 8bc2dcc4e


Added template language to pre and post commands


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8bc2dcc4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8bc2dcc4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8bc2dcc4

Branch: refs/heads/develop
Commit: 8bc2dcc4eb3a45c09fa32903c38bd97e349b4599
Parents: e04785a
Author: Shameera Rathnayaka <shameerai...@gmail.com>
Authored: Mon Oct 17 17:15:06 2016 -0400
Committer: Shameera Rathnayaka <shameerai...@gmail.com>
Committed: Mon Oct 17 17:15:41 2016 -0400

----------------------------------------------------------------------
 .../apache/airavata/gfac/core/GFacUtils.java    | 48 ++++++++++++--------
 .../apache/airavata/gfac/core/GroovyMap.java    |  9 +++-
 .../org/apache/airavata/gfac/core/Script.java   |  4 ++
 .../gfac/core/context/ProcessContext.java       | 20 +++++---
 .../airavata/gfac/core/GFacUtilsTest.java       | 13 ++++++
 .../impl/task/DefaultJobSubmissionTask.java     |  2 +-
 .../gfac/impl/task/ForkJobSubmissionTask.java   |  2 +-
 .../gfac/impl/task/LocalJobSubmissionTask.java  |  2 +-
 8 files changed, 69 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index fe9fd01..b69764e 100644
--- 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -21,6 +21,7 @@ package org.apache.airavata.gfac.core;
 
 import groovy.lang.Writable;
 import groovy.text.GStringTemplateEngine;
+import groovy.text.SimpleTemplateEngine;
 import groovy.text.TemplateEngine;
 import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.*;
@@ -64,9 +65,6 @@ import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
-import javax.xml.transform.*;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
 import javax.xml.xpath.*;
 import java.io.*;
 import java.net.InetAddress;
@@ -416,8 +414,11 @@ public class GFacUtils {
     public static String getZKGfacServersParentPath() {
         return ZKPaths.makePath(ZkConstants.ZOOKEEPER_SERVERS_NODE, 
ZkConstants.ZOOKEEPER_GFAC_SERVER_NODE);
     }
-
-    public static GroovyMap creatGroovyMap(ProcessContext processContext, 
TaskContext taskContext)
+    public static GroovyMap crateGroovyMap(ProcessContext processContext)
+            throws ApplicationSettingsException, AppCatalogException, 
GFacException {
+        return createGroovyMap(processContext, null);
+    }
+    public static GroovyMap createGroovyMap(ProcessContext processContext, 
TaskContext taskContext)
             throws GFacException, AppCatalogException, 
ApplicationSettingsException {
 
         GroovyMap groovyMap = new GroovyMap();
@@ -430,6 +431,10 @@ public class GFacUtils {
         groovyMap.add(Script.EXECUTABLE_PATH, 
processContext.getApplicationDeploymentDescription().getExecutablePath());
         groovyMap.add(Script.STANDARD_OUT_FILE, 
processContext.getStdoutLocation());
         groovyMap.add(Script.STANDARD_ERROR_FILE, 
processContext.getStderrLocation());
+        groovyMap.add(Script.SCRATCH_LOCATION, 
processContext.getScratchLocation());
+        groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
+        groovyMap.add(Script.GATEWAY_USER_NAME, 
processContext.getProcessModel().getUserName());
+        groovyMap.add(Script.APPLICATION_NAME, 
processContext.getApplicationInterfaceDescription().getApplicationName());
 
         ComputeResourcePreference crp = 
getComputeResourcePreference(processContext);
         if (isValid(crp.getAllocationProjectNumber())) {
@@ -448,14 +453,16 @@ public class GFacUtils {
         groovyMap.add(Script.USER_NAME, 
processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
         groovyMap.add(Script.SHELL_NAME, "/bin/bash");
         // get walltime
-        try {
-            JobSubmissionTaskModel jobSubmissionTaskModel = 
((JobSubmissionTaskModel) taskContext.getSubTaskModel());
-            if (jobSubmissionTaskModel.getWallTime() > 0) {
-                groovyMap.add(Script.MAX_WALL_TIME,
-                        
GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+        if (taskContext != null) {
+            try {
+                JobSubmissionTaskModel jobSubmissionTaskModel = 
((JobSubmissionTaskModel) taskContext.getSubTaskModel());
+                if (jobSubmissionTaskModel.getWallTime() > 0) {
+                    groovyMap.add(Script.MAX_WALL_TIME,
+                            
GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+                }
+            } catch (TException e) {
+                log.error("Error while getting job submission sub task model", 
e);
             }
-        } catch (TException e) {
-            log.error("Error while getting job submissiont sub task model", e);
         }
 
         // NOTE: Give precedence to data comes with experiment
@@ -522,7 +529,7 @@ public class GFacUtils {
         if (preJobCommands != null) {
             List<String> preJobCmdCollect = preJobCommands.stream()
                     .sorted((e1, e2) -> e1.getCommandOrder() - 
e2.getCommandOrder())
-                    .map(map -> map.getCommand())
+                    .map(map -> parseCommands(map.getCommand(), groovyMap))
                     .collect(Collectors.toList());
             groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
         }
@@ -531,7 +538,7 @@ public class GFacUtils {
         if (postJobCommands != null) {
             List<String> postJobCmdCollect = postJobCommands.stream()
                     .sorted((e1, e2) -> e1.getCommandOrder() - 
e2.getCommandOrder())
-                    .map(map -> map.getCommand())
+                    .map(map -> parseCommands(map.getCommand(), groovyMap))
                     .collect(Collectors.toList());
             groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
         }
@@ -703,11 +710,14 @@ public class GFacUtils {
         return i;
     }
 
-    private static String parseCommand(String value, ProcessContext context) {
-        String parsedValue = value.replaceAll("\\$workingDir", 
context.getWorkingDir());
-        parsedValue = parsedValue.replaceAll("\\$inputDir", 
context.getInputDir());
-        parsedValue = parsedValue.replaceAll("\\$outputDir", 
context.getOutputDir());
-        return parsedValue;
+    static String parseCommands(String value, GroovyMap bindMap) {
+        TemplateEngine templateEngine = new GStringTemplateEngine();
+        try {
+            return 
templateEngine.createTemplate(value).make(bindMap).toString();
+        } catch (ClassNotFoundException | IOException e) {
+            throw new IllegalArgumentException("Error while parsing command " 
+ value
+                    + " , Invalid command or incomplete bind map");
+        }
     }
 
     public static ResourceJobManager getResourceJobManager(ProcessContext 
processContext) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
index d618d40..1abc878 100644
--- 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
+++ 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
@@ -45,7 +45,8 @@ public class GroovyMap extends HashMap<String, Object> {
 
     public GroovyMap() {
         super();
-        addDefaultValues(); // to mitigate groovy exception 
groovy.lang.MissingPropertyException: No such property: <name> for class: 
groovy.lang.Binding
+        // to mitigate groovy exception groovy.lang.MissingPropertyException: 
No such property: <name> for class: groovy.lang.Binding
+        addDefaultValues();
     }
 
     public GroovyMap add(Script name, Object value){
@@ -77,6 +78,7 @@ public class GroovyMap extends HashMap<String, Object> {
                 .add(Script.RESERVATION, null)
                 .add(Script.EXPORTS, null)
                 .add(Script.MODULE_COMMANDS, null)
+                .add(Script.SCRATCH_LOCATION, null)
                 .add(Script.WORKING_DIR, null)
                 .add(Script.PRE_JOB_COMMANDS, null)
                 .add(Script.JOB_SUBMITTER_COMMAND, null)
@@ -88,7 +90,10 @@ public class GroovyMap extends HashMap<String, Object> {
                 .add(Script.CHASSIS_NAME, null)
                 .add(Script.INPUT_DIR, null)
                 .add(Script.OUTPUT_DIR, null)
-                .add(Script.USER_NAME, null);
+                .add(Script.USER_NAME, null)
+                .add(Script.GATEWAY_ID, null)
+                .add(Script.GATEWAY_USER_NAME, null)
+                .add(Script.APPLICATION_NAME, null);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
index 1e3ecd9..3ac5797 100644
--- 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
+++ 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
@@ -54,6 +54,7 @@ public enum Script {
     RESERVATION("reservation"),
     EXPORTS("exports"),
     MODULE_COMMANDS("moduleCommands"),
+    SCRATCH_LOCATION("scratchLocation"),
     WORKING_DIR("workingDirectory"),
     PRE_JOB_COMMANDS("preJobCommands"),
     JOB_SUBMITTER_COMMAND("jobSubmitterCommand"),
@@ -66,6 +67,9 @@ public enum Script {
     INPUT_DIR("inputDir"),
     OUTPUT_DIR("outputDir"),
     USER_NAME("userName"),
+    GATEWAY_ID("gatewayId"),
+    GATEWAY_USER_NAME("gatewayUserName"),
+    APPLICATION_NAME("applicationName"),
     ;
 
     String name;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
index 8a2cc4e..8079e2a 100644
--- 
a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
+++ 
b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
@@ -64,6 +64,7 @@ public class ProcessContext {
        private final String tokenId;
        private ProcessModel processModel;
        private String workingDir;
+       private String scratchLocation;
        private String inputDir;
        private String outputDir;
        private String localWorkingDir;
@@ -166,13 +167,7 @@ public class ProcessContext {
             if 
(processModel.getProcessResourceSchedule().getStaticWorkingDir() != null){
                 workingDir = 
processModel.getProcessResourceSchedule().getStaticWorkingDir();
             }else {
-                String scratchLocation = null;
-                               String overrideScratchLocation = 
processModel.getProcessResourceSchedule().getOverrideScratchLocation();
-                if (overrideScratchLocation != null && 
!overrideScratchLocation.equals("")) {
-                                       scratchLocation = 
overrideScratchLocation;
-                               } else {
-                                       scratchLocation = 
computeResourcePreference.getScratchLocation();
-                               }
+                String scratchLocation = getScratchLocation();
                 workingDir = (scratchLocation.endsWith("/") ? scratchLocation 
+ processId : scratchLocation + "/" +
                         processId);
             }
@@ -180,6 +175,16 @@ public class ProcessContext {
                return workingDir;
        }
 
+       public String getScratchLocation() {
+               if (scratchLocation == null) {
+                       scratchLocation = 
processModel.getProcessResourceSchedule().getOverrideScratchLocation();
+                       if(scratchLocation == null || 
scratchLocation.isEmpty()){
+                               scratchLocation = 
computeResourcePreference.getScratchLocation();
+                       }
+               }
+               return scratchLocation;
+       }
+
        public void setWorkingDir(String workingDir) {
                this.workingDir = workingDir;
        }
@@ -524,5 +529,6 @@ public class ProcessContext {
                this.recoveryWithCancel = recoveryWithCancel;
        }
 
+
 }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
 
b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
index 3262f86..cf56780 100644
--- 
a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
+++ 
b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
@@ -46,4 +46,17 @@ public class GFacUtilsTest {
         String shared = GFacUtils.getQoS(qos, "compute");
         Assert.assertNull(shared);
     }
+
+    @Test
+    public void parserCommandTest() throws Exception {
+        String command = "mkdir -p 
$scratchLocation/$gatewayId/$gatewayUserName/$applicationName";
+        GroovyMap groovyMap = new GroovyMap();
+        groovyMap.add(Script.SCRATCH_LOCATION, "/my/scratch");
+        groovyMap.add(Script.GATEWAY_ID, "seagrid");
+        groovyMap.add(Script.GATEWAY_USER_NAME, "John");
+        groovyMap.add(Script.APPLICATION_NAME, "gaussian");
+        String value = GFacUtils.parseCommands(command, groovyMap);
+        Assert.assertNotNull(value);
+        Assert.assertEquals("mkdir -p /my/scratch/seagrid/John/gaussian", 
value);
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
index cabb77f..657de00 100644
--- 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
+++ 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
@@ -72,7 +72,7 @@ public class DefaultJobSubmissionTask implements 
JobSubmissionTask {
                    JobModel jobModel = processContext.getJobModel();
                    jobModel.setTaskId(taskContext.getTaskId());
                    RemoteCluster remoteCluster = 
processContext.getJobSubmissionRemoteCluster();
-                   GroovyMap groovyMap = 
GFacUtils.creatGroovyMap(processContext,taskContext);
+                       GroovyMap groovyMap = 
GFacUtils.createGroovyMap(processContext, taskContext);
                        
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
                        ResourceJobManager resourceJobManager = 
GFacUtils.getResourceJobManager(processContext);
                    JobManagerConfiguration jConfig = null;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
index e231cb9..1b43116 100644
--- 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
+++ 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
@@ -64,7 +64,7 @@ public class ForkJobSubmissionTask implements 
JobSubmissionTask {
             JobModel jobModel = processContext.getJobModel();
             jobModel.setTaskId(taskContext.getTaskId());
             RemoteCluster remoteCluster = 
processContext.getJobSubmissionRemoteCluster();
-            GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext, 
taskContext);
+            GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, 
taskContext);
             jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
             ResourceJobManager resourceJobManager = 
GFacUtils.getResourceJobManager(processContext);
             JobManagerConfiguration jConfig = null;

http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
----------------------------------------------------------------------
diff --git 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
index a8b8e39..7e989e4 100644
--- 
a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
+++ 
b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
@@ -68,7 +68,7 @@ public class LocalJobSubmissionTask implements 
JobSubmissionTask{
             jobModel.setTaskId(taskContext.getTaskId());
 
             RemoteCluster remoteCluster = 
processContext.getJobSubmissionRemoteCluster();
-            GroovyMap groovyMap = 
GFacUtils.creatGroovyMap(processContext,taskContext);
+            GroovyMap groovyMap = 
GFacUtils.createGroovyMap(processContext,taskContext);
 
             String jobId = AiravataUtils.getId("JOB_ID_");
             jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());

Reply via email to