Replaced xslt templates with groovy templates, fixed AIRAVATA-2124
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/c3064689 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/c3064689 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/c3064689 Branch: refs/heads/develop Commit: c3064689d04a1e7f348d48ae733c04b8c3756480 Parents: 67a9f64 Author: Shameera Rathnayaka <[email protected]> Authored: Mon Oct 3 17:15:37 2016 -0400 Committer: Shameera Rathnayaka <[email protected]> Committed: Mon Oct 3 17:24:28 2016 -0400 ---------------------------------------------------------------------- modules/commons/pom.xml | 10 - .../apache/airavata/common/utils/XMLUtil.java | 586 --- .../airavata/common/utils/XmlFormatter.java | 82 - .../airavata/common/utils/XMLUtilTest.java | 56 - .../src/main/resources/FORK_Groovy.template | 13 + .../src/main/resources/LSF_Groovy.template | 27 + .../src/main/resources/PBS_Groovy.template | 31 + .../src/main/resources/SLURM_Groovy.template | 28 + .../src/main/resources/UGE_Groovy.template | 28 + modules/distribution/pom.xml | 56 +- .../src/main/assembly/bin-assembly.xml | 15 +- modules/gfac/gfac-core/pom.xml | 60 +- .../apache/airavata/gfac/core/GFacUtils.java | 154 +- .../apache/airavata/gfac/core/GroovyMap.java | 94 + .../airavata/gfac/core/JobDescriptor.java | 489 -- .../org/apache/airavata/gfac/core/Script.java | 75 + .../gfac/core/cluster/OutputParser.java | 8 - .../gfac/core/x2012/x12/AfterAnyList.java | 185 - .../gfac/core/x2012/x12/AfterOKList.java | 185 - .../gfac/core/x2012/x12/ExportProperties.java | 202 - .../airavata/gfac/core/x2012/x12/InputList.java | 185 - .../core/x2012/x12/JobDescriptorDocument.java | 131 - .../gfac/core/x2012/x12/ModuleLoadCommands.java | 185 - .../airavata/gfac/core/x2012/x12/PbsParams.java | 1500 ------ .../gfac/core/x2012/x12/PostJobCommands.java | 185 - .../gfac/core/x2012/x12/PreJobCommands.java | 185 - .../core/x2012/x12/impl/AfterAnyListImpl.java | 254 - .../core/x2012/x12/impl/AfterOKListImpl.java | 254 - .../x2012/x12/impl/ExportPropertiesImpl.java | 252 - .../gfac/core/x2012/x12/impl/InputListImpl.java | 254 - .../x12/impl/JobDescriptorDocumentImpl.java | 96 - .../x2012/x12/impl/ModuleLoadCommandsImpl.java | 254 - .../gfac/core/x2012/x12/impl/PbsParamsImpl.java | 4381 ------------------ .../x2012/x12/impl/PostJobCommandsImpl.java | 254 - .../core/x2012/x12/impl/PreJobCommandsImpl.java | 254 - .../org/apache/airavata/gfac/impl/Factory.java | 11 +- .../gfac/impl/job/ForkOutputParser.java | 5 - .../airavata/gfac/impl/job/LSFOutputParser.java | 10 +- .../airavata/gfac/impl/job/PBSOutputParser.java | 90 +- .../gfac/impl/job/SlurmOutputParser.java | 61 +- .../airavata/gfac/impl/job/UGEOutputParser.java | 89 +- .../impl/task/DefaultJobSubmissionTask.java | 8 +- .../gfac/impl/task/ForkJobSubmissionTask.java | 6 +- .../engine/interpretor/WorkflowInterpreter.java | 1 - .../invoker/MsgBoxWsaResponsesCorrelator.java | 2 +- .../component/ws/WSComponentApplication.java | 1 - .../workflow/model/gpel/script/BPELScript.java | 2 - .../model/gpel/script/WorkflowWSDL.java | 1 - .../airavata/workflow/model/wf/Workflow.java | 1 - modules/workflow/workflow-core/pom.xml | 11 +- .../apache/airavata/workflow/core/XMLUtil.java | 587 +++ .../airavata/workflow/core/XmlFormatter.java | 82 + .../airavata/workflow/core/XMLUtilTest.java | 56 + pom.xml | 2 + 54 files changed, 1198 insertions(+), 10836 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/commons/pom.xml ---------------------------------------------------------------------- diff --git a/modules/commons/pom.xml b/modules/commons/pom.xml index f4231ce..b204e48 100644 --- a/modules/commons/pom.xml +++ b/modules/commons/pom.xml @@ -30,16 +30,6 @@ <version>${project.parent.version}</version> </dependency> <dependency> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - <version>2.9.1</version> - </dependency> - <dependency> - <groupId>org.apache.xmlbeans</groupId> - <artifactId>xmlbeans</artifactId> - <version>${xmlbeans.version}</version> - </dependency> - <dependency> <groupId>org.ogce</groupId> <artifactId>xpp3</artifactId> <version>${xpp3.version}</version> http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/commons/src/main/java/org/apache/airavata/common/utils/XMLUtil.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/XMLUtil.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/XMLUtil.java deleted file mode 100644 index 95ee1f3..0000000 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/XMLUtil.java +++ /dev/null @@ -1,586 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.common.utils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringReader; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import org.apache.airavata.common.exception.AiravataException; -import org.apache.xmlbeans.XmlError; -import org.apache.xmlbeans.XmlObject; -import org.apache.xmlbeans.XmlOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Text; -import org.xml.sax.SAXException; -import org.xmlpull.infoset.XmlDocument; -import org.xmlpull.infoset.XmlElement; -import org.xmlpull.infoset.XmlNamespace; -import org.xmlpull.mxp1.MXParserFactory; -import org.xmlpull.mxp1_serializer.MXSerializer; - -public class XMLUtil { - - /** - * The XML builder for XPP5 - */ - public static final org.xmlpull.infoset.XmlInfosetBuilder BUILDER = org.xmlpull.infoset.XmlInfosetBuilder - .newInstance(); - - /** - * The XML builder for XPP3. - */ - public static final org.xmlpull.v1.builder.XmlInfosetBuilder BUILDER3 = org.xmlpull.v1.builder.XmlInfosetBuilder - .newInstance(new MXParserFactory()); - - private static final Logger logger = LoggerFactory.getLogger(XMLUtil.class); - - private final static String PROPERTY_SERIALIZER_INDENTATION = "http://xmlpull.org/v1/doc/properties.html#serializer-indentation";; - - private final static String INDENT = " "; - - /** - * Parses a specified string and returns the XmlElement (XPP3). - * - * @param string - * @return The XmlElement (XPP3) parsed. - */ - public static org.xmlpull.v1.builder.XmlElement stringToXmlElement3(String string) { - return BUILDER3.parseFragmentFromReader(new StringReader(string)); - } - - /** - * Parses a specified string and returns the XmlElement (XPP5). - * - * @param string - * @return The XmlElement (XPP5) parsed. - */ - public static org.xmlpull.infoset.XmlElement stringToXmlElement(String string) { - XmlDocument document = BUILDER.parseString(string); - org.xmlpull.infoset.XmlElement element = document.getDocumentElement(); - return element; - } - - /** - * Converts a specified XmlElement (XPP3) to the XmlElement (XPP5). - * - * @param element - * @return The XmlElement (XPP5) converted. - */ - public static org.xmlpull.infoset.XmlElement xmlElement3ToXmlElement5(org.xmlpull.v1.builder.XmlElement element) { - String string = xmlElementToString(element); - return stringToXmlElement(string); - } - - /** - * Converts a specified XmlElement (XPP5) to the XmlElement (XPP3). - * - * @param element - * @return The XmlElement (XPP3) converted. - */ - public static org.xmlpull.v1.builder.XmlElement xmlElement5ToXmlElement3(org.xmlpull.infoset.XmlElement element) { - String string = xmlElementToString(element); - return stringToXmlElement3(string); - } - - /** - * Returns the XML string of a specified XmlElement. - * - * @param element - * The specified XmlElement - * @return The XML string - */ - public static String xmlElementToString(org.xmlpull.v1.builder.XmlElement element) { - MXSerializer serializer = new MXSerializer(); - StringWriter writer = new StringWriter(); - serializer.setOutput(writer); - serializer.setProperty(PROPERTY_SERIALIZER_INDENTATION, INDENT); - BUILDER3.serialize(element, serializer); - String xmlText = writer.toString(); - return xmlText; - } - - /** - * Returns the XML string as a specified XmlElement (XPP5). - * - * @param element - * The specified XmlElement - * @return The XML string - */ - public static String xmlElementToString(org.xmlpull.infoset.XmlElement element) { - String string; - if (element == null) { - string = ""; - } else { - string = BUILDER.serializeToStringPretty(element); - } - return string; - } - - /** - * Converts a specified XPP5 XML element to a DOM element under a specified document. - * - * @param xppElement - * @param document - * @return The converted DOM element. - */ - public static Element xppElementToDomElement(org.xmlpull.infoset.XmlElement xppElement, Document document) { - Element domElement = document.createElement(xppElement.getName()); - - for (org.xmlpull.infoset.XmlNamespace namespace : xppElement.namespaces()) { - logger.debug("namespace: " + namespace); - } - - for (org.xmlpull.infoset.XmlAttribute attribute : xppElement.attributes()) { - domElement.setAttribute(attribute.getName(), attribute.getValue()); - } - - for (Object object : xppElement.children()) { - if (object instanceof org.xmlpull.infoset.XmlElement) { - domElement.appendChild(xppElementToDomElement((org.xmlpull.infoset.XmlElement) object, document)); - } else if (object instanceof String) { - Text text = document.createTextNode((String) object); - domElement.appendChild(text); - } else { - logger.debug("object.getClass(): " + object.getClass()); - } - } - return domElement; - } - -// /** -// * @param definitions3 -// * @return The WsdlDefinitions (XSUL5) -// */ -// @Deprecated -// public static xsul5.wsdl.WsdlDefinitions wsdlDefinitions3ToWsdlDefintions5(xsul.wsdl.WsdlDefinitions definitions3) { -// return WSDLUtil.wsdlDefinitions3ToWsdlDefintions5(definitions3); -// } -// -// /** -// * @param definitions5 -// * @return The WsdlDefinitions (XSUL3) -// */ -// @Deprecated -// public static xsul.wsdl.WsdlDefinitions wsdlDefinitions5ToWsdlDefintions3(xsul5.wsdl.WsdlDefinitions definitions5) { -// return WSDLUtil.wsdlDefinitions5ToWsdlDefintions3(definitions5); -// } - - /** - * Converts a specified XPP3 XML element to a DOM element under a specified document. - * - * @param xppElement - * @param document - * @return The converted DOM element. - */ - public static Element xppElementToDomElement(org.xmlpull.v1.builder.XmlElement xppElement, Document document) { - Element domElement = document.createElement(xppElement.getName()); - - Iterator nsIt = xppElement.namespaces(); - while (nsIt.hasNext()) { - org.xmlpull.v1.builder.XmlNamespace namespace = (org.xmlpull.v1.builder.XmlNamespace) nsIt.next(); - logger.debug("namespace: " + namespace); - // TODO - } - - Iterator attrIt = xppElement.attributes(); - while (attrIt.hasNext()) { - org.xmlpull.v1.builder.XmlAttribute attribute = (org.xmlpull.v1.builder.XmlAttribute) attrIt.next(); - // TODO namespace - domElement.setAttribute(attribute.getName(), attribute.getValue()); - } - - Iterator elementIt = xppElement.children(); - while (elementIt.hasNext()) { - Object object = elementIt.next(); - if (object instanceof org.xmlpull.v1.builder.XmlElement) { - domElement.appendChild(xppElementToDomElement((org.xmlpull.v1.builder.XmlElement) object, document)); - } else if (object instanceof String) { - Text text = document.createTextNode((String) object); - domElement.appendChild(text); - } else { - logger.debug("object.getClass(): " + object.getClass()); - } - } - return domElement; - } - - /** - * @param element - * @return The cloned XmlElement. - */ - public static org.xmlpull.infoset.XmlElement deepClone(org.xmlpull.infoset.XmlElement element) - throws AiravataException { - try { - XmlElement clonedElement = element.clone(); - clonedElement.setParent(null); - return clonedElement; - } catch (CloneNotSupportedException e) { - // This should not happen because we don't put any special Objects. - throw new AiravataException(e.getMessage(), e); - } - } - - /** - * Saves a specified XmlElement to a specified file. - * - * @param element - * @param file - * @throws IOException - */ - public static void saveXML(org.xmlpull.infoset.XmlElement element, File file) throws IOException { - XmlDocument document = BUILDER.newDocument(); - document.setDocumentElement(element); - String xmlText = BUILDER.serializeToStringPretty(document); - IOUtil.writeToFile(xmlText, file); - } - - /** - * Saves a specified XmlElement to a specified file. - * - * @param element - * @param file - * @throws IOException - */ - public static void saveXML(org.xmlpull.v1.builder.XmlElement element, File file) throws IOException { - saveXML(xmlElement3ToXmlElement5(element), file); - } - - /** - * @param file - * @return The XmlElement in the document. - * @throws IOException - */ - public static org.xmlpull.infoset.XmlElement loadXML(InputStream stream) throws IOException { - String xmlText = IOUtil.readToString(stream); - XmlDocument document = BUILDER.parseString(xmlText); - return document.getDocumentElement(); - } - - /** - * @param file - * @return The XmlElement in the document. - * @throws IOException - */ - public static org.xmlpull.infoset.XmlElement loadXML(File file) throws IOException { - return loadXML(new FileInputStream(file)); - } - - /** - * @param string - * @return true if the specified string is XML, false otherwise - */ - public static boolean isXML(String string) { - try { - stringToXmlElement(string); - return true; - } catch (RuntimeException e) { - return false; - } - } - - /** - * Validates a specified XmlObject along with logging errors if any. - * - * @param xmlObject - */ - public static void validate(XmlObject xmlObject) throws AiravataException { - XmlOptions validateOptions = new XmlOptions(); - ArrayList errorList = new ArrayList(); - validateOptions.setErrorListener(errorList); - - boolean isValid = xmlObject.validate(validateOptions); - if (isValid) { - // Valid - return; - } - - // Error - StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < errorList.size(); i++) { - XmlError error = (XmlError) errorList.get(i); - logger.warn("Message: " + error.getMessage()); - logger.warn("Location of invalid XML: " + error.getCursorLocation().xmlText()); - stringBuilder.append("Message:" + error.getMessage()); - stringBuilder.append("Location of invalid XML: " + error.getCursorLocation().xmlText()); - } - throw new AiravataException(stringBuilder.toString()); - } - - /** - * Returns the local part of a specified QName. - * - * @param qname - * the specified QName in string, e.g. ns:value - * @return the local part of the QName, e.g. value - */ - public static String getLocalPartOfQName(String qname) { - int index = qname.indexOf(':'); - if (index < 0) { - return qname; - } else { - return qname.substring(index + 1); - } - } - - /** - * Returns the prefix of a specified QName. - * - * @param qname - * the specified QName in string, e.g. ns:value - * @return the prefix of the QName, e.g. ns - */ - public static String getPrefixOfQName(String qname) { - int index = qname.indexOf(':'); - if (index < 0) { - return null; - } else { - return qname.substring(0, index); - } - } - - /** - * @param prefixCandidate - * @param uri - * @param alwaysUseSuffix - * @param element - * @return The namespace found or declared. - */ - public static XmlNamespace declareNamespaceIfNecessary(String prefixCandidate, String uri, boolean alwaysUseSuffix, - XmlElement element) { - XmlNamespace namespace = element.lookupNamespaceByName(uri); - if (namespace == null) { - return declareNamespace(prefixCandidate, uri, alwaysUseSuffix, element); - } else { - return namespace; - } - } - - /** - * @param prefixCandidate - * @param uri - * @param alwaysUseSuffix - * @param element - * @return The namespace declared. - */ - public static XmlNamespace declareNamespace(String prefixCandidate, String uri, boolean alwaysUseSuffix, - XmlElement element) { - if (prefixCandidate == null || prefixCandidate.length() == 0) { - prefixCandidate = "a"; - } - String prefix = prefixCandidate; - if (alwaysUseSuffix) { - prefix += "0"; - } - if (element.lookupNamespaceByPrefix(prefix) != null) { - int i = 1; - prefix = prefixCandidate + i; - while (element.lookupNamespaceByPrefix(prefix) != null) { - i++; - } - } - XmlNamespace namespace = element.declareNamespace(prefix, uri); - return namespace; - } - - /** - * - * @param xml - * @param name - */ - public static void removeElements(XmlElement xml, String name) { - - Iterable<XmlElement> removeElements = xml.elements(null, name); - LinkedList<XmlElement> removeList = new LinkedList<XmlElement>(); - for (XmlElement xmlElement : removeElements) { - removeList.add(xmlElement); - } - for (XmlElement xmlElement : removeList) { - xml.removeChild(xmlElement); - } - Iterable children = xml.children(); - for (Object object : children) { - if (object instanceof XmlElement) { - XmlElement element = (XmlElement) object; - removeElements(element, name); - } - } - } - - /** - * @param url - * @return Document - */ - public static Document retrievalXMLDocFromUrl(String url) { - try { - URL xmlUrl = new URL(url); - InputStream in = xmlUrl.openStream(); - Document ret = null; - DocumentBuilderFactory domFactory; - DocumentBuilder builder; - - domFactory = DocumentBuilderFactory.newInstance(); - domFactory.setValidating(false); - domFactory.setNamespaceAware(false); - builder = domFactory.newDocumentBuilder(); - - ret = builder.parse(in); - - return ret; - } catch (MalformedURLException e) { - logger.error(e.getMessage(), e); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (ParserConfigurationException e) { - logger.error(e.getMessage(), e); - } catch (SAXException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - /** - * @param url - * @return Document - */ - public static Document retrievalXMLDocForParse(String url) { - try { - URL xmlUrl = new URL(url); - InputStream in = xmlUrl.openStream(); - DocumentBuilderFactory xmlFact = DocumentBuilderFactory.newInstance(); - xmlFact.setNamespaceAware(true); - DocumentBuilder builder = xmlFact.newDocumentBuilder(); - Document doc = builder.parse(in); - - return doc; - } catch (MalformedURLException e) { - logger.error("Malformed URL", e); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (ParserConfigurationException e) { - logger.error(e.getMessage(), e); - } catch (SAXException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - public static boolean isEqual(XmlElement elem1, XmlElement elem2) throws Exception { - - if (elem1 == null && elem2 == null) { - return true; - } else if (elem1 == null) { - return false; - } else if (elem2 == null) { - return false; - } - - if (!elem1.getName().equals(elem2.getName())) { - return false; - } else { - // now check if children are the same - Iterator children1 = elem1.children().iterator(); - Iterator children2 = elem2.children().iterator(); - - //check first ones for string - Object child1 = null; - Object child2 = null; - if (children1.hasNext() && children2.hasNext()) { - child1 = children1.next(); - child2 = children2.next(); - - if (!children1.hasNext() && !children2.hasNext()) { - //only one node could be string could be xmlelement - return compareObjs(child1, child2); - } else { - //get new iterators - - List<XmlElement> elemSet1 = getXmlElementsOnly(elem1.children().iterator()); - List<XmlElement> elemSet2 = getXmlElementsOnly(elem2.children().iterator()); - - if(elemSet1.size() != elemSet2.size()){ - return false; - } - for(int i =0; i< elemSet1.size(); ++i){ - if(!isEqual(elemSet1.get(i), elemSet2.get(i))){ - return false; - } - } - return true; - } - - - }else { - //no internal element - - return true; - } - } - - - } - - - - private static List<XmlElement> getXmlElementsOnly(Iterator itr){ - LinkedList<XmlElement> list = new LinkedList<XmlElement>(); - while(itr.hasNext()){ - Object obj = itr.next(); - if(obj instanceof XmlElement){ - list.add((XmlElement) obj); - } - } - return list; - } - - - - private static boolean compareObjs(Object child1, Object child2) throws Exception { - if (child1 instanceof String && child2 instanceof String) { - return child1.equals(child2); - - - } else if (child1 instanceof XmlElement && child2 instanceof XmlElement) { - return isEqual((XmlElement) child1, (XmlElement) child2); - } else { - return false; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/commons/src/main/java/org/apache/airavata/common/utils/XmlFormatter.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/XmlFormatter.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/XmlFormatter.java deleted file mode 100644 index 1440dbf..0000000 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/XmlFormatter.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.common.utils; - -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import org.apache.xml.serialize.OutputFormat; -import org.apache.xml.serialize.XMLSerializer; -import org.w3c.dom.Document; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -/** - * Pretty-prints xml, supplied as a string. - * <p/> - * eg. <code> - * String formattedXml = new XmlFormatter().format("<tag><nested>hello</nested></tag>"); - * </code> - */ -public class XmlFormatter { - - /** - * @param unformattedXml - * @return formattedXml - */ - public static String format(String unformattedXml) { - try { - final Document document = parseXmlFile(unformattedXml); - OutputFormat format = new OutputFormat(document); - format.setLineWidth(65); - format.setIndenting(true); - format.setIndent(2); - Writer out = new StringWriter(); - XMLSerializer serializer = new XMLSerializer(out, format); - serializer.serialize(document); - return out.toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static Document parseXmlFile(String in) { - try { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - InputSource is = new InputSource(new StringReader(in)); - return db.parse(is); - } catch (ParserConfigurationException e) { - throw new RuntimeException(e); - } catch (SAXException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/commons/src/test/java/org/apache/airavata/common/utils/XMLUtilTest.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/test/java/org/apache/airavata/common/utils/XMLUtilTest.java b/modules/commons/src/test/java/org/apache/airavata/common/utils/XMLUtilTest.java deleted file mode 100644 index 3c2c189..0000000 --- a/modules/commons/src/test/java/org/apache/airavata/common/utils/XMLUtilTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.common.utils; - -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class XMLUtilTest { - private final static Logger logger = LoggerFactory.getLogger(XMLUtilTest.class); - - @Test - public void isXMLTest(){ - String xml = "<test>testing</test>"; - org.junit.Assert.assertTrue(XMLUtil.isXML(xml)); - org.junit.Assert.assertFalse(XMLUtil.isXML("NonXMLString")); - } - - @Test - public void isEqualTest(){ - String xml1 = "<test><inner>innerValue</inner></test>"; - String xml2 = "<test><inner>innerValue</inner></test>"; - String xml3 = "<test1><inner>innerValue</inner></test1>"; - try { - org.junit.Assert.assertTrue(XMLUtil.isEqual(XMLUtil.stringToXmlElement(xml1), XMLUtil.stringToXmlElement(xml2))); - org.junit.Assert.assertFalse(XMLUtil.isEqual(XMLUtil.stringToXmlElement(xml1), XMLUtil.stringToXmlElement(xml3))); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - @Test - public void getQNameTest(){ - String qname = "ns1:a"; - org.junit.Assert.assertEquals("a",XMLUtil.getLocalPartOfQName(qname)); - org.junit.Assert.assertEquals("ns1",XMLUtil.getPrefixOfQName(qname)); - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/configuration/server/src/main/resources/FORK_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/FORK_Groovy.template b/modules/configuration/server/src/main/resources/FORK_Groovy.template new file mode 100644 index 0000000..d25037b --- /dev/null +++ b/modules/configuration/server/src/main/resources/FORK_Groovy.template @@ -0,0 +1,13 @@ +#!${shellName} +# FORK job submission script generated by Apache Airavata + +<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n' + if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n' + if (workingDirectory != null) out.print 'cd ' + workingDirectory +'\n' + if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n' + if (jobSubmitterCommand != null) out.print jobSubmitterCommand + ' ' + if (executablePath != null) out.print executablePath + ' ' + if (inputs != null) for(input in inputs) out.print input + ' ' + out.print '\n' + if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n' +%> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/configuration/server/src/main/resources/LSF_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/LSF_Groovy.template b/modules/configuration/server/src/main/resources/LSF_Groovy.template new file mode 100644 index 0000000..428b42b --- /dev/null +++ b/modules/configuration/server/src/main/resources/LSF_Groovy.template @@ -0,0 +1,27 @@ +#!${shellName} +# LSF batch job submission script generated by Apache Airavata + +<% + if(shellName != null && shellName != "") out.print '#BSUB -L ' + shellName + '\n' + if (queueName != null && queueName != "") out.print '#BSUB -q ' + queueName + '\n' + if (nodes != null && nodes != "") out.print '#BSUB -n ' + nodes + '\n' + if (jobName != null && jobName != "") out.print '#BSUB -J ' + jobName + '\n' + if (mailAddress != null && mailAddress != "") out.print '#BSUB -u ' + mailAddress + '\n' + if (accountString != null && accountString != "") out.print '#BSUB -P ' + accountString + '\n' + if (maxWallTime != null && maxWallTime != "") out.print '#BSUB -W ' + maxWallTime + '\n' + if (standardOutFile != null && standardOutFile != "") out.print '#BSUB -o ' + standardOutFile + '\n' + if (standardErrorFile != null && standardErrorFile != "") out.print '#BSUB -e ' + standardErrorFile + '\n' + if (chassisName != null && chassisName != "") out.print '#BSUB -m c' + chassisName + '\n' + if (usedMem != null && usedMem != "") out.print '#BSUB -R rusage[mem=' + usedMem + ']\n' +%> + +<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n' + if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n' + if (workingDirectory != null && workingDirectory != "") out.print 'cd ' + workingDirectory +'\n' + if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n' + if (jobSubmitterCommand != null && jobSubmitterCommand != "") out.print jobSubmitterCommand + ' ' + if (executablePath != null && executablePath != "") out.print executablePath + ' ' + if (inputs != null) for(input in inputs) out.print input + ' ' + out.print '\n' + if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n' +%> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/configuration/server/src/main/resources/PBS_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/PBS_Groovy.template b/modules/configuration/server/src/main/resources/PBS_Groovy.template new file mode 100644 index 0000000..4056910 --- /dev/null +++ b/modules/configuration/server/src/main/resources/PBS_Groovy.template @@ -0,0 +1,31 @@ +#!${shellName} + +#PBS batch job submission script generated by Apache Airavata +<% + if(shellName != null && shellName != "") out.print '#PBS -S ' + shellName + '\n' + if (queueName != null && queueName != "") out.print '#PBS -q ' + queueName + '\n' + if (jobName != null && jobName != "") out.print '#PBS -N ' + jobName + '\n' + if (mailAddress != null && mailAddress != "") out.print '#PBS -M ' + mailAddress + '\n' + if (accountString != null && accountString != "") out.print '#PBS -A ' + accountString + '\n' + if (maxWallTime != null && maxWallTime != "") out.print '#PBS -l walltime=' + maxWallTime + '\n' + if (jobSubmitterCommand != null && jobSubmitterCommand != "" && jobSubmitterCommand == "ccmrun") + out.print '#PBS -l gres=ccm \n' + if (standardOutFile != null && standardOutFile != "") out.print '#PBS -o ' + standardOutFile + '\n' + if (standardErrorFile != null && standardErrorFile != "") out.print '#PBS -e ' + standardErrorFile + '\n' + if (usedMem != null && usedMem != "") out.print '#PBS -l mem=' + usedMem + '\n' + if (nodes != null && nodes != "" && processPerNode != null && processPerNode != "") + out.print '#PBS -l nodes=' + nodes + ':ppn=' + processPerNode + '\n' +%> +#PBS -m abe + +<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n' + if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n' + if (workingDirectory != null && workingDirectory != "") out.print 'cd ' + workingDirectory +'\n' + if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n' + if (jobSubmitterCommand != null && jobSubmitterCommand != "") out.print jobSubmitterCommand + ' ' + if (jobSubmitterCommand != "ccmrun" && jobSubmitterCommand != "") out.print cpuCount + ' ' + if (executablePath != null && executablePath != "") out.print executablePath + ' ' + if (inputs != null) for(input in inputs) out.print input + ' ' + out.print '\n' + if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n' +%> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/configuration/server/src/main/resources/SLURM_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/SLURM_Groovy.template b/modules/configuration/server/src/main/resources/SLURM_Groovy.template new file mode 100644 index 0000000..bae646e --- /dev/null +++ b/modules/configuration/server/src/main/resources/SLURM_Groovy.template @@ -0,0 +1,28 @@ +#!${shellName} + +# SLURM job submission script generated by Apache Airavata +<% +if (queueName != null && queueName != "") out.print '#SBATCH -p ' + queueName + '\n' + if (nodes != null && nodes != "") out.print '#SBATCH -N ' + nodes + '\n' + if (cpuCount != null && cpuCount != "") out.print '#SBATCH -n ' + cpuCount + '\n' + if (mailAddress != null && mailAddress != "") out.print '#SBATCH --mail-user=' + mailAddress + '\n' + if (accountString != null && accountString != "" ) out.print '#SBATCH -A ' + accountString + '\n' + if (maxWallTime != null && maxWallTime != "") out.print '#SBATCH -t ' + maxWallTime + '\n' + if (jobName != null && jobName != "") out.print '#SBATCH -J ' + jobName + '\n' + if (standardOutFile != null && standardOutFile != "") out.print '#SBATCH -o ' + standardOutFile + '\n' + if (standardErrorFile != null && standardErrorFile != "") out.print '#SBATCH -e ' + standardErrorFile + '\n' + if (qualityOfService != null && qualityOfService != "") out.print '#SBATCH --qos=' + qualityOfService + '\n' + if (reservation != null && reservation != "") out.print '#SBATCH --reservation=' + reservation + '\n' +%> +#SBATCH --mail-type=ALL + +<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n' + if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n' + if (workingDirectory != null && workingDirectory != "") out.print 'cd ' + workingDirectory +'\n' + if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n' + if (jobSubmitterCommand != null && jobSubmitterCommand != "") out.print jobSubmitterCommand + ' ' + if (executablePath != null && executablePath != "") out.print executablePath + ' ' + if (inputs != null) for(input in inputs) out.print input + ' ' + out.print '\n' + if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n' +%> http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/configuration/server/src/main/resources/UGE_Groovy.template ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/UGE_Groovy.template b/modules/configuration/server/src/main/resources/UGE_Groovy.template new file mode 100644 index 0000000..df454dd --- /dev/null +++ b/modules/configuration/server/src/main/resources/UGE_Groovy.template @@ -0,0 +1,28 @@ +#!${shellName} +# Grid Engine batch job script built by Apache Airavata + +<% + if(shellName != null && shellName != "") out.print '#$ -S ' + shellName + '\n' + if (queueName != null && queueName != "") out.print '#$ -q ' + queueName + '\n' + if (jobName != null && jobName != "") out.print '#$ -N ' + jobName + '\n' + if (mailAddress != null && mailAddress != "") out.print '#$ -M ' + mailAddress + '\n' + if (accountString != null && accountString != "") out.print '#$ -A ' + accountString + '\n' + if (maxWallTime != null && maxWallTime != "") out.print '#$ -l h_rt=' + maxWallTime + '\n' + if (standardOutFile != null && standardOutFile != "") out.print '#$ -o ' + standardOutFile + '\n' + if (standardErrorFile != null && standardErrorFile != "") out.print '#$ -e ' + standardErrorFile + '\n' + if (nodes != null && nodes != "" && processPerNode != null && processPerNode != "") + out.print '#$ -pe ' + processPerNode + 'way' + (12 * nodes) +%> +#\$ -V +#\$ -m beas + +<% if (exports != null) for(com in exports) out.print 'export ' + com +'\n' + if (moduleCommands != null) for(mc in moduleCommands) out.print mc +'\n' + if (workingDirectory != null && workingDirectory != "") out.print 'cd ' + workingDirectory +'\n' + if (preJobCommands != null) for(pjc in preJobCommands) out.print pjc +'\n' + if (jobSubmitterCommand != null && jobSubmitterCommand != "") out.print jobSubmitterCommand + ' ' + if (executablePath != null && executablePath != "") out.print executablePath + ' ' + if (inputs != null) for(input in inputs) out.print input + ' ' + out.print '\n' + if (postJobCommands != null) for(pjc in postJobCommands) out.print pjc +'\n' +%> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/modules/distribution/pom.xml b/modules/distribution/pom.xml index d096739..1199647 100644 --- a/modules/distribution/pom.xml +++ b/modules/distribution/pom.xml @@ -55,30 +55,30 @@ </executions> </plugin> - <plugin> - <groupId>org.codehaus.gmaven</groupId> - <artifactId>gmaven-plugin</artifactId> - <version>1.4</version> - <executions> - <execution> - <id>generate-timestamp</id> - <phase>package</phase> - <goals> - <goal>execute</goal> - </goals> - <configuration> - <source> - import java.util.Date - import java.text.MessageFormat + <!--<plugin>--> + <!--<groupId>org.codehaus.gmaven</groupId>--> + <!--<artifactId>gmaven-plugin</artifactId>--> + <!--<version>1.4</version>--> + <!--<executions>--> + <!--<execution>--> + <!--<id>generate-timestamp</id>--> + <!--<phase>package</phase>--> + <!--<goals>--> + <!--<goal>execute</goal>--> + <!--</goals>--> + <!--<configuration>--> + <!--<source>--> + <!--import java.util.Date--> + <!--import java.text.MessageFormat--> - project.properties['buildTimestamp'] = - MessageFormat.format("{0,date,dd-MM-yyyy}", new - Date()) - </source> - </configuration> - </execution> - </executions> - </plugin> + <!--project.properties['buildTimestamp'] =--> + <!--MessageFormat.format("{0,date,dd-MM-yyyy}", new--> + <!--Date())--> + <!--</source>--> + <!--</configuration>--> + <!--</execution>--> + <!--</executions>--> + <!--</plugin>--> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> @@ -561,6 +561,16 @@ <artifactId>logback-classic</artifactId> <version>${logback.version}</version> </dependency> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy</artifactId> + <version>${groovy.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-templates</artifactId> + <version>${groovy.version}</version> + </dependency> <!-- ======================== Sample =================== --> <dependency> <groupId>org.apache.airavata</groupId> http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/distribution/src/main/assembly/bin-assembly.xml ---------------------------------------------------------------------- diff --git a/modules/distribution/src/main/assembly/bin-assembly.xml b/modules/distribution/src/main/assembly/bin-assembly.xml index d88521d..79778e7 100644 --- a/modules/distribution/src/main/assembly/bin-assembly.xml +++ b/modules/distribution/src/main/assembly/bin-assembly.xml @@ -98,11 +98,16 @@ <include>persistence.xml</include> <include>provenance.sql</include> <include>gfac-config.yaml</include> - <include>PBSTemplate.xslt</include> - <include>SLURMTemplate.xslt</include> - <include>LSFTemplate.xslt</include> - <include>UGETemplate.xslt</include> - <include>ForkTemplate.xslt</include> + <!--<include>PBSTemplate.xslt</include>--> + <!--<include>SLURMTemplate.xslt</include>--> + <!--<include>LSFTemplate.xslt</include>--> + <!--<include>UGETemplate.xslt</include>--> + <!--<include>ForkTemplate.xslt</include>--> + <include>PBS_Groovy.template</include> + <include>SLURM_Groovy.template</include> + <include>LSF_Groovy.template</include> + <include>UGE_Groovy.template</include> + <include>FORK_Groovy.template</include> <include>gsissh.properties</include> <include>airavata.jks</include> <include>client_truststore.jks</include> http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/gfac/gfac-core/pom.xml ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/pom.xml b/modules/gfac/gfac-core/pom.xml index 8d358ff..de3703a 100644 --- a/modules/gfac/gfac-core/pom.xml +++ b/modules/gfac/gfac-core/pom.xml @@ -52,10 +52,20 @@ <artifactId>airavata-registry-core</artifactId> <version>${project.version}</version> </dependency> + <!--<dependency>--> + <!--<groupId>org.apache.xmlbeans</groupId>--> + <!--<artifactId>xmlbeans</artifactId>--> + <!--<version>${xmlbeans.version}</version>--> + <!--</dependency>--> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy</artifactId> + <version>${groovy.version}</version> + </dependency> <dependency> - <groupId>org.apache.xmlbeans</groupId> - <artifactId>xmlbeans</artifactId> - <version>${xmlbeans.version}</version> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-templates</artifactId> + <version>${groovy.version}</version> </dependency> <!-- Credential Store --> <dependency> @@ -132,28 +142,28 @@ <build> <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>xmlbeans-maven-plugin</artifactId> - <version>2.3.3</version> - <executions> - <execution> - <phase>generate-sources</phase> - <goals> - <goal>xmlbeans</goal> - </goals> - </execution> - </executions> - <inherited>true</inherited> - <configuration> - <schemaDirectory>src/main/resources</schemaDirectory> - <xmlConfigs> - <xmlConfig implementation="java.io.File">src/main/resources/gsissh-schemas.xsdconfig</xmlConfig> - </xmlConfigs> - <sourceGenerationDirectory>src/main/java</sourceGenerationDirectory> - <outputJar>target/generated/${project.artifactId}-${project.version}.jar</outputJar> - </configuration> - </plugin> + <!--<plugin>--> + <!--<groupId>org.codehaus.mojo</groupId>--> + <!--<artifactId>xmlbeans-maven-plugin</artifactId>--> + <!--<version>2.3.3</version>--> + <!--<executions>--> + <!--<execution>--> + <!--<phase>generate-sources</phase>--> + <!--<goals>--> + <!--<goal>xmlbeans</goal>--> + <!--</goals>--> + <!--</execution>--> + <!--</executions>--> + <!--<inherited>true</inherited>--> + <!--<configuration>--> + <!--<schemaDirectory>src/main/resources</schemaDirectory>--> + <!--<xmlConfigs>--> + <!--<xmlConfig implementation="java.io.File">src/main/resources/gsissh-schemas.xsdconfig</xmlConfig>--> + <!--</xmlConfigs>--> + <!--<sourceGenerationDirectory>src/main/java</sourceGenerationDirectory>--> + <!--<outputJar>target/generated/${project.artifactId}-${project.version}.jar</outputJar>--> + <!--</configuration>--> + <!--</plugin>--> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java index 7e2154e..18c3515 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java @@ -19,6 +19,9 @@ */ package org.apache.airavata.gfac.core; +import groovy.lang.Writable; +import groovy.text.GStringTemplateEngine; +import groovy.text.TemplateEngine; import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.common.utils.*; import org.apache.airavata.credential.store.store.CredentialReader; @@ -75,6 +78,7 @@ import java.security.SecureRandom; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; //import org.apache.airavata.commons.gfac.type.ActualParameter; @@ -413,42 +417,42 @@ public class GFacUtils { return ZKPaths.makePath(ZkConstants.ZOOKEEPER_SERVERS_NODE, ZkConstants.ZOOKEEPER_GFAC_SERVER_NODE); } - public static JobDescriptor createJobDescriptor(ProcessContext processContext, TaskContext taskContext) + public static GroovyMap creatGroovyMap(ProcessContext processContext, TaskContext taskContext) throws GFacException, AppCatalogException, ApplicationSettingsException { - JobDescriptor jobDescriptor = new JobDescriptor(); + GroovyMap groovyMap = new GroovyMap(); ProcessModel processModel = processContext.getProcessModel(); ResourceJobManager resourceJobManager = getResourceJobManager(processContext); - setMailAddresses(processContext, jobDescriptor); // set email options and addresses + setMailAddresses(processContext, groovyMap); // set email options and addresses + + groovyMap.add(Script.INPUT_DIR, processContext.getInputDir()); + groovyMap.add(Script.OUTPUT_DIR, processContext.getOutputDir()); + groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath()); + groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation()); + groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation()); - jobDescriptor.setInputDirectory(processContext.getInputDir()); - jobDescriptor.setOutputDirectory(processContext.getOutputDir()); - jobDescriptor.setExecutablePath(processContext.getApplicationDeploymentDescription().getExecutablePath()); - jobDescriptor.setStandardOutFile(processContext.getStdoutLocation()); - jobDescriptor.setStandardErrorFile(processContext.getStderrLocation()); ComputeResourcePreference crp = getComputeResourcePreference(processContext); if (crp.getAllocationProjectNumber() != null) { - jobDescriptor.setAcountString(crp.getAllocationProjectNumber()); + groovyMap.add(Script.ACCOUNT_STRING, crp.getAllocationProjectNumber()); } - jobDescriptor.setReservation(getReservation(crp)); + groovyMap.add(Script.RESERVATION, getReservation(crp)); // To make job name alpha numeric - jobDescriptor.setJobName("A" + String.valueOf(generateJobName())); - jobDescriptor.setWorkingDirectory(processContext.getWorkingDir()); + groovyMap.add(Script.JOB_NAME, "A" + String.valueOf(generateJobName())); + groovyMap.add(Script.WORKING_DIR, processContext.getWorkingDir()); List<String> inputValues = getProcessInputValues(processModel.getProcessInputs()); inputValues.addAll(getProcessOutputValues(processModel.getProcessOutputs())); - jobDescriptor.setInputValues(inputValues); + groovyMap.add(Script.INPUTS, inputValues); - jobDescriptor.setUserName(processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName()); - jobDescriptor.setShellName("/bin/bash"); - jobDescriptor.setAllEnvExport(true); - jobDescriptor.setOwner(processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName()); + groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName()); + groovyMap.add(Script.SHELL_NAME, "/bin/bash"); // get walltime try { JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel()); if (jobSubmissionTaskModel.getWallTime() > 0) { - jobDescriptor.setMaxWallTime(jobSubmissionTaskModel.getWallTime() + ""); + groovyMap.add(Script.MAX_WALL_TIME, + GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime())); } } catch (TException e) { log.error("Error while getting job submissiont sub task model", e); @@ -460,33 +464,35 @@ public class GFacUtils { int totalCPUCount = scheduling.getTotalCPUCount(); if (scheduling.getQueueName() != null) { - jobDescriptor.setQueueName(scheduling.getQueueName()); + groovyMap.add(Script.QUEUE_NAME, scheduling.getQueueName()); } if (totalNodeCount > 0) { - jobDescriptor.setNodes(totalNodeCount); + groovyMap.add(Script.NODES, totalNodeCount); } // qos per queue String qoS = getQoS(crp.getQualityOfService(), scheduling.getQueueName()); if (qoS != null) { - jobDescriptor.setQoS(qoS); + groovyMap.add(Script.QUALITY_OF_SERVICE, qoS); } if (totalCPUCount > 0) { int ppn = totalCPUCount / totalNodeCount; - jobDescriptor.setProcessesPerNode(ppn); - jobDescriptor.setCPUCount(totalCPUCount); + groovyMap.add(Script.PROCESS_PER_NODE, ppn); + groovyMap.add(Script.CPU_COUNT, totalCPUCount); } // max wall time may be set before this level if jobsubmission task has wall time configured to this job, // if so we ignore scheduling configuration. - if (scheduling.getWallTimeLimit() > 0 && jobDescriptor.getMaxWallTime() == null) { - jobDescriptor.setMaxWallTime(String.valueOf(scheduling.getWallTimeLimit())); + if (scheduling.getWallTimeLimit() > 0 && groovyMap.get(Script.MAX_WALL_TIME) == null) { + groovyMap.add(Script.MAX_WALL_TIME, + GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit())); if (resourceJobManager != null) { if (resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)) { - jobDescriptor.setMaxWallTimeForLSF(String.valueOf(scheduling.getWallTimeLimit())); + groovyMap.add(Script.MAX_WALL_TIME, + GFacUtils.maxWallTimeCalculator(scheduling.getWallTimeLimit())); } } } if (scheduling.getTotalPhysicalMemory() > 0) { - jobDescriptor.setUsedMemory(scheduling.getTotalPhysicalMemory() + ""); + groovyMap.add(Script.USED_MEM, scheduling.getTotalPhysicalMemory() + ""); } } else { log.error("Task scheduling cannot be null at this point.."); @@ -495,28 +501,29 @@ public class GFacUtils { ApplicationDeploymentDescription appDepDescription = processContext.getApplicationDeploymentDescription(); List<CommandObject> moduleCmds = appDepDescription.getModuleLoadCmds(); if (moduleCmds != null) { - Collections.sort(moduleCmds, - (o1, o2) -> ((CommandObject) o1).getCommandOrder() - ((CommandObject) o2).getCommandOrder()); - for (CommandObject moduleCmd : moduleCmds) { - jobDescriptor.addModuleLoadCommands(moduleCmd.getCommand()); - } + List<String> modulesCmdCollect = moduleCmds.stream() + .sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()) + .map(map -> map.getCommand()) + .collect(Collectors.toList()); + groovyMap.add(Script.MODULE_COMMANDS, modulesCmdCollect); } + List<CommandObject> preJobCommands = appDepDescription.getPreJobCommands(); if (preJobCommands != null) { - Collections.sort(preJobCommands, - (o1, o2) -> ((CommandObject) o1).getCommandOrder() - ((CommandObject) o2).getCommandOrder()); - for (CommandObject preJobCommand : preJobCommands) { - jobDescriptor.addPreJobCommand(parseCommand(preJobCommand.getCommand(), processContext)); - } + List<String> preJobCmdCollect = preJobCommands.stream() + .sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()) + .map(map -> map.getCommand()) + .collect(Collectors.toList()); + groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect); } List<CommandObject> postJobCommands = appDepDescription.getPostJobCommands(); if (postJobCommands != null) { - Collections.sort(postJobCommands, - (o1, o2) -> ((CommandObject) o1).getCommandOrder() - ((CommandObject) o2).getCommandOrder()); - for (CommandObject postJobCommand : postJobCommands) { - jobDescriptor.addPostJobCommand(parseCommand(postJobCommand.getCommand(), processContext)); - } + List<String> postJobCmdCollect = postJobCommands.stream() + .sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder()) + .map(map -> map.getCommand()) + .collect(Collectors.toList()); + groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect); } ApplicationParallelismType parallelism = appDepDescription.getParallelism(); @@ -526,17 +533,17 @@ public class GFacUtils { if (parallelismPrefix != null){ String parallelismCommand = parallelismPrefix.get(parallelism); if (parallelismCommand != null){ - jobDescriptor.setJobSubmitter(parallelismCommand); + groovyMap.add(Script.JOB_SUBMITTER_COMMAND, parallelismCommand); }else { throw new GFacException("Parallelism prefix is not defined for given parallelism type " + parallelism + ".. Please define the parallelism prefix at App Catalog"); } } } } - return jobDescriptor; + return groovyMap; } - private static void setMailAddresses(ProcessContext processContext, JobDescriptor jobDescriptor) + private static void setMailAddresses(ProcessContext processContext, GroovyMap groovyMap) throws GFacException, AppCatalogException, ApplicationSettingsException { ProcessModel processModel = processContext.getProcessModel(); @@ -545,12 +552,6 @@ public class GFacUtils { emailIds = ServerSettings.getEmailBasedMonitorAddress(); } if (ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_ENABLE).equalsIgnoreCase("true")) { - String flags = ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_FLAGS); - if (flags != null && processContext.getComputeResourceDescription().getHostName().equals("stampede.tacc.xsede.org")) { - flags = "ALL"; - } - jobDescriptor.setMailOptions(flags); - String userJobNotifEmailIds = ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_EMAILIDS); if (userJobNotifEmailIds != null && !userJobNotifEmailIds.isEmpty()) { if (emailIds != null && !emailIds.isEmpty()) { @@ -573,7 +574,7 @@ public class GFacUtils { } if (emailIds != null && !emailIds.isEmpty()) { log.info("Email list: " + emailIds); - jobDescriptor.setMailAddress(emailIds); + groovyMap.add(Script.MAIL_ADDRESS, emailIds); } } @@ -806,46 +807,31 @@ public class GFacUtils { } } - public static File createJobFile(TaskContext taskContext, JobDescriptor jobDescriptor, JobManagerConfiguration jobManagerConfiguration) throws GFacException { - try { - TransformerFactory factory = TransformerFactory.newInstance(); - URL resource = ApplicationSettings.loadFile(jobManagerConfiguration.getJobDescriptionTemplateName()); + public static File createJobFile(GroovyMap groovyMap, TaskContext tc, JobManagerConfiguration jMC) + throws GFacException{ - if (resource == null) { - String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName() - + "' not found in the classpath"; - throw new GFacException(error); - } + URL templateUrl = ApplicationSettings.loadFile(jMC.getJobDescriptionTemplateName()); + if (templateUrl == null) { + String error = "System configuration file '" + jMC.getJobDescriptionTemplateName() + + "' not found in the classpath"; + throw new GFacException(error); + } + try { + File template = new File(templateUrl.getPath()); + TemplateEngine engine = new GStringTemplateEngine(); + Writable make = engine.createTemplate(template).make(groovyMap); - Source xslt = new StreamSource(new File(resource.getPath())); - Transformer transformer; - StringWriter results = new StringWriter(); - File tempJobFile = null; - // generate the pbs script using xslt - transformer = factory.newTransformer(xslt); - Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes())); - transformer.transform(text, new StreamResult(results)); - String scriptContent = results.toString().replaceAll("^[ |\t]*\n$", ""); - if (scriptContent.startsWith("\n")) { - scriptContent = scriptContent.substring(1); - } - // creating a temporary file using pbs script generated above int number = new SecureRandom().nextInt(); number = (number < 0 ? -number : number); - - tempJobFile = new File(GFacUtils.getLocalDataDir(taskContext), "job_" + Integer.toString(number) + - jobManagerConfiguration.getScriptExtension()); - FileUtils.writeStringToFile(tempJobFile, scriptContent); + File tempJobFile = new File(GFacUtils.getLocalDataDir(tc), "job_" + Integer.toString(number) + jMC.getScriptExtension()); + FileUtils.writeStringToFile(tempJobFile, make.toString()); return tempJobFile; - } catch (IOException e) { - throw new GFacException("Error occurred while creating the temp job script file", e); - } catch (TransformerConfigurationException e) { - throw new GFacException("Error occurred while creating the temp job script file", e); - } catch (TransformerException e) { - throw new GFacException("Error occurred while creating the temp job script file", e); + } catch (ClassNotFoundException | IOException e) { + throw new GFacException("Error while parsing template and generating script file"); } } + public static File getLocalDataDir(TaskContext taskContext) { String outputPath = ServerSettings.getLocalDataLocation(); outputPath = (outputPath.endsWith(File.separator) ? outputPath : outputPath + File.separator); http://git-wip-us.apache.org/repos/asf/airavata/blob/c3064689/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java new file mode 100644 index 0000000..d618d40 --- /dev/null +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java @@ -0,0 +1,94 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.airavata.gfac.core;/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +import java.util.HashMap; + +public class GroovyMap extends HashMap<String, Object> { + + + public GroovyMap() { + super(); + addDefaultValues(); // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding + } + + public GroovyMap add(Script name, Object value){ + put(name.name, value); + return this; + } + + @Override + public Object get(Object key) { + return super.getOrDefault(key, null); + } + + public Object get(Script script) { + return get(script.name); + } + + private void addDefaultValues() { + this.add(Script.SHELL_NAME, null) + .add(Script.QUEUE_NAME, null) + .add(Script.NODES, null) + .add(Script.CPU_COUNT, null) + .add(Script.MAIL_ADDRESS, null) + .add(Script.ACCOUNT_STRING, null) + .add(Script.MAX_WALL_TIME, null) + .add(Script.JOB_NAME, null) + .add(Script.STANDARD_OUT_FILE, null) + .add(Script.STANDARD_ERROR_FILE, null) + .add(Script.QUALITY_OF_SERVICE, null) + .add(Script.RESERVATION, null) + .add(Script.EXPORTS, null) + .add(Script.MODULE_COMMANDS, null) + .add(Script.WORKING_DIR, null) + .add(Script.PRE_JOB_COMMANDS, null) + .add(Script.JOB_SUBMITTER_COMMAND, null) + .add(Script.EXECUTABLE_PATH, null) + .add(Script.INPUTS, null) + .add(Script.POST_JOB_COMMANDS, null) + .add(Script.USED_MEM, null) + .add(Script.PROCESS_PER_NODE, null) + .add(Script.CHASSIS_NAME, null) + .add(Script.INPUT_DIR, null) + .add(Script.OUTPUT_DIR, null) + .add(Script.USER_NAME, null); + } + +}
