Repository: incubator-systemml Updated Branches: refs/heads/master 20d3efa0f -> e81b9059e
[SYSTEMML-1354] Move minimum recommended Spark version to pom.xml Move minimum recommended Spark version to pom.xml to simplify project maintenance. For SystemML jar, put value in manifest and read from there. If no jar file present (development environment), read value from pom.xml Closes #408. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/e81b9059 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/e81b9059 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/e81b9059 Branch: refs/heads/master Commit: e81b9059ec56bc69cdf9c2875dca7ad1f56e4e5a Parents: 20d3efa Author: Deron Eriksson <[email protected]> Authored: Mon Mar 13 17:17:14 2017 -0700 Committer: Deron Eriksson <[email protected]> Committed: Mon Mar 13 17:17:14 2017 -0700 ---------------------------------------------------------------------- pom.xml | 1 + .../apache/sysml/api/mlcontext/MLContext.java | 36 ++++++---- .../sysml/api/mlcontext/MLContextUtil.java | 73 ++++++++++++++++++-- .../apache/sysml/api/mlcontext/ProjectInfo.java | 24 ++++++- 4 files changed, 115 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e81b9059/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8be7b18..7730d8b 100644 --- a/pom.xml +++ b/pom.xml @@ -519,6 +519,7 @@ <Group-Id>${project.groupId}</Group-Id> <Artifact-Id>${project.artifactId}</Artifact-Id> <Version>${project.version}</Version> + <Minimum-Recommended-Spark-Version>${spark.version}</Minimum-Recommended-Spark-Version> </manifestEntries> </archive> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e81b9059/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index 7d57065..cb98083 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -57,11 +57,6 @@ import org.apache.sysml.utils.Explain.ExplainType; */ public class MLContext { /** - * Minimum Spark version supported by SystemML. - */ - public static final String SYSTEMML_MINIMUM_SPARK_VERSION = "2.1.0"; - - /** * Logger for MLContext */ public static Logger log = Logger.getLogger(MLContext.class); @@ -116,11 +111,6 @@ public class MLContext { private ExplainLevel explainLevel = null; /** - * Project information such as the version and the build time. - */ - private ProjectInfo projectInfo = null; - - /** * Whether or not all values should be maintained in the symbol table * after execution. */ @@ -225,7 +215,16 @@ public class MLContext { try { MLContextUtil.verifySparkVersionSupported(sc); } catch (MLContextException e) { - log.warn("Apache Spark " + SYSTEMML_MINIMUM_SPARK_VERSION + " or above is recommended for SystemML " + this.info().version()); + if (info() != null) { + log.warn("Apache Spark " + this.info().minimumRecommendedSparkVersion() + " or above is recommended for SystemML " + this.info().version()); + } else { + try { + String minSparkVersion = MLContextUtil.getMinimumRecommendedSparkVersionFromPom(); + log.warn("Apache Spark " + minSparkVersion + " or above is recommended for this version of SystemML."); + } catch (MLContextException e1) { + log.error("Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml"); + } + } } if (activeMLContext == null) { @@ -652,10 +651,13 @@ public class MLContext { * @return information about the project */ public ProjectInfo info() { - if (projectInfo == null) { - projectInfo = new ProjectInfo(); + try { + ProjectInfo projectInfo = ProjectInfo.getProjectInfo(); + return projectInfo; + } catch (Exception e) { + log.warn("Project information not available"); + return null; } - return projectInfo; } /** @@ -664,6 +666,9 @@ public class MLContext { * @return the SystemML version number */ public String version() { + if (info() == null) { + return "Version not available"; + } return info().version(); } @@ -673,6 +678,9 @@ public class MLContext { * @return the SystemML jar file build time */ public String buildTime() { + if (info() == null) { + return "Build time not available"; + } return info().buildTime(); } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e81b9059/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java index 22595c0..17aa4a7 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java @@ -19,6 +19,7 @@ package org.apache.sysml.api.mlcontext; +import java.io.File; import java.io.FileNotFoundException; import java.net.URL; import java.text.DateFormat; @@ -32,6 +33,9 @@ import java.util.Map.Entry; import java.util.Scanner; import java.util.Set; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.text.WordUtils; @@ -72,6 +76,9 @@ import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; /** * Utility class containing methods for working with the MLContext API. @@ -151,10 +158,12 @@ public final class MLContextUtil { * * @param sparkVersion * Spark version string (ie, "1.5.0"). + * @param minimumRecommendedSparkVersion + * Minimum recommended Spark version string (ie, "2.1.0"). * @return {@code true} if Spark version supported; otherwise {@code false}. */ - public static boolean isSparkVersionSupported(String sparkVersion) { - return compareVersion(sparkVersion, MLContext.SYSTEMML_MINIMUM_SPARK_VERSION) >= 0; + public static boolean isSparkVersionSupported(String sparkVersion, String minimumRecommendedSparkVersion) { + return compareVersion(sparkVersion, minimumRecommendedSparkVersion) >= 0; } /** @@ -167,9 +176,65 @@ public final class MLContextUtil { * thrown if Spark version isn't supported */ public static void verifySparkVersionSupported(SparkContext sc) { - if (!MLContextUtil.isSparkVersionSupported(sc.version())) { + String minimumRecommendedSparkVersion = null; + try { + // If this is being called using the SystemML jar file, + // ProjectInfo should be available. + ProjectInfo projectInfo = ProjectInfo.getProjectInfo(); + minimumRecommendedSparkVersion = projectInfo.minimumRecommendedSparkVersion(); + } catch (MLContextException e) { + try { + // During development (such as in an IDE), there is no jar file typically + // built, so attempt to obtain the minimum recommended Spark version from + // the pom.xml file + minimumRecommendedSparkVersion = getMinimumRecommendedSparkVersionFromPom(); + } catch (MLContextException e1) { + throw new MLContextException("Minimum recommended Spark version could not be determined from SystemML jar file manifest or pom.xml"); + } + } + String sparkVersion = sc.version(); + if (!MLContextUtil.isSparkVersionSupported(sparkVersion, minimumRecommendedSparkVersion)) { throw new MLContextException( - "This version of SystemML requires Spark " + MLContext.SYSTEMML_MINIMUM_SPARK_VERSION + " or greater."); + "Spark " + sparkVersion + " or greater is recommended for this version of SystemML."); + } + } + + /** + * Obtain minimum recommended Spark version from the pom.xml file. + * + * @return the minimum recommended Spark version from XML parsing of the pom file (during development). + */ + static String getMinimumRecommendedSparkVersionFromPom() { + return getUniquePomProperty("spark.version"); + } + + /** + * Obtain the text associated with an XML element from the pom.xml file. In this implementation, + * the element should be uniquely named, or results will be unpredicable. + * + * @param property unique property (element) from the pom.xml file + * @return the text value associated with the given property + */ + static String getUniquePomProperty(String property) { + File f = new File("pom.xml"); + if (!f.exists()) { + throw new MLContextException("pom.xml not found"); + } + try { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = dbf.newDocumentBuilder(); + Document document = builder.parse(f); + + NodeList nodes = document.getElementsByTagName(property); + int length = nodes.getLength(); + if (length == 0) { + throw new MLContextException("Property not found in pom.xml"); + } + Node node = nodes.item(0); + String value = node.getTextContent(); + return value; + } catch (Exception e) { + throw new MLContextException("MLContextException when reading property '" + property + "' from pom.xml", e); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e81b9059/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java b/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java index 7d5a6c1..7d6163b 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ProjectInfo.java @@ -36,8 +36,21 @@ import java.util.jar.Manifest; public class ProjectInfo { SortedMap<String, String> properties = null; + static ProjectInfo projectInfo = null; - public ProjectInfo() { + /** + * Return a ProjectInfo singleton instance. + * + * @return the ProjectInfo singleton instance + */ + public static ProjectInfo getProjectInfo() { + if (projectInfo == null) { + projectInfo = new ProjectInfo(); + } + return projectInfo; + } + + private ProjectInfo() { JarFile systemMlJar = null; try { String path = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); @@ -102,6 +115,15 @@ public class ProjectInfo { } /** + * Obtain the minimum recommended Spark version from the manifest. + * + * @return the minimum recommended Spark version + */ + public String minimumRecommendedSparkVersion() { + return property("Minimum-Recommended-Spark-Version"); + } + + /** * Obtain all the properties from the manifest as a sorted map. * * @return the manifest properties as a sorted map
