Repository: systemml Updated Branches: refs/heads/master f31548007 -> a4ce06461
[SYSTEMML-1765] Support for reading dml scripts from object stores This patch generates the various methods for reading dml scripts files to support (apart from local fs and hdfs) also the read from object stores such as swift and s3. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/586c67b6 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/586c67b6 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/586c67b6 Branch: refs/heads/master Commit: 586c67b6a47950305f1bb57ba809aa295a83861b Parents: f315480 Author: Matthias Boehm <[email protected]> Authored: Thu Jul 13 17:20:20 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Thu Jul 13 19:46:25 2017 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/api/DMLScript.java | 7 +++---- .../java/org/apache/sysml/api/jmlc/Connection.java | 4 ++-- .../org/apache/sysml/api/mlcontext/ScriptFactory.java | 14 +++++++------- src/main/java/org/apache/sysml/conf/DMLConfig.java | 4 ++-- .../java/org/apache/sysml/parser/ParserWrapper.java | 7 ++++--- 5 files changed, 18 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 6dd7e89..515d632 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -567,8 +567,8 @@ public class DMLScript try { //read from hdfs or gpfs file system - if( fileName.startsWith("hdfs:") - || fileName.startsWith("gpfs:") ) + if( fileName.startsWith("hdfs:") || fileName.startsWith("gpfs:") + || IOUtilFunctions.isObjectStoreFileScheme(new Path(fileName)) ) { Path scriptPath = new Path(fileName); FileSystem fs = IOUtilFunctions.getFileSystem(scriptPath); @@ -588,8 +588,7 @@ public class DMLScript sb.append( "\n" ); } } - catch (IOException ex) - { + catch (IOException ex) { LOG.error("Failed to read the script from the file system", ex); throw ex; } http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/jmlc/Connection.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index af50139..7739a35 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -250,8 +250,8 @@ public class Connection implements Closeable try { //read from hdfs or gpfs file system - if( fname.startsWith("hdfs:") - || fname.startsWith("gpfs:") ) + if( fname.startsWith("hdfs:") || fname.startsWith("gpfs:") + || IOUtilFunctions.isObjectStoreFileScheme(new Path(fname)) ) { Path scriptPath = new Path(fname); FileSystem fs = IOUtilFunctions.getFileSystem(scriptPath); http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java index 4a30f00..10cd947 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java @@ -351,20 +351,20 @@ public class ScriptFactory { throw new MLContextException("Script file path is null"); } try { - if (scriptFilePath.startsWith("hdfs:") || scriptFilePath.startsWith("gpfs:")) { + if ( scriptFilePath.startsWith("hdfs:") || scriptFilePath.startsWith("gpfs:") + || IOUtilFunctions.isObjectStoreFileScheme(new Path(scriptFilePath))) { Path path = new Path(scriptFilePath); FileSystem fs = IOUtilFunctions.getFileSystem(path); - FSDataInputStream fsdis = fs.open(path); - return IOUtils.toString(fsdis); + try( FSDataInputStream fsdis = fs.open(path) ) { + return IOUtils.toString(fsdis); + } } else {// from local file system File scriptFile = new File(scriptFilePath); return FileUtils.readFileToString(scriptFile); } - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | IOException e) { throw new MLContextException("Error trying to read script string from file: " + scriptFilePath, e); - } catch (IOException e) { - throw new MLContextException("Error trying to read script string from file: " + scriptFilePath, e); - } + } } /** http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 3a21d95..00a591c 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -172,8 +172,8 @@ public class DMLConfig factory.setIgnoringComments(true); //ignore XML comments DocumentBuilder builder = factory.newDocumentBuilder(); Document domTree = null; - if (_fileName.startsWith("hdfs:") || - _fileName.startsWith("gpfs:") ) // config file from DFS + if( _fileName.startsWith("hdfs:") || _fileName.startsWith("gpfs:") + || IOUtilFunctions.isObjectStoreFileScheme(new Path(_fileName)) ) { Path configFilePath = new Path(_fileName); FileSystem DFS = IOUtilFunctions.getFileSystem(configFilePath); http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/parser/ParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java index 9a32b00..71e405b 100644 --- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java +++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java @@ -94,11 +94,12 @@ public abstract class ParserWrapper { try { //read from hdfs or gpfs file system - if( script.startsWith("hdfs:") - || script.startsWith("gpfs:") ) + if( script.startsWith("hdfs:") || script.startsWith("gpfs:") + || IOUtilFunctions.isObjectStoreFileScheme(new Path(script)) ) { - LOG.debug("Looking for the following file in HDFS or GPFS: " + script); Path scriptPath = new Path(script); + String scheme = (scriptPath.toUri()!=null) ? scriptPath.toUri().getScheme() : null; + LOG.debug("Looking for the following file in "+scheme+": " + script); FileSystem fs = IOUtilFunctions.getFileSystem(scriptPath); in = new BufferedReader(new InputStreamReader(fs.open(scriptPath))); }
