[SYSTEMML-2101] Fix JMLC robustness for missing fs impl classes This patch increases the robustness of JMLC for missing file system classes. Thrown NoClassDefFoundErrors so far passed the exception handling leading to failures even though the persistent reads are later replaced by in-memory inputs.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/56205d02 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/56205d02 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/56205d02 Branch: refs/heads/master Commit: 56205d025bf049cc52acd99c87cbf287ddf2929a Parents: 4b5b14b Author: Matthias Boehm <[email protected]> Authored: Tue Jan 30 22:56:18 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Tue Jan 30 22:56:18 2018 -0800 ---------------------------------------------------------------------- .../controlprogram/parfor/opt/OptimizerRuleBased.java | 8 ++++---- .../org/apache/sysml/runtime/io/IOUtilFunctions.java | 14 +++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/56205d02/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java index d551d58..0359f7f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java @@ -95,6 +95,7 @@ import org.apache.sysml.runtime.instructions.cp.Data; import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction; import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool; import org.apache.sysml.runtime.instructions.spark.data.RDDObject; +import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.MetaDataFormat; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -1096,15 +1097,14 @@ public class OptimizerRuleBased extends Optimizer //account for remaining hdfs capacity try { - FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf()); + FileSystem fs = IOUtilFunctions.getFileSystem(ConfigurationManager.getCachedJobConf()); long hdfsCapacityRemain = fs.getStatus().getRemaining(); long sizeInputs = 0; //sum of all input sizes (w/o replication) - for( String var : partitionedMatrices.keySet() ) - { + for( String var : partitionedMatrices.keySet() ) { MatrixObject mo = (MatrixObject)vars.get(var); Path fname = new Path(mo.getFileName()); if( fs.exists( fname ) ) //non-existing (e.g., CP) -> small file - sizeInputs += fs.getContentSummary(fname).getLength(); + sizeInputs += fs.getContentSummary(fname).getLength(); } replication = (int) Math.min(replication, Math.floor(0.9*hdfsCapacityRemain/sizeInputs)); http://git-wip-us.apache.org/repos/asf/systemml/blob/56205d02/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 8d55eab..526ad98 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -67,8 +67,20 @@ public class IOUtilFunctions ConfigurationManager.getCachedJobConf()); } + public static FileSystem getFileSystem(Configuration conf) throws IOException { + try{ + return FileSystem.get(conf); + } catch(NoClassDefFoundError err) { + throw new IOException(err.getMessage()); + } + } + public static FileSystem getFileSystem(Path fname, Configuration conf) throws IOException { - return FileSystem.get(fname.toUri(), conf); + try { + return FileSystem.get(fname.toUri(), conf); + } catch(NoClassDefFoundError err) { + throw new IOException(err.getMessage()); + } } public static boolean isSameFileScheme(Path path1, Path path2) {
