Repository: incubator-systemml Updated Branches: refs/heads/master 8d5f3ceab -> 062ebc67a
[SYSTEMML-929] Enable frame csv reblock for frame-based transform Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/062ebc67 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/062ebc67 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/062ebc67 Branch: refs/heads/master Commit: 062ebc67a821e12c7e3c04eab3b5d863242a7445 Parents: 8d5f3ce Author: Matthias Boehm <mbo...@us.ibm.com> Authored: Sat Sep 17 01:02:26 2016 -0700 Committer: Matthias Boehm <mbo...@us.ibm.com> Committed: Sat Sep 17 01:02:26 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/OptimizerUtils.java | 5 +---- .../parser/ParameterizedBuiltinFunctionExpression.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/062ebc67/src/main/java/org/apache/sysml/hops/OptimizerUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java index c19ad57..b0a938d 100644 --- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java @@ -205,11 +205,8 @@ public class OptimizerUtils /** * Enables automatic csv-binary block reblock. - * - * TODO enable by default and remove once file-based transform completely - * removed via frame-based transform/transformapply */ - public static boolean ALLOW_FRAME_CSV_REBLOCK = false; + public static boolean ALLOW_FRAME_CSV_REBLOCK = true; public static long GPU_MEMORY_BUDGET = -1; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/062ebc67/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java index 679075b..90364d7 100644 --- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.HashSet; import org.apache.sysml.hops.Hop.ParamBuiltinOp; +import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.parser.LanguageException.LanguageErrorCodes; @@ -298,7 +299,9 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier } // example: A = transform(data=D, txmtd="", txspec="") - private void validateTransform(DataIdentifier output, boolean conditional) throws LanguageException { + private void validateTransform(DataIdentifier output, boolean conditional) + throws LanguageException + { //validate data checkDataType("transform", TF_FN_PARAM_DATA, DataType.FRAME, conditional); @@ -337,6 +340,14 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier raiseValidateError("Only one of '" + TF_FN_PARAM_APPLYMTD + "' or '" + TF_FN_PARAM_OUTNAMES + "' can be specified in transform().", conditional, LanguageErrorCodes.INVALID_PARAMETERS); } + // disable frame csv reblocks as transform operates directly over csv files + // (this is required to support both file-based transform and frame-based + // transform at the same time; hence, transform and frame-based transform + // functions over csv cannot be used in the same script; accordingly we + // give an appropriate warning) + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = false; + raiseValidateError("Disable frame csv reblock to support file-based transform.", true); + // Output is a matrix with same dims as input output.setDataType(DataType.MATRIX); output.setFormatType(FormatType.CSV);
