This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 070667d [MINOR] Fix local libsvm write (no output file in
pread-pwrite case)
070667d is described below
commit 070667d7793a5df47617bc72ec526f23662955d1
Author: Matthias Boehm <[email protected]>
AuthorDate: Wed May 12 18:29:27 2021 +0200
[MINOR] Fix local libsvm write (no output file in pread-pwrite case)
---
.../cp/ParameterizedBuiltinCPInstruction.java | 1 -
.../instructions/cp/VariableCPInstruction.java | 81 ++++++++++------------
2 files changed, 37 insertions(+), 45 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index 5ff1e8e..c8a1166 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -341,7 +341,6 @@ public class ParameterizedBuiltinCPInstruction extends
ComputationCPInstruction
int decimal = (getParam("decimal") != null) ?
Integer.parseInt(getParam("decimal")) : TOSTRING_DECIMAL;
boolean sparse = (getParam("sparse") != null) ?
Boolean.parseBoolean(getParam("sparse")) : TOSTRING_SPARSE;
String separator = (getParam("sep") != null) ?
getParam("sep") : TOSTRING_SEPARATOR;
- String indexSeparator = (getParam("indSep") != null) ?
getParam("indSep") : TOSTRING_SEPARATOR;
String lineSeparator = (getParam("linesep") != null) ?
getParam("linesep") : TOSTRING_LINESEPARATOR;
// get input matrix/frame and convert to string
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
index 7ca7ef2..c361f89 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
@@ -54,7 +54,6 @@ import org.apache.sysds.runtime.io.FileFormatPropertiesLIBSVM;
import org.apache.sysds.runtime.io.IOUtilFunctions;
import org.apache.sysds.runtime.io.WriterMatrixMarket;
import org.apache.sysds.runtime.io.WriterTextCSV;
-import org.apache.sysds.runtime.io.WriterTextLIBSVM;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.lineage.LineageItemUtils;
import org.apache.sysds.runtime.lineage.LineageTraceable;
@@ -624,7 +623,7 @@ public class VariableCPInstruction extends CPInstruction
implements LineageTrace
/**
* Handler for processInstruction "CreateVariable" case
- *
+ *
* @param ec execution context of the instruction
*/
private void processCreateVariableInstruction(ExecutionContext ec){
@@ -989,8 +988,8 @@ public class VariableCPInstruction extends CPInstruction
implements LineageTrace
writeMMFile(ec, fname);
else if( fmt == FileFormat.CSV )
writeCSVFile(ec, fname);
- else if(fmt == FileFormat.LIBSVM)
- writeLIBSVMFile(ec, fname);
+ else if(fmt == FileFormat.LIBSVM)
+ writeLIBSVMFile(ec, fname);
else {
// Default behavior
MatrixObject mo =
ec.getMatrixObject(getInput1().getName());
@@ -1022,10 +1021,10 @@ public class VariableCPInstruction extends
CPInstruction implements LineageTrace
if ( getInput3().getName().equalsIgnoreCase("remote") )
((MatrixObject)data).setFileName(getInput2().getName());
else
- throw new DMLRuntimeException(
- "Invalid location (" + getInput3().getName() + ") in SetFileName
instruction: " + instString);
- }
- else
+ throw new DMLRuntimeException(
+ "Invalid location (" +
getInput3().getName() + ") in SetFileName instruction: " + instString);
+ }
+ else
throw new DMLRuntimeException("Invalid data type (" +
getInput1().getDataType() + ") in SetFileName instruction: " + instString);
}
@@ -1072,42 +1071,36 @@ public class VariableCPInstruction extends
CPInstruction implements LineageTrace
else {
mo.exportData(fname, outFmt,
_formatProperties);
}
- HDFSTool.writeMetaDataFile(fname + ".mtd", mo.getValueType(), dc,
FileFormat.CSV, _formatProperties,
- mo.getPrivacyConstraint());
- }
- catch(IOException e) {
- throw new DMLRuntimeException(e);
- }
- }
- }
-
- /**
- * Helper function to write LIBSVM files to HDFS.
- *
- * @param ec execution context
- * @param fname file name
- */
- private void writeLIBSVMFile(ExecutionContext ec, String fname) {
- MatrixObject mo = ec.getMatrixObject(getInput1().getName());
- String outFmt = "libsvm";
-
- if(mo.isDirty()) {
- // there exist data computed in CP that is not backed up on HDFS
- // i.e., it is either in-memory or in evicted space
- mo.exportData(fname, outFmt, _formatProperties);
- }
- else {
- try {
- FileFormat fmt = ((MetaDataFormat) mo.getMetaData()).getFileFormat();
- DataCharacteristics dc = (mo.getMetaData()).getDataCharacteristics();
- if(fmt == FileFormat.LIBSVM &&
!getInput1().getName().startsWith(org.apache.sysds.lops.Data.PREAD_PREFIX)) {
- WriterTextLIBSVM writer = new
WriterTextLIBSVM((FileFormatPropertiesLIBSVM) _formatProperties);
- }
- else {
- mo.exportData(fname, outFmt, _formatProperties);
- }
- HDFSTool.writeMetaDataFile(fname + ".mtd", mo.getValueType(), dc,
FileFormat.LIBSVM, _formatProperties,
- mo.getPrivacyConstraint());
+ HDFSTool.writeMetaDataFile(fname + ".mtd",
mo.getValueType(),
+ dc, FileFormat.CSV, _formatProperties,
mo.getPrivacyConstraint());
+ }
+ catch(IOException e) {
+ throw new DMLRuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Helper function to write LIBSVM files to HDFS.
+ *
+ * @param ec execution context
+ * @param fname file name
+ */
+ private void writeLIBSVMFile(ExecutionContext ec, String fname) {
+ MatrixObject mo = ec.getMatrixObject(getInput1().getName());
+ String outFmt = "libsvm";
+
+ if(mo.isDirty()) {
+ // there exist data computed in CP that is not backed
up on HDFS
+ // i.e., it is either in-memory or in evicted space
+ mo.exportData(fname, outFmt, _formatProperties);
+ }
+ else {
+ try {
+ mo.exportData(fname, outFmt, _formatProperties);
+ HDFSTool.writeMetaDataFile(fname + ".mtd",
mo.getValueType(),
+
mo.getMetaData().getDataCharacteristics(), FileFormat.LIBSVM, _formatProperties,
+ mo.getPrivacyConstraint());
}
catch (IOException e) {
throw new DMLRuntimeException(e);
