systemml git commit: [SYSTEMML-2359] Additional paramserv update frequency: per-epoch
Repository: systemml Updated Branches: refs/heads/master 095781868 -> 51057e471 [SYSTEMML-2359] Additional paramserv update frequency: per-epoch Closes #780. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/51057e47 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/51057e47 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/51057e47 Branch: refs/heads/master Commit: 51057e4712d6ab9a190a9c1f8e9f36d48a8a1fd5 Parents: 0957818 Author: EdgarLGB Authored: Mon Jun 4 21:25:53 2018 -0700 Committer: Matthias Boehm Committed: Mon Jun 4 22:02:23 2018 -0700 -- .../ParameterizedBuiltinFunctionExpression.java | 24 +- .../controlprogram/paramserv/LocalPSWorker.java | 139 +-- .../paramserv/LocalParamServer.java | 8 +- .../controlprogram/paramserv/PSWorker.java | 2 +- .../controlprogram/paramserv/ParamServer.java | 42 ++- .../cp/ParamservBuiltinCPInstruction.java | 61 ++- .../functions/paramserv/ParamservFuncTest.java | 22 +- .../paramserv/mnist_lenet_paramserv.dml | 4 +- .../paramserv/mnist_lenet_paramserv_asp.dml | 376 --- .../mnist_lenet_paramserv_minimum_version.dml | 4 +- .../paramserv/paramserv-nn-asp-batch.dml| 52 +++ .../paramserv/paramserv-nn-asp-epoch.dml| 52 +++ .../functions/paramserv/paramserv-nn-asp.dml| 52 --- .../paramserv/paramserv-nn-bsp-batch.dml| 52 +++ .../paramserv/paramserv-nn-bsp-epoch.dml| 52 +++ .../functions/paramserv/paramserv-nn-test.dml | 52 --- .../paramserv/paramserv-wrong-args.dml | 19 +- 17 files changed, 425 insertions(+), 588 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/51057e47/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java -- diff --git a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java index 99aec78..33c5b0e 100644 --- a/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/ParameterizedBuiltinFunctionExpression.java @@ -334,22 +334,15 @@ public class ParameterizedBuiltinFunctionExpression extends DataIdentifier checkDataType(fname, Statement.PS_VAL_LABELS, DataType.MATRIX, conditional); checkDataValueType(false, fname, Statement.PS_UPDATE_FUN, DataType.SCALAR, ValueType.STRING, conditional); checkDataValueType(false, fname, Statement.PS_AGGREGATION_FUN, DataType.SCALAR, ValueType.STRING, conditional); - Set modes = Arrays.stream(Statement.PSModeType.values()).map(Enum::name) - .collect(Collectors.toSet()); - checkStringParam(false, fname, Statement.PS_MODE, modes, conditional); - Set utypes = Arrays.stream(Statement.PSUpdateType.values()).map(Enum::name) - .collect(Collectors.toSet()); - checkStringParam(false, fname, Statement.PS_UPDATE_TYPE, utypes, conditional); - Set frequencies = Arrays.stream(Statement.PSFrequency.values()).map(Enum::name).collect(Collectors.toSet()); - checkStringParam(true, fname, Statement.PS_FREQUENCY, frequencies, conditional); + checkStringParam(false, fname, Statement.PS_MODE, conditional); + checkStringParam(false, fname, Statement.PS_UPDATE_TYPE, conditional); + checkStringParam(true, fname, Statement.PS_FREQUENCY, conditional); checkDataValueType(false, fname, Statement.PS_EPOCHS, DataType.SCALAR, ValueType.INT, conditional); checkDataValueType(true, fname, Statement.PS_BATCH_SIZE, DataType.SCALAR, ValueType.INT, conditional); checkDataValueType(true, fname, Statement.PS_PARALLELISM, DataType.SCALAR, ValueType.INT, conditional); - Set schemes = Arrays.stream(Statement.PSScheme.values()).map(Enum::name).collect(Collectors.toSet()); - checkStringParam(true, fname, Statement.PS_SCHEME, schemes, conditional); + checkStringParam(true, fname, Statement.PS_SCHEME, conditional); checkDataValueType(true, fname, Statement.PS_HYPER_PARAMS, DataType.LIST, ValueType.UNKNOWN, conditional); - Set checkpointings = Arrays.stream(Statement.PSCheckpointing.values()).map(Enum::name).collect(Collectors.toSet()); - checkStringParam(true, fname, Statement.PS_CHECKPOINTING, checkpointings, conditional); + checkStringParam(true, fname, Statement.PS_CHECKPOINTING, conditional);
systemml git commit: [SYSTEMML-2342] Cleanup ARIMA dml script
Repository: systemml
Updated Branches:
refs/heads/master add561b38 -> 095781868
[SYSTEMML-2342] Cleanup ARIMA dml script
Closes #779.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/09578186
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/09578186
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/09578186
Branch: refs/heads/master
Commit: 0957818688c8819a5601a61837504ee83fc22745
Parents: add561b
Author: Tobias Schmidt
Authored: Mon Jun 4 20:34:23 2018 -0700
Committer: Matthias Boehm
Committed: Mon Jun 4 20:42:20 2018 -0700
--
.../integration/applications/ArimaTest.java | 37 +-
.../applications/arima_box-jenkins/arima.dml| 344 ++-
.../applications/arima_box-jenkins/arima_old.R | 278 +++
.../arima_box-jenkins/arima_old.dml | 289
4 files changed, 774 insertions(+), 174 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/systemml/blob/09578186/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
--
diff --git
a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
index 91fa55b..149c0b1 100644
---
a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
+++
b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java
@@ -34,7 +34,9 @@ import org.junit.runners.Parameterized.Parameters;
public abstract class ArimaTest extends AutomatedTestBase {
protected final static String TEST_DIR =
"applications/arima_box-jenkins/";
- protected final static String TEST_NAME = "arima";
+ protected final static String TEST_NAME1 = "arima";
+ protected final static String TEST_NAME2 = "arima_old";
+
protected String TEST_CLASS_DIR = TEST_DIR +
ArimaTest.class.getSimpleName() + "/";
protected int max_func_invoc, p, d, q, P, D, Q, s, include_mean,
useJacobi;
@@ -64,11 +66,12 @@ public abstract class ArimaTest extends AutomatedTestBase {
@Override
public void setUp() {
- addTestConfiguration(TEST_CLASS_DIR, TEST_NAME);
+ addTestConfiguration(TEST_CLASS_DIR, TEST_NAME1);
+ addTestConfiguration(TEST_CLASS_DIR, TEST_NAME2);
}
protected void testArima(ScriptType scriptType) {
- System.out.println(" BEGIN " + TEST_NAME + " " +
scriptType + " TEST WITH {" +
+ System.out.println(" BEGIN " + TEST_NAME1 + " " +
scriptType + " TEST WITH {" +
max_func_invoc + ", " +
p + ", " +
d + ", " +
@@ -81,12 +84,16 @@ public abstract class ArimaTest extends AutomatedTestBase {
useJacobi+ "} ");
this.scriptType = scriptType;
- getAndLoadTestConfiguration(TEST_NAME);
-
List proArgs = new ArrayList();
+
if (scriptType == ScriptType.PYDML) {
proArgs.add("-python");
+ getAndLoadTestConfiguration(TEST_NAME1);
}
+ else {
+ getAndLoadTestConfiguration(TEST_NAME2);
+ }
+
proArgs.add("-args");
proArgs.add(input("col.mtx"));
proArgs.add(Integer.toString(max_func_invoc));
@@ -100,8 +107,26 @@ public abstract class ArimaTest extends AutomatedTestBase {
proArgs.add(Integer.toString(include_mean));
proArgs.add(Integer.toString(useJacobi));
proArgs.add(output("learnt.model"));
- programArgs = proArgs.toArray(new String[proArgs.size()]);
+
+ /* TODO use after R script is made consistent
+ getAndLoadTestConfiguration(TEST_NAME2);
+ proArgs.add("-nvargs");
+ proArgs.add("X="+input("col.mtx"));
+
proArgs.add("max_func="+Integer.toString(max_func_invoc));
+ proArgs.add("p="+Integer.toString(p));
+ proArgs.add("d="+Integer.toString(d));
+ proArgs.add("q="+Integer.toString(q));
+ proArgs.add("P="+Integer.toString(P));
+ proArgs.add("D="+Integer.toString(D));
+ proArgs.add("Q="+Integer.toString(Q));
+ proArgs.add("s="+Integer.toString(s));
+
systemml git commit: [SYSTEMML-2361] Fix invalid cleanup of matrices/frames read in functions
Repository: systemml
Updated Branches:
refs/heads/master 1bcdfaac1 -> add561b38
[SYSTEMML-2361] Fix invalid cleanup of matrices/frames read in functions
This patch fixes a severe bug that led to invalid cleanup of matrices or
frames that are read from persistent storage (e.g., HDFS) in functions.
Although there are never rmvar instructions generated for pread
variables, the function call cleans up unexpected function outputs to
safe guard against special case memory leaks. We now globally disable
the cleanup of all persistently read matrices or frames.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/add561b3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/add561b3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/add561b3
Branch: refs/heads/master
Commit: add561b38e55e16dca7fd0d8a520b5abca5671a3
Parents: 1bcdfaa
Author: Matthias Boehm
Authored: Mon Jun 4 19:44:32 2018 -0700
Committer: Matthias Boehm
Committed: Mon Jun 4 19:44:32 2018 -0700
--
src/main/java/org/apache/sysml/lops/Data.java | 6 +++---
.../sysml/runtime/instructions/cp/VariableCPInstruction.java | 4
2 files changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/lops/Data.java
--
diff --git a/src/main/java/org/apache/sysml/lops/Data.java
b/src/main/java/org/apache/sysml/lops/Data.java
index 53f169b..95df961 100644
--- a/src/main/java/org/apache/sysml/lops/Data.java
+++ b/src/main/java/org/apache/sysml/lops/Data.java
@@ -37,11 +37,11 @@ import org.apache.sysml.parser.Expression.ValueType;
* variables, literals. Can be for both input and output.
*/
-public class Data extends Lop
+public class Data extends Lop
{
-
public enum OperationTypes {READ,WRITE}
-
+ public static final String PREAD_PREFIX =
"p"+OperationTypes.READ.name();
+
FileFormatTypes formatType = FileFormatTypes.BINARY;
OperationTypes operation;
boolean literal_var = false;
http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
--
diff --git
a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index b46f4df..26fcb2e 100644
---
a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++
b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -485,6 +485,8 @@ public class VariableCPInstruction extends CPInstruction {
mobj.setMetaData((MetaData)metadata.clone());
mobj.setFileFormatProperties(_formatProperties);
mobj.setUpdateType(_updateType);
+ mobj.enableCleanup(!getInput1().getName()
+
.startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX));
ec.setVariable(getInput1().getName(), mobj);
if(DMLScript.STATISTICS &&
_updateType.isInPlace())
Statistics.incrementTotalUIPVar();
@@ -496,6 +498,8 @@ public class VariableCPInstruction extends CPInstruction {
fobj.setFileFormatProperties(_formatProperties);
if( _schema != null )
fobj.setSchema(_schema); //after
metadata
+ fobj.enableCleanup(!getInput1().getName()
+
.startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX));
ec.setVariable(getInput1().getName(), fobj);
}
else if ( getInput1().getDataType() == DataType.SCALAR
){
systemml git commit: [SYSTEMML-2134] Fix codegen row tmpl support for vector ternary axpy
Repository: systemml
Updated Branches:
refs/heads/master 8084dc127 -> 1bcdfaac1
[SYSTEMML-2134] Fix codegen row tmpl support for vector ternary axpy
This patch fixes the CPlan construction of row templates for ternary
axpy operations with row vector intermediates. Specifically, we now
correctly handle index loopkups only for scalar intermediates, which
otherwise causes codegen compilation errors.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1bcdfaac
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1bcdfaac
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1bcdfaac
Branch: refs/heads/master
Commit: 1bcdfaac138d8f68e4144ca4ddbaf8cf03329ca1
Parents: 8084dc1
Author: Matthias Boehm
Authored: Mon Jun 4 14:25:05 2018 -0700
Committer: Matthias Boehm
Committed: Mon Jun 4 17:45:07 2018 -0700
--
.../org/apache/sysml/hops/codegen/cplan/CNodeBinary.java | 7 +++
.../apache/sysml/hops/codegen/cplan/CNodeTernary.java| 8 +++-
.../org/apache/sysml/hops/codegen/cplan/CNodeUnary.java | 7 +++
.../apache/sysml/hops/codegen/template/TemplateRow.java | 11 ++-
4 files changed, 15 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/systemml/blob/1bcdfaac/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeBinary.java
--
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeBinary.java
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeBinary.java
index 80a7f83..7ef21c1 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeBinary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeBinary.java
@@ -19,6 +19,8 @@
package org.apache.sysml.hops.codegen.cplan;
+import java.util.Arrays;
+
import org.apache.commons.lang.StringUtils;
import org.apache.sysml.hops.codegen.template.TemplateUtils;
import org.apache.sysml.parser.Expression.DataType;
@@ -56,10 +58,7 @@ public class CNodeBinary extends CNode
MINUS1_MULT, MINUS_NZ;
public static boolean contains(String value) {
- for( BinType bt : values() )
- if( bt.name().equals(value) )
- return true;
- return false;
+ return Arrays.stream(values()).anyMatch(bt ->
bt.name().equals(value));
}
public boolean isCommutative() {
http://git-wip-us.apache.org/repos/asf/systemml/blob/1bcdfaac/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
--
diff --git
a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
index dc8ff82..61140b4 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeTernary.java
@@ -19,6 +19,8 @@
package org.apache.sysml.hops.codegen.cplan;
+import java.util.Arrays;
+
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -30,12 +32,8 @@ public class CNodeTernary extends CNode
REPLACE, REPLACE_NAN, IFELSE,
LOOKUP_RC1, LOOKUP_RVECT1;
-
public static boolean contains(String value) {
- for( TernaryType tt : values() )
- if( tt.name().equals(value) )
- return true;
- return false;
+ return Arrays.stream(values()).anyMatch(tt ->
tt.name().equals(value));
}
public String getTemplate(boolean sparse) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/1bcdfaac/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
--
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
index d7721a1..b269139 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeUnary.java
@@ -19,6 +19,8 @@
package org.apache.sysml.hops.codegen.cplan;
+import java.util.Arrays;
+
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.sysml.hops.codegen.template.TemplateUtils;
@@ -43,10 +45,7 @@ public class CNodeUnary extends CNode
SPROP, SIGMOID;
systemml git commit: [MINOR] add 2 new DML examples to Jupyter notebook
Repository: systemml
Updated Branches:
refs/heads/master d44b3280f -> 8084dc127
[MINOR] add 2 new DML examples to Jupyter notebook
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8084dc12
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8084dc12
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8084dc12
Branch: refs/heads/master
Commit: 8084dc1276663c7dfaa8983b13efb0ec11e6ee1e
Parents: d44b328
Author: Berthold Reinwald
Authored: Mon Jun 4 16:54:14 2018 -0700
Committer: Berthold Reinwald
Committed: Mon Jun 4 16:54:14 2018 -0700
--
...DML Tips and Tricks (aka Fun With DML).ipynb | 691 ---
1 file changed, 576 insertions(+), 115 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/systemml/blob/8084dc12/samples/jupyter-notebooks/DML
Tips and Tricks (aka Fun With DML).ipynb
--
diff --git a/samples/jupyter-notebooks/DML Tips and Tricks (aka Fun With
DML).ipynb b/samples/jupyter-notebooks/DML Tips and Tricks (aka Fun With
DML).ipynb
index b2d2fad..6dd096c 100644
--- a/samples/jupyter-notebooks/DML Tips and Tricks (aka Fun With DML).ipynb
+++ b/samples/jupyter-notebooks/DML Tips and Tricks (aka Fun With DML).ipynb
@@ -4,13 +4,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
-"1. [Replace NaN with mode](#NaN2Mode)\n",
+"1. [Create all value pairs for v1 and v2](#AllValuePairs)\n",
+"* [Replace NaN with mode](#NaN2Mode)\n",
"* [Use sample builtin function to create sample from matrix](#sample)\n",
"* [Count of Matching Values in two Matrices/Vectors](#MatchinRows)\n",
"* [Cross Validation](#CrossValidation)\n",
"* [Value-based join of two Matrices](#JoinMatrices)\n",
"* [Filter Matrix to include only Frequent Column
Values](#FilterMatrix)\n",
-"* [Construct (sparse) Matrix from (rowIndex, colIndex, values)
triplets](#Construct_sparse_Matrix)\n",
+"* [(Sparse) Matrix to/from (rowIndex, colIndex, values) conversions
(i,j,v)](#Construct_sparse_Matrix)\n",
"* [Find and remove duplicates in columns or
rows](#Find_and_remove_duplicates)\n",
"* [Set based Indexing](#Set_based_Indexing)\n",
"* [Group by Aggregate using Linear Algebra](#Multi_column_Sorting)\n",
@@ -20,25 +21,95 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": false
},
+ "outputs": [],
+ "source": [
+"from systemml import MLContext, dml\n",
+"ml = MLContext(sc)\n",
+"\n",
+"print (ml.buildTime())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+"## Create all value pairs for v1 and v2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {
+"collapsed": false
+ },
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2017-09-22 07:57:57 UTC\n"
+ "2.000\n",
+ "1.000\n",
+ "8.000\n",
+ "3.000\n",
+ "5.000\n",
+ "6.000\n",
+ "7.000\n",
+ "\n",
+ "80.000\n",
+ "20.000\n",
+ "50.000\n",
+ "\n",
+ "2.000 80.000\n",
+ "2.000 20.000\n",
+ "2.000 50.000\n",
+ "1.000 80.000\n",
+ "1.000 20.000\n",
+ "1.000 50.000\n",
+ "8.000 80.000\n",
+ "8.000 20.000\n",
+ "8.000 50.000\n",
+ "3.000 80.000\n",
+ "3.000 20.000\n",
+ "3.000 50.000\n",
+ "5.000 80.000\n",
+ "5.000 20.000\n",
+ "5.000 50.000\n",
+ "6.000 80.000\n",
+ "6.000 20.000\n",
+ "6.000 50.000\n",
+ "7.000 80.000\n",
+ "7.000 20.000\n",
+ "7.000 50.000\n",
+ "\n",
+ "SystemML Statistics:\n",
+ "Total execution time:\t\t0.000 sec.\n",
+ "Number of executed Spark inst:\t0.\n",
+ "\n",
+ "\n"
]
}
],
"source": [
-"from systemml import MLContext, dml, jvm_stdout\n",
-"ml = MLContext(sc)\n",
-"\n",
-"print (ml.buildTime())"
+"prog=\"\"\"\n",
+"v1 = matrix ('2 1 8 3 5 6 7', rows = 7, cols = 1 )\n",
+"v2 = matrix ('80 20 50', rows = 3, cols = 1 )\n",
+"\n",
+"nv1 = nrow (v1);\n",
+"nv2 = nrow (v2);\n",
+"R = cbind (\n",
+" matrix (v1 %*% matrix(1, 1, nv2), nv1*nv2, 1),\n",
+" matrix (matrix(1, nv1, 1) %*% t(v2), nv1*nv2, 1))\n",
+"\n",
+"print(toString(v1));\n",
+"print(toString(v2));\n",
+"print(toString(R));\n",
+"\"\"\"\n",
+"res = ml.execute(dml(prog))"
]
},
{
@@ -52,12 +123,12 @@
"cell_type": "markdown",
"metadata": {},
"source": [
-"This functions replaces NaN in column with mode of column"
+"This functions replaces NaN in
systemml git commit: [SYSTEMML-2344, 48, 49, 52] Various improvements local paramserv backend
Repository: systemml
Updated Branches:
refs/heads/master 2b86a4d92 -> d44b3280f
[SYSTEMML-2344,48,49,52] Various improvements local paramserv backend
Closes #777.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d44b3280
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d44b3280
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d44b3280
Branch: refs/heads/master
Commit: d44b3280f2132deb303e955ff5b9a17daac4c31e
Parents: 2b86a4d
Author: EdgarLGB
Authored: Sun Jun 3 23:07:02 2018 -0700
Committer: Matthias Boehm
Committed: Sun Jun 3 23:07:03 2018 -0700
--
.../java/org/apache/sysml/parser/Statement.java | 8 +-
.../controlprogram/paramserv/LocalPSWorker.java | 99 ++---
.../paramserv/LocalParamServer.java | 40 +-
.../controlprogram/paramserv/PSWorker.java | 113 +++---
.../controlprogram/paramserv/ParamServer.java | 227 +--
.../paramserv/ParamservUtils.java | 12 +-
.../cp/ParamservBuiltinCPInstruction.java | 260 +
.../functions/paramserv/ParamservFuncTest.java | 39 +-
.../paramserv/mnist_lenet_paramserv.dml | 1 -
.../paramserv/mnist_lenet_paramserv_asp.dml | 376 +++
.../mnist_lenet_paramserv_minimum_version.dml | 1 -
.../paramserv/paramserv-agg-service-failed.dml | 53 +++
.../paramserv/paramserv-large-parallelism.dml | 52 +++
.../functions/paramserv/paramserv-nn-asp.dml| 52 +++
.../paramserv/paramserv-worker-failed.dml | 53 +++
.../paramserv-wrong-aggregate-func.dml | 50 +++
16 files changed, 1117 insertions(+), 319 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/systemml/blob/d44b3280/src/main/java/org/apache/sysml/parser/Statement.java
--
diff --git a/src/main/java/org/apache/sysml/parser/Statement.java
b/src/main/java/org/apache/sysml/parser/Statement.java
index 1987d31..d49eada 100644
--- a/src/main/java/org/apache/sysml/parser/Statement.java
+++ b/src/main/java/org/apache/sysml/parser/Statement.java
@@ -77,7 +77,13 @@ public abstract class Statement implements ParseInfo
}
public static final String PS_UPDATE_TYPE = "utype";
public enum PSUpdateType {
- BSP, ASP, SSP
+ BSP, ASP, SSP;
+ public boolean isBSP() {
+ return this == BSP;
+ }
+ public boolean isASP() {
+ return this == ASP;
+ }
}
public static final String PS_FREQUENCY = "freq";
public enum PSFrequency {
http://git-wip-us.apache.org/repos/asf/systemml/blob/d44b3280/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
--
diff --git
a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
index 181b866..e902aea 100644
---
a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
+++
b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java
@@ -19,79 +19,82 @@
package org.apache.sysml.runtime.controlprogram.paramserv;
+import java.util.concurrent.Callable;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.cp.ListObject;
-public class LocalPSWorker extends PSWorker implements Runnable {
+public class LocalPSWorker extends PSWorker implements Callable {
protected static final Log LOG =
LogFactory.getLog(LocalPSWorker.class.getName());
- public LocalPSWorker(long workerID, String updFunc,
Statement.PSFrequency freq, int epochs, long batchSize,
- ListObject hyperParams, ExecutionContext ec,
ParamServer ps) {
- super(workerID, updFunc, freq, epochs, batchSize, hyperParams,
ec, ps);
+ public LocalPSWorker(int workerID, String updFunc,
Statement.PSFrequency freq, int epochs, long batchSize,
+ ExecutionContext ec, ParamServer ps) {
+ super(workerID, updFunc, freq, epochs, batchSize, ec, ps);
}
@Override
- public void run() {
+ public Void call() throws Exception {
+ try {
+ long dataSize = _features.getNumRows();
+ for (int i = 0; i < _epochs; i++) {
+
