Repository: systemml Updated Branches: refs/heads/master 9593b7fbe -> 504617758
[SYSTEMML-2446] Fix paramserv model list cleanup for partial updates Closes #802. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/bca1f1c7 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/bca1f1c7 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/bca1f1c7 Branch: refs/heads/master Commit: bca1f1c758b076ceb39febe3c4a6f8757655005d Parents: 9593b7f Author: EdgarLGB <guobao...@atos.net> Authored: Wed Jul 18 22:16:34 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Thu Jul 19 00:09:09 2018 -0700 ---------------------------------------------------------------------- .../controlprogram/paramserv/LocalPSWorker.java | 6 ++-- .../controlprogram/paramserv/ParamServer.java | 8 ++--- .../paramserv/ParamservUtils.java | 34 +++++++++++++++++--- 3 files changed, 36 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/bca1f1c7/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java index 307669e..bbf2dbe 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/LocalPSWorker.java @@ -85,7 +85,7 @@ public class LocalPSWorker extends PSWorker implements Callable<Void> { // Update the local model with gradients if( j < totalIter - 1 ) params = updateModel(params, gradients, i, j, totalIter); - ParamservUtils.cleanupListObject(gradients); + ParamservUtils.cleanupListObject(_ec, gradients); } // Push the gradients to ps @@ -183,8 +183,8 @@ public class LocalPSWorker extends PSWorker implements Callable<Void> { // Get the gradients ListObject gradients = (ListObject) _ec.getVariable(_output.getName()); - ParamservUtils.cleanupData(bFeatures); - ParamservUtils.cleanupData(bLabels); + ParamservUtils.cleanupData(_ec, bFeatures); + ParamservUtils.cleanupData(_ec, bLabels); return gradients; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/bca1f1c7/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java index 432d4fc..bd8ee36 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamServer.java @@ -138,7 +138,7 @@ public abstract class ParamServer _accGradients, gradients, true); else updateGlobalModel(gradients); - ParamservUtils.cleanupListObject(gradients); + ParamservUtils.cleanupListObject(_ec, gradients); if (allFinished()) { // Update the global model with accrued gradients @@ -192,11 +192,11 @@ public abstract class ParamServer // Invoke the aggregate function _inst.processInstruction(ec); - // Get the output + // Get the new model ListObject newModel = (ListObject) ec.getVariable(_outputName); - // Update the model with the new output - ParamservUtils.cleanupListObject(ec, Statement.PS_MODEL); + // Clean up the list according to the data referencing status + ParamservUtils.cleanupListObject(ec, Statement.PS_MODEL, newModel.getStatus()); ParamservUtils.cleanupListObject(ec, Statement.PS_GRADIENTS); return newModel; } http://git-wip-us.apache.org/repos/asf/systemml/blob/bca1f1c7/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java index ee15709..b9fd7a8 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java @@ -101,21 +101,45 @@ public class ParamservUtils { return new ListObject(newData, lo.getNames()); } + /** + * Clean up the list object according to its own data status + * @param ec execution context + * @param lName list var name + */ public static void cleanupListObject(ExecutionContext ec, String lName) { ListObject lo = (ListObject) ec.removeVariable(lName); - cleanupListObject(lo); + cleanupListObject(ec, lo, lo.getStatus()); + } + + /** + * Clean up the list object according to the given array of data status (i.e., false => not be removed) + * @param ec execution context + * @param lName list var name + * @param status data status + */ + public static void cleanupListObject(ExecutionContext ec, String lName, boolean[] status) { + ListObject lo = (ListObject) ec.removeVariable(lName); + cleanupListObject(ec, lo, status); } - public static void cleanupListObject(ListObject lo) { - lo.getData().forEach(ParamservUtils::cleanupData); + public static void cleanupListObject(ExecutionContext ec, ListObject lo) { + cleanupListObject(ec, lo, lo.getStatus()); + } + + public static void cleanupListObject(ExecutionContext ec, ListObject lo, boolean[] status) { + for (int i = 0; i < lo.getLength(); i++) { + if (status != null && !status[i]) + continue; // data ref by other object must not be cleaned up + ParamservUtils.cleanupData(ec, lo.getData().get(i)); + } } - public static void cleanupData(Data data) { + public static void cleanupData(ExecutionContext ec, Data data) { if (!(data instanceof CacheableData)) return; CacheableData<?> cd = (CacheableData<?>) data; cd.enableCleanup(true); - cd.clearData(); + ec.cleanupCacheableData(cd); } public static MatrixObject newMatrixObject(MatrixBlock mb) {
