Repository: systemml Updated Branches: refs/heads/master bdf27084b -> 8f8743c81
[SYSTEMML-2481,2482] Fix paramserv list cleanup (unnecessary evictions) This patch fixes paramserv performance issues due to unnecessary evictions. The reason was that the new list cleanup no longer maintains the status of included objects explicitly but the paramserv list cleanup depends on this information. We now redundantly maintain this information which restores the original paramserv behavior. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8f8743c8 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8f8743c8 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8f8743c8 Branch: refs/heads/master Commit: 8f8743c81c110738ea796972ebeb61ce13830874 Parents: bdf2708 Author: Matthias Boehm <[email protected]> Authored: Thu Aug 2 13:45:55 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Thu Aug 2 13:46:18 2018 -0700 ---------------------------------------------------------------------- .../sysml/runtime/instructions/cp/ListObject.java | 12 +++++++++++- .../cp/ParameterizedBuiltinCPInstruction.java | 2 +- .../instructions/cp/ScalarBuiltinNaryCPInstruction.java | 2 +- .../functions/paramserv/ParamservLocalNNTest.java | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/8f8743c8/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java index 8834514..863b77c 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ListObject.java @@ -55,6 +55,15 @@ public class ListObject extends Data { _dataState = Arrays.copyOf(that._dataState, getLength()); } + public void deriveAndSetStatusFromData() { + _dataState = new boolean[_data.size()]; + for(int i=0; i<_data.size(); i++) { + Data dat = _data.get(i); + if( dat instanceof CacheableData<?> ) + _dataState[i] = ((CacheableData<?>) dat).isCleanupEnabled(); + } + } + public void setStatus(boolean[] status) { _dataState = status; } @@ -129,7 +138,8 @@ public class ListObject extends Data { ListObject ret = isNamedList() ? new ListObject(new ArrayList<>(getData()), new ArrayList<>(getNames())) : new ListObject(new ArrayList<>(getData())); - ret.setStatus(Arrays.copyOf(getStatus(), getLength())); + if( getStatus() != null ) + ret.setStatus(Arrays.copyOf(getStatus(), getLength())); return ret; } http://git-wip-us.apache.org/repos/asf/systemml/blob/8f8743c8/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index 4435738..8b76ad7 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -344,7 +344,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction //create list object over all inputs ListObject list = new ListObject(data, names); - list.setStatus(new boolean[params.size()]); + list.deriveAndSetStatusFromData(); ec.setVariable(output.getName(), list); } http://git-wip-us.apache.org/repos/asf/systemml/blob/8f8743c8/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java index c3363da..75e22c2 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinNaryCPInstruction.java @@ -97,7 +97,7 @@ public class ScalarBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction { //create list object over all inputs ListObject list = new ListObject(data); - list.setStatus(new boolean[data.size()]); + list.deriveAndSetStatusFromData(); ec.setVariable(output.getName(), list); } http://git-wip-us.apache.org/repos/asf/systemml/blob/8f8743c8/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservLocalNNTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservLocalNNTest.java b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservLocalNNTest.java index 905bfd1..fca52c3 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservLocalNNTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/ParamservLocalNNTest.java @@ -74,7 +74,7 @@ public class ParamservLocalNNTest extends AutomatedTestBase { private void runDMLTest(int epochs, int workers, Statement.PSUpdateType utype, Statement.PSFrequency freq, int batchsize, Statement.PSScheme scheme) { TestConfiguration config = getTestConfiguration(ParamservLocalNNTest.TEST_NAME); loadTestConfiguration(config); - programArgs = new String[] { "-explain", "-nvargs", "mode=LOCAL", "epochs=" + epochs, + programArgs = new String[] { "-explain", "-stats", "-nvargs", "mode=LOCAL", "epochs=" + epochs, "workers=" + workers, "utype=" + utype, "freq=" + freq, "batchsize=" + batchsize, "scheme=" + scheme }; String HOME = SCRIPT_DIR + TEST_DIR;
