This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 6029c07 [MINOR] Federated ops count in stats
6029c07 is described below
commit 6029c07a3ddce313e27d914a9c7dffb53b48b4cc
Author: baunsgaard <[email protected]>
AuthorDate: Sat Aug 15 15:08:10 2020 +0200
[MINOR] Federated ops count in stats
Closes #1022.
---
.../controlprogram/federated/FederatedData.java | 1 +
.../controlprogram/federated/FederatedRequest.java | 2 ++
.../java/org/apache/sysds/utils/Statistics.java | 39 ++++++++++++++++++++++
3 files changed, 42 insertions(+)
diff --git
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
index 2c5f902..5e83dc2 100644
---
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
+++
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedData.java
@@ -36,6 +36,7 @@ import org.apache.sysds.common.Types;
import org.apache.sysds.conf.DMLConfig;
import org.apache.sysds.runtime.DMLRuntimeException;
import
org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
+import org.apache.sysds.utils.Statistics;
import java.net.InetSocketAddress;
import java.util.concurrent.Future;
diff --git
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
index 5880851..5618d36 100644
---
a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
+++
b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedRequest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.utils.Statistics;
public class FederatedRequest implements Serializable {
private static final long serialVersionUID = 5946781306963870394L;
@@ -57,6 +58,7 @@ public class FederatedRequest implements Serializable {
}
public FederatedRequest(RequestType method, long id, List<Object> data)
{
+ Statistics.incFederated(method);
_method = method;
_id = id;
_data = data;
diff --git a/src/main/java/org/apache/sysds/utils/Statistics.java
b/src/main/java/org/apache/sysds/utils/Statistics.java
index cab22cd..0a6ec38 100644
--- a/src/main/java/org/apache/sysds/utils/Statistics.java
+++ b/src/main/java/org/apache/sysds/utils/Statistics.java
@@ -37,6 +37,7 @@ import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext;
+import
org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
import org.apache.sysds.runtime.instructions.Instruction;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import org.apache.sysds.runtime.instructions.cp.FunctionCallCPInstruction;
@@ -135,6 +136,13 @@ public class Statistics
private static final LongAdder lTotalLix = new LongAdder();
private static final LongAdder lTotalLixUIP = new LongAdder();
+ // Federated stats
+ private static final LongAdder federatedReadCount = new LongAdder();
+ private static final LongAdder federatedPutCount = new LongAdder();
+ private static final LongAdder federatedGetCount = new LongAdder();
+ private static final LongAdder federatedExecuteInstructionCount = new
LongAdder();
+ private static final LongAdder federatedExecuteUDFCount = new
LongAdder();
+
private static LongAdder numNativeFailures = new LongAdder();
public static LongAdder numNativeLibMatrixMultCalls = new LongAdder();
public static LongAdder numNativeConv2dCalls = new LongAdder();
@@ -376,6 +384,28 @@ public class Statistics
parforMergeTime += time;
}
+ public static synchronized void incFederated(RequestType rqt){
+ switch (rqt) {
+ case READ_VAR:
+ federatedReadCount.increment();
+ break;
+ case PUT_VAR:
+ federatedPutCount.increment();
+ break;
+ case GET_VAR:
+ federatedGetCount.increment();
+ break;
+ case EXEC_INST:
+ federatedExecuteInstructionCount.increment();
+ break;
+ case EXEC_UDF:
+ federatedExecuteUDFCount.increment();
+ break;
+ default:
+ break;
+ }
+ }
+
public static void startCompileTimer() {
if( DMLScript.STATISTICS )
compileStartTime = System.nanoTime();
@@ -989,6 +1019,15 @@ public class Statistics
sb.append("ParFor result merge time:\t" +
String.format("%.3f", ((double)getParforMergeTime())/1000) + " sec.\n");
sb.append("ParFor total update in-place:\t" +
lTotalUIPVar + "/" + lTotalLixUIP + "/" + lTotalLix + "\n");
}
+ if( federatedReadCount.longValue() > 0){
+ sb.append("Federated (Reads,Puts,Gets) :\t(" +
+ federatedReadCount.longValue() + "," +
+ federatedPutCount.longValue() + "," +
+ federatedGetCount.longValue() + ")\n");
+ sb.append("Federated Execute (In,UDF) :\t(" +
+
federatedExecuteInstructionCount.longValue() + "," +
+ federatedExecuteUDFCount.longValue() +
")\n");
+ }
sb.append("Total JIT compile time:\t\t" +
((double)getJITCompileTime())/1000 + " sec.\n");
sb.append("Total JVM GC count:\t\t" + getJVMgcCount() +
".\n");
