sebwrede commented on a change in pull request #1113:
URL: https://github.com/apache/systemds/pull/1113#discussion_r545943709
##########
File path:
src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
##########
@@ -117,62 +115,53 @@ private void federatedParamserv(ExecMode mode) {
ExecMode platformOld = setExecMode(mode);
try {
-
- // dml name
- fullDMLScriptName = HOME + TEST_NAME + ".dml";
- // generate program args
- List<String> programArgsList = new
ArrayList<>(Arrays.asList("-stats",
- "-nvargs",
- "examples_per_worker=" + _examplesPerWorker,
- "num_features=" + numFeatures,
- "num_labels=" + numLabels,
- "epochs=" + _epochs,
- "batch_size=" + _batch_size,
- "eta=" + _eta,
- "utype=" + _utype,
- "freq=" + _freq,
- "network_type=" + _networkType,
- "channels=" + C,
- "hin=" + Hin,
- "win=" + Win));
-
- // for each worker
+ // start threads
List<Integer> ports = new ArrayList<>();
List<Thread> threads = new ArrayList<>();
for(int i = 0; i < _numFederatedWorkers; i++) {
- // write row partitioned features to disk
- writeInputMatrixWithMTD("X" + i,
-
generateDummyMNISTFeatures(_examplesPerWorker, C, Hin, Win),
- false,
- new
MatrixCharacteristics(_examplesPerWorker, numFeatures, _blocksize,
- _examplesPerWorker *
numFeatures));
- // write row partitioned labels to disk
- writeInputMatrixWithMTD("y" + i,
-
generateDummyMNISTLabels(_examplesPerWorker, numLabels),
- false,
- new
MatrixCharacteristics(_examplesPerWorker, numLabels, _blocksize,
- _examplesPerWorker *
numLabels));
-
- // start worker
ports.add(getRandomAvailablePort());
threads.add(startLocalFedWorkerThread(ports.get(i), FED_WORKER_WAIT_S));
-
- // add worker to program args
- programArgsList.add("X" + i + "=" +
TestUtils.federatedAddress(ports.get(i), input("X" + i)));
- programArgsList.add("y" + i + "=" +
TestUtils.federatedAddress(ports.get(i), input("y" + i)));
}
+
+ double[][] features =
generateDummyMNISTFeatures(_dataSetSize, C, Hin, Win);
+ double[][] labels =
generateDummyMNISTLabels(_dataSetSize, numLabels);
+ String featuresName = "X_" + _numFederatedWorkers;
+ String labelsName = "y_" + _numFederatedWorkers;
+
+
federateBalancedAndWriteInputMatrixWithMTD(featuresName, features,
_numFederatedWorkers, ports);
+ federateBalancedAndWriteInputMatrixWithMTD(labelsName,
labels, _numFederatedWorkers, ports);
+
try {
- Thread.sleep(1000);
+ Thread.sleep(2000);
Review comment:
Why does it need to sleep before continuing?
##########
File path:
src/test/java/org/apache/sysds/test/functions/federated/paramserv/FederatedParamservTest.java
##########
@@ -117,62 +115,53 @@ private void federatedParamserv(ExecMode mode) {
ExecMode platformOld = setExecMode(mode);
try {
-
- // dml name
- fullDMLScriptName = HOME + TEST_NAME + ".dml";
- // generate program args
- List<String> programArgsList = new
ArrayList<>(Arrays.asList("-stats",
- "-nvargs",
- "examples_per_worker=" + _examplesPerWorker,
- "num_features=" + numFeatures,
- "num_labels=" + numLabels,
- "epochs=" + _epochs,
- "batch_size=" + _batch_size,
- "eta=" + _eta,
- "utype=" + _utype,
- "freq=" + _freq,
- "network_type=" + _networkType,
- "channels=" + C,
- "hin=" + Hin,
- "win=" + Win));
-
- // for each worker
+ // start threads
List<Integer> ports = new ArrayList<>();
List<Thread> threads = new ArrayList<>();
for(int i = 0; i < _numFederatedWorkers; i++) {
- // write row partitioned features to disk
- writeInputMatrixWithMTD("X" + i,
-
generateDummyMNISTFeatures(_examplesPerWorker, C, Hin, Win),
- false,
- new
MatrixCharacteristics(_examplesPerWorker, numFeatures, _blocksize,
- _examplesPerWorker *
numFeatures));
- // write row partitioned labels to disk
- writeInputMatrixWithMTD("y" + i,
-
generateDummyMNISTLabels(_examplesPerWorker, numLabels),
- false,
- new
MatrixCharacteristics(_examplesPerWorker, numLabels, _blocksize,
- _examplesPerWorker *
numLabels));
-
- // start worker
ports.add(getRandomAvailablePort());
threads.add(startLocalFedWorkerThread(ports.get(i), FED_WORKER_WAIT_S));
-
- // add worker to program args
- programArgsList.add("X" + i + "=" +
TestUtils.federatedAddress(ports.get(i), input("X" + i)));
- programArgsList.add("y" + i + "=" +
TestUtils.federatedAddress(ports.get(i), input("y" + i)));
}
+
+ double[][] features =
generateDummyMNISTFeatures(_dataSetSize, C, Hin, Win);
+ double[][] labels =
generateDummyMNISTLabels(_dataSetSize, numLabels);
+ String featuresName = "X_" + _numFederatedWorkers;
+ String labelsName = "y_" + _numFederatedWorkers;
+
+
federateBalancedAndWriteInputMatrixWithMTD(featuresName, features,
_numFederatedWorkers, ports);
+ federateBalancedAndWriteInputMatrixWithMTD(labelsName,
labels, _numFederatedWorkers, ports);
+
try {
- Thread.sleep(1000);
+ Thread.sleep(2000);
}
catch(InterruptedException e) {
e.printStackTrace();
}
-
+
+ // dml name
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ // generate program args
+ List<String> programArgsList = new
ArrayList<>(Arrays.asList("-stats",
+ "-nvargs",
+ "features=" + input(featuresName),
+ "labels=" + input(labelsName),
+ "epochs=" + _epochs,
+ "batch_size=" + _batch_size,
+ "eta=" + _eta,
+ "utype=" + _utype,
+ "freq=" + _freq,
+ "scheme=" + _scheme,
+ "network_type=" + _networkType,
+ "channels=" + C,
+ "hin=" + Hin,
+ "win=" + Win,
+ "seed=" + 25));
+
programArgs = programArgsList.toArray(new String[0]);
LOG.debug(runTest(null));
Assert.assertEquals(0,
Statistics.getNoOfExecutedSPInst());
Review comment:
It could make sense to do some assertion on the output of the execution
to make sure that it executed correctly.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
