This is an automated email from the ASF dual-hosted git repository.
sanha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new a065ea0 [NEMO-28] Javadoc and Travis (#4)
a065ea0 is described below
commit a065ea0ce5ca57ff8652bacb2b62ac93f83cd5b1
Author: Won Wook SONG <won...@apache.org>
AuthorDate: Tue Mar 6 23:03:38 2018 +0900
[NEMO-28] Javadoc and Travis (#4)
JIRA: [NEMO-28](https://issues.apache.org/jira/browse/NEMO-28)
**Major changes:**
- Travis script fixed to catch errors
**Minor changes to note:**
- Some trivial fixes to make travis pass have been made to the code
- Integration test code have been reordered to look neat and consistent.
**Tests for the changes:**
- Not required
**Other comments:**
- Other PRs should be merged after this PR has been merged.
---
.travis.yml | 7 ++--
.../DuplicateEdgeGroupPropertyValue.java | 3 +-
.../java/edu/snu/nemo/common/test/ArgBuilder.java | 15 ++++++---
.../edu/snu/nemo/common/test/ExampleTestUtil.java | 5 +++
.../beam/AlternatingLeastSquareITCase.java | 25 ++++++--------
.../snu/nemo/examples/beam/BroadcastITCase.java | 17 +++++-----
.../snu/nemo/examples/beam/MapReduceITCase.java | 22 +++++++-----
.../beam/MultinomialLogisticRegressionITCase.java | 39 ++++++++--------------
.../runtime/executor/data/LimitedInputStream.java | 1 -
.../executor/data/streamchainer/Serializer.java | 2 +-
.../snu/nemo/tests/compiler/CompilerTestUtil.java | 2 +-
11 files changed, 69 insertions(+), 69 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 651caea..1a34289 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,11 +16,12 @@
# .travis.yml
# For maven builds
language: java
-after_script: "mvn verify -B"
+script: mvn verify -B -q -ff -Dsurefire.useFile=false
-Dorg.slf4j.simpleLogger.defaultLogLevel=warn
notifications:
- slack: snuspl:owgt4d5uZwww5nGovPAMgvZB
+ slack:
+ secure:
iDNSliZcKu4PW55tOe8a2jxvr1OGfqaX2sCqAmHMkAH7rvOBFwln5UBaeUY8p8BIa0P12oHLCOAtqWzHiXqe2CGVz7xyXhMHD2FzAtUIQ6GocNRXkdU5JJBXMN4XOPGMX/r38GbHRhO1ThjHr3hJIhfISjnXaOMxUoIOE6aLr/Dk9LUm6iQ2eBUajmcz4vy7BuS8Wec1DOaUIH025SRDyxzkhezJjB2JgPdLsc91amWfV04tZg+NBLZmn3DhB6Jl3dRrvABbszhBqDM2tZfV1MXuI522fzKa2tMuT2dd1BxUIkIicF+IB5tLnlKmnbRXCT7gQR0KAP5bXzdZnRY8UZo+Bbd6AlHHdIHWcNItJ2b0k4LLOVJ0MfoaKmoOJoQgkAYNPIGaDHM+q1FrhJSGtPCRKN4oaGevwWQPBZot9RY8QCU0v07p+MG9wQ8sR1nbfjF0jLQiLODSVsSNVSv2c01t6HkwmOfH
[...]
cache:
directories:
- - $HOME/.m2
+ - "$HOME/.m2"
diff --git
a/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
b/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
index 2f1f6ae..074f615 100644
---
a/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
+++
b/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
@@ -32,6 +32,7 @@ public final class DuplicateEdgeGroupPropertyValue implements
Serializable {
/**
* Constructor.
+ * @param groupId Group ID.
*/
public DuplicateEdgeGroupPropertyValue(final String groupId) {
this.isRepresentativeEdgeDecided = false;
@@ -44,7 +45,7 @@ public final class DuplicateEdgeGroupPropertyValue implements
Serializable {
* @param representativeEdgeId physical edge id of representative edge.
*/
public void setRepresentativeEdgeId(final String representativeEdgeId) {
- if (isRepresentativeEdgeDecided) {
+ if (isRepresentativeEdgeDecided &&
!this.representativeEdgeId.equals(representativeEdgeId)) {
throw new RuntimeException("edge id is already decided");
}
this.isRepresentativeEdgeDecided = true;
diff --git a/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
b/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
index 751325f..7fc6e33 100644
--- a/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
+++ b/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
@@ -19,13 +19,20 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.StringJoiner;
-import java.util.stream.Collectors;
/**
* Argument builder.
*/
public final class ArgBuilder {
- private List<List<String>> args = new ArrayList<>();
+ private List<List<String>> args;
+
+ /**
+ * Constructor with default values.
+ */
+ public ArgBuilder() {
+ this.args = new ArrayList<>();
+ this.args.add(Arrays.asList("-executor_json",
"../resources/sample_executor_resources.json"));
+ }
/**
* @param jobId job id.
@@ -78,8 +85,6 @@ public final class ArgBuilder {
* @return the built arguments.
*/
public String[] build() {
- // new String[0] is good for performance
- // see
http://stackoverflow.com/questions/4042434/converting-arrayliststring-to-string-in-java
- return
args.stream().flatMap(List::stream).collect(Collectors.toList()).toArray(new
String[0]);
+ return args.stream().flatMap(List::stream).toArray(String[]::new);
}
}
diff --git a/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
b/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
index eeec1a9..22c1127 100644
--- a/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
+++ b/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
@@ -82,6 +82,11 @@ public final class ExampleTestUtil {
* This method test the output validity of AlternatingLeastSquareITCase.
* Due to the floating point math error, the output of the test can be
different every time.
* Thus we cannot compare plain text output, but have to check its numeric
error.
+ *
+ * @param resourcePath path to resources.
+ * @param outputFileName name of output file.
+ * @param testResourceFileName name of the file to compare the outputs to.
+ * @throws IOException exception.
*/
public static void ensureALSOutputValidity(final String resourcePath,
final String outputFileName,
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
index 5585cd7..7eac5f5 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
@@ -33,8 +33,10 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class AlternatingLeastSquareITCase {
- private static final int TIMEOUT = 180000;
+ private static final int TIMEOUT = 240000;
+ private static ArgBuilder builder = new ArgBuilder();
private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String input = fileBasePath + "sample_input_als";
private static final String outputFileName = "sample_output_als";
private static final String output = fileBasePath + outputFileName;
@@ -42,33 +44,24 @@ public final class AlternatingLeastSquareITCase {
private static final String numFeatures = "10";
private static final String numIteration = "3";
private static final String lambda = "0.05";
- // If you don't want to check validity of output, make this variable false.
- private static final boolean checkOutput = true;
-
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(AlternatingLeastSquareITCase.class.getSimpleName())
- .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numIteration, lambda, output);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numIteration, lambda, output);
+ builder = new ArgBuilder();
}
@After
public void tearDown() throws Exception {
- if (checkOutput) {
- ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName,
testResourceFileName);
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
- }
+ ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName,
testResourceFileName);
+ ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
@Test (timeout = TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(AlternatingLeastSquareITCase.class.getSimpleName())
+ .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numIteration, lambda, output)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -77,6 +70,8 @@ public final class AlternatingLeastSquareITCase {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(AlternatingLeastSquareITCase.class.getSimpleName() + "_pado")
+ .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numIteration, lambda, output)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
index 5dc7a11..ec1dbc5 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
@@ -34,23 +34,18 @@ import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest(JobLauncher.class)
public final class BroadcastITCase {
private static final int TIMEOUT = 180000;
+ private static ArgBuilder builder = new ArgBuilder();
+ private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String inputFileName = "sample_input_mr";
private static final String outputFileName = "sample_output_broadcast";
private static final String testResourceFileName = "test_output_broadcast";
- private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(BroadcastITCase.class.getSimpleName())
- .addUserMain(Broadcast.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
-
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(Broadcast.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
+ builder = new ArgBuilder();
}
@After
@@ -63,6 +58,8 @@ public final class BroadcastITCase {
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName())
+ .addUserMain(Broadcast.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -71,6 +68,8 @@ public final class BroadcastITCase {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName() + "_pado")
+ .addUserMain(Broadcast.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
index 82ffeff..9ff5100 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
@@ -33,23 +33,19 @@ import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest(JobLauncher.class)
public final class MapReduceITCase {
private static final int TIMEOUT = 120000;
+ private static ArgBuilder builder = new ArgBuilder();
+ private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String inputFileName = "sample_input_mr";
private static final String outputFileName = "sample_output_mr";
private static final String testResourceFileName = "test_output_mr";
- private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(MapReduceITCase.class.getSimpleName())
- .addUserMain(MapReduce.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(MapReduce.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
+ builder = new ArgBuilder();
}
@After
@@ -62,6 +58,8 @@ public final class MapReduceITCase {
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName())
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -70,6 +68,8 @@ public final class MapReduceITCase {
public void testSailfish() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_sailfish")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(SailfishPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -78,6 +78,8 @@ public final class MapReduceITCase {
public void testDisagg() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_disagg")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DisaggregationPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -86,6 +88,8 @@ public final class MapReduceITCase {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_pado")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -98,6 +102,8 @@ public final class MapReduceITCase {
public void testDataSkew() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_dataskew")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DataSkewPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
index bf42e2f..c37cbbc 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
@@ -17,7 +17,7 @@ package edu.snu.nemo.examples.beam;
import edu.snu.nemo.client.JobLauncher;
import edu.snu.nemo.common.test.ArgBuilder;
-import edu.snu.nemo.examples.beam.policy.PadoPolicyParallelsimFive;
+import edu.snu.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -30,38 +30,27 @@ import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class MultinomialLogisticRegressionITCase {
- private static final int TIMEOUT = 180000;
+ private static final int TIMEOUT = 240000;
+ private static ArgBuilder builder = new ArgBuilder();
private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
- private static final String input = fileBasePath + "sample_input_mlr";
- private static final String numFeatures = "100";
- private static final String numClasses = "5";
- private static final String numIteration = "3";
-
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
- .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numClasses, numIteration);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numClasses, numIteration);
+ builder = new ArgBuilder();
}
-// @Test (timeout = TIMEOUT)
-// public void test() throws Exception {
-// JobLauncher.main(builder
-// .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
-//
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
-// .build());
-// }
-
@Test (timeout = TIMEOUT)
- public void testPado() throws Exception {
+ public void test() throws Exception {
+ final String input = fileBasePath + "sample_input_mlr";
+ final String numFeatures = "100";
+ final String numClasses = "5";
+ final String numIteration = "3";
+
JobLauncher.main(builder
- .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName() +
"_pado")
-
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
+ .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
+ .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numClasses, numIteration)
+ .addOptimizationPolicy(DefaultPolicy.class.getCanonicalName())
.build());
}
}
diff --git
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
index 1c40ec4..c87424f 100644
---
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
+++
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
@@ -32,7 +32,6 @@ public final class LimitedInputStream extends InputStream {
*
* @param in {@link InputStream} that should be limited.
* @param limit bytes to limit.
- * @throws IOException if failed to read stream.
*/
public LimitedInputStream(final InputStream in, final long limit) {
this.in = in;
diff --git
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
index 78c5317..d79d28f 100644
---
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
+++
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
@@ -20,7 +20,7 @@ import edu.snu.nemo.common.coder.Coder;
import java.util.List;
/**
- * class that contains {@link Coder} and {@link List< StreamChainer >}.
+ * class that contains {@link Coder} and {@link List} of {@link StreamChainer}.
* @param <T> coder element type.
*/
public final class Serializer<T> {
diff --git
a/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
b/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
index 045ccd5..1b3d1f3 100644
--- a/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
+++ b/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
@@ -61,7 +61,7 @@ public final class CompilerTestUtil {
public static DAG<IRVertex, IREdge> compileMRDAG() throws Exception {
final String input = rootDir + "/../examples/resources/sample_input_mr";
- final String output = rootDir +
"/../examples-beam/src/main/resources/sample_output";
+ final String output = rootDir + "/../examples/resources/sample_output";
final String main = "edu.snu.nemo.examples.beam.MapReduce";
final ArgBuilder mrArgBuilder = new ArgBuilder()
--
To stop receiving notification emails like this one, please contact
sa...@apache.org.
