sanha closed pull request #4: [NEMO-28] Javadoc and Travis
URL: https://github.com/apache/incubator-nemo/pull/4
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 651caea5..1a342896 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,11 +16,12 @@
# .travis.yml
# For maven builds
language: java
-after_script: "mvn verify -B"
+script: mvn verify -B -q -ff -Dsurefire.useFile=false
-Dorg.slf4j.simpleLogger.defaultLogLevel=warn
notifications:
- slack: snuspl:owgt4d5uZwww5nGovPAMgvZB
+ slack:
+ secure:
iDNSliZcKu4PW55tOe8a2jxvr1OGfqaX2sCqAmHMkAH7rvOBFwln5UBaeUY8p8BIa0P12oHLCOAtqWzHiXqe2CGVz7xyXhMHD2FzAtUIQ6GocNRXkdU5JJBXMN4XOPGMX/r38GbHRhO1ThjHr3hJIhfISjnXaOMxUoIOE6aLr/Dk9LUm6iQ2eBUajmcz4vy7BuS8Wec1DOaUIH025SRDyxzkhezJjB2JgPdLsc91amWfV04tZg+NBLZmn3DhB6Jl3dRrvABbszhBqDM2tZfV1MXuI522fzKa2tMuT2dd1BxUIkIicF+IB5tLnlKmnbRXCT7gQR0KAP5bXzdZnRY8UZo+Bbd6AlHHdIHWcNItJ2b0k4LLOVJ0MfoaKmoOJoQgkAYNPIGaDHM+q1FrhJSGtPCRKN4oaGevwWQPBZot9RY8QCU0v07p+MG9wQ8sR1nbfjF0jLQiLODSVsSNVSv2c01t6HkwmOfHe+YbTy9WkKIxQq3wMLcPT7iNcHAzsY3QQ+MSJn+xWrWcvJmfsGocVUUy20DV946NGUzpfHlXnfAxLTSOkUIj4kTXmUthIolzsRDqueoTeliIN5yeHFhQr7aX+NMrrVPba48EXRLsdQUzq6okTF6XnTDoDiLSu/AxKzItqz2lAVmpc011L6F9YIN/RQi6kK44++CvHf5kaHw=
cache:
directories:
- - $HOME/.m2
+ - "$HOME/.m2"
diff --git
a/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
b/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
index 2f1f6ae0..074f615c 100644
---
a/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
+++
b/common/src/main/java/edu/snu/nemo/common/ir/edge/executionproperty/DuplicateEdgeGroupPropertyValue.java
@@ -32,6 +32,7 @@
/**
* Constructor.
+ * @param groupId Group ID.
*/
public DuplicateEdgeGroupPropertyValue(final String groupId) {
this.isRepresentativeEdgeDecided = false;
@@ -44,7 +45,7 @@ public DuplicateEdgeGroupPropertyValue(final String groupId) {
* @param representativeEdgeId physical edge id of representative edge.
*/
public void setRepresentativeEdgeId(final String representativeEdgeId) {
- if (isRepresentativeEdgeDecided) {
+ if (isRepresentativeEdgeDecided &&
!this.representativeEdgeId.equals(representativeEdgeId)) {
throw new RuntimeException("edge id is already decided");
}
this.isRepresentativeEdgeDecided = true;
diff --git a/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
b/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
index 751325f6..7fc6e335 100644
--- a/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
+++ b/common/src/main/java/edu/snu/nemo/common/test/ArgBuilder.java
@@ -19,13 +19,20 @@
import java.util.Arrays;
import java.util.List;
import java.util.StringJoiner;
-import java.util.stream.Collectors;
/**
* Argument builder.
*/
public final class ArgBuilder {
- private List<List<String>> args = new ArrayList<>();
+ private List<List<String>> args;
+
+ /**
+ * Constructor with default values.
+ */
+ public ArgBuilder() {
+ this.args = new ArrayList<>();
+ this.args.add(Arrays.asList("-executor_json",
"../resources/sample_executor_resources.json"));
+ }
/**
* @param jobId job id.
@@ -78,8 +85,6 @@ public ArgBuilder addDAGDirectory(final String directory) {
* @return the built arguments.
*/
public String[] build() {
- // new String[0] is good for performance
- // see
http://stackoverflow.com/questions/4042434/converting-arrayliststring-to-string-in-java
- return
args.stream().flatMap(List::stream).collect(Collectors.toList()).toArray(new
String[0]);
+ return args.stream().flatMap(List::stream).toArray(String[]::new);
}
}
diff --git a/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
b/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
index eeec1a91..22c1127a 100644
--- a/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
+++ b/common/src/main/java/edu/snu/nemo/common/test/ExampleTestUtil.java
@@ -82,6 +82,11 @@ public static void ensureOutputValidity(final String
resourcePath,
* This method test the output validity of AlternatingLeastSquareITCase.
* Due to the floating point math error, the output of the test can be
different every time.
* Thus we cannot compare plain text output, but have to check its numeric
error.
+ *
+ * @param resourcePath path to resources.
+ * @param outputFileName name of output file.
+ * @param testResourceFileName name of the file to compare the outputs to.
+ * @throws IOException exception.
*/
public static void ensureALSOutputValidity(final String resourcePath,
final String outputFileName,
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
index 5585cd76..7eac5f55 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
@@ -33,8 +33,10 @@
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class AlternatingLeastSquareITCase {
- private static final int TIMEOUT = 180000;
+ private static final int TIMEOUT = 240000;
+ private static ArgBuilder builder = new ArgBuilder();
private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String input = fileBasePath + "sample_input_als";
private static final String outputFileName = "sample_output_als";
private static final String output = fileBasePath + outputFileName;
@@ -42,33 +44,24 @@
private static final String numFeatures = "10";
private static final String numIteration = "3";
private static final String lambda = "0.05";
- // If you don't want to check validity of output, make this variable false.
- private static final boolean checkOutput = true;
-
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(AlternatingLeastSquareITCase.class.getSimpleName())
- .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numIteration, lambda, output);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numIteration, lambda, output);
+ builder = new ArgBuilder();
}
@After
public void tearDown() throws Exception {
- if (checkOutput) {
- ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName,
testResourceFileName);
- ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
- }
+ ExampleTestUtil.ensureALSOutputValidity(fileBasePath, outputFileName,
testResourceFileName);
+ ExampleTestUtil.deleteOutputFile(fileBasePath, outputFileName);
}
@Test (timeout = TIMEOUT)
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(AlternatingLeastSquareITCase.class.getSimpleName())
+ .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numIteration, lambda, output)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -77,6 +70,8 @@ public void test() throws Exception {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(AlternatingLeastSquareITCase.class.getSimpleName() + "_pado")
+ .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numIteration, lambda, output)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
index 5dc7a111..ec1dbc59 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
@@ -34,23 +34,18 @@
@PrepareForTest(JobLauncher.class)
public final class BroadcastITCase {
private static final int TIMEOUT = 180000;
+ private static ArgBuilder builder = new ArgBuilder();
+ private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String inputFileName = "sample_input_mr";
private static final String outputFileName = "sample_output_broadcast";
private static final String testResourceFileName = "test_output_broadcast";
- private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(BroadcastITCase.class.getSimpleName())
- .addUserMain(Broadcast.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
-
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(Broadcast.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
+ builder = new ArgBuilder();
}
@After
@@ -63,6 +58,8 @@ public void tearDown() throws Exception {
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName())
+ .addUserMain(Broadcast.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -71,6 +68,8 @@ public void test() throws Exception {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(BroadcastITCase.class.getSimpleName() + "_pado")
+ .addUserMain(Broadcast.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
index 82ffeffc..9ff51004 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
@@ -33,23 +33,19 @@
@PrepareForTest(JobLauncher.class)
public final class MapReduceITCase {
private static final int TIMEOUT = 120000;
+ private static ArgBuilder builder = new ArgBuilder();
+ private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
+
private static final String inputFileName = "sample_input_mr";
private static final String outputFileName = "sample_output_mr";
private static final String testResourceFileName = "test_output_mr";
- private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
private static final String inputFilePath = fileBasePath + inputFileName;
private static final String outputFilePath = fileBasePath + outputFileName;
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(MapReduceITCase.class.getSimpleName())
- .addUserMain(MapReduce.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(MapReduce.class.getCanonicalName())
- .addUserArgs(inputFilePath, outputFilePath);
+ builder = new ArgBuilder();
}
@After
@@ -62,6 +58,8 @@ public void tearDown() throws Exception {
public void test() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName())
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
.build());
}
@@ -70,6 +68,8 @@ public void test() throws Exception {
public void testSailfish() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_sailfish")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(SailfishPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -78,6 +78,8 @@ public void testSailfish() throws Exception {
public void testDisagg() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_disagg")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DisaggregationPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -86,6 +88,8 @@ public void testDisagg() throws Exception {
public void testPado() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_pado")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
.build());
}
@@ -98,6 +102,8 @@ public void testPado() throws Exception {
public void testDataSkew() throws Exception {
JobLauncher.main(builder
.addJobId(MapReduceITCase.class.getSimpleName() + "_dataskew")
+ .addUserMain(MapReduce.class.getCanonicalName())
+ .addUserArgs(inputFilePath, outputFilePath)
.addOptimizationPolicy(DataSkewPolicyParallelsimFive.class.getCanonicalName())
.build());
}
diff --git
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
index bf42e2f9..c37cbbcf 100644
---
a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
+++
b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MultinomialLogisticRegressionITCase.java
@@ -17,7 +17,7 @@
import edu.snu.nemo.client.JobLauncher;
import edu.snu.nemo.common.test.ArgBuilder;
-import edu.snu.nemo.examples.beam.policy.PadoPolicyParallelsimFive;
+import edu.snu.nemo.compiler.optimizer.policy.DefaultPolicy;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -30,38 +30,27 @@
@RunWith(PowerMockRunner.class)
@PrepareForTest(JobLauncher.class)
public final class MultinomialLogisticRegressionITCase {
- private static final int TIMEOUT = 180000;
+ private static final int TIMEOUT = 240000;
+ private static ArgBuilder builder = new ArgBuilder();
private static final String fileBasePath = System.getProperty("user.dir") +
"/../resources/";
- private static final String input = fileBasePath + "sample_input_mlr";
- private static final String numFeatures = "100";
- private static final String numClasses = "5";
- private static final String numIteration = "3";
-
- private static ArgBuilder builder = new ArgBuilder()
- .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
- .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numClasses, numIteration);
@Before
public void setUp() throws Exception {
- builder = new ArgBuilder()
- .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
- .addUserArgs(input, numFeatures, numClasses, numIteration);
+ builder = new ArgBuilder();
}
-// @Test (timeout = TIMEOUT)
-// public void test() throws Exception {
-// JobLauncher.main(builder
-// .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
-//
.addOptimizationPolicy(DefaultPolicyParallelismFive.class.getCanonicalName())
-// .build());
-// }
-
@Test (timeout = TIMEOUT)
- public void testPado() throws Exception {
+ public void test() throws Exception {
+ final String input = fileBasePath + "sample_input_mlr";
+ final String numFeatures = "100";
+ final String numClasses = "5";
+ final String numIteration = "3";
+
JobLauncher.main(builder
- .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName() +
"_pado")
-
.addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
+ .addJobId(MultinomialLogisticRegressionITCase.class.getSimpleName())
+ .addUserMain(MultinomialLogisticRegression.class.getCanonicalName())
+ .addUserArgs(input, numFeatures, numClasses, numIteration)
+ .addOptimizationPolicy(DefaultPolicy.class.getCanonicalName())
.build());
}
}
diff --git
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
index 1c40ec4a..c87424f8 100644
---
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
+++
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/LimitedInputStream.java
@@ -32,7 +32,6 @@
*
* @param in {@link InputStream} that should be limited.
* @param limit bytes to limit.
- * @throws IOException if failed to read stream.
*/
public LimitedInputStream(final InputStream in, final long limit) {
this.in = in;
diff --git
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
index 78c53178..d79d28f4 100644
---
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
+++
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/data/streamchainer/Serializer.java
@@ -20,7 +20,7 @@
import java.util.List;
/**
- * class that contains {@link Coder} and {@link List< StreamChainer >}.
+ * class that contains {@link Coder} and {@link List} of {@link StreamChainer}.
* @param <T> coder element type.
*/
public final class Serializer<T> {
diff --git
a/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
b/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
index 045ccd5c..1b3d1f38 100644
--- a/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
+++ b/tests/src/test/java/edu/snu/nemo/tests/compiler/CompilerTestUtil.java
@@ -61,7 +61,7 @@
public static DAG<IRVertex, IREdge> compileMRDAG() throws Exception {
final String input = rootDir + "/../examples/resources/sample_input_mr";
- final String output = rootDir +
"/../examples-beam/src/main/resources/sample_output";
+ final String output = rootDir + "/../examples/resources/sample_output";
final String main = "edu.snu.nemo.examples.beam.MapReduce";
final ArgBuilder mrArgBuilder = new ArgBuilder()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
