Repository: reef Updated Branches: refs/heads/master a1ef9374d -> 8ef1b8361
[REEF-1709] Make unit tests in reef-runtime-yarn compile and work again Move unit tests into a proper directory in maven directory structure and fix checkstyle errors (indentation, long lines). JIRA: [REEF-1709](https://issues.apache.org/jira/browse/REEF-1709) Pull request: This closes #1223 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/8ef1b836 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/8ef1b836 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/8ef1b836 Branch: refs/heads/master Commit: 8ef1b836109450434391fc788c6fb2db74d51736 Parents: a1ef937 Author: Sergiy Matusevych <[email protected]> Authored: Wed Jan 11 18:33:52 2017 -0800 Committer: Mariia Mykhailova <[email protected]> Committed: Thu Jan 12 17:40:28 2017 -0800 ---------------------------------------------------------------------- .../runtime/yarn/driver/TestTrackingUri.java | 116 ---------------- .../driver/YarnResourceRequestHandlerTest.java | 126 ----------------- .../runtime/yarn/driver/TestTrackingUri.java | 116 ++++++++++++++++ .../driver/YarnResourceRequestHandlerTest.java | 135 +++++++++++++++++++ .../reef/runtime/yarn/driver/package-info.java | 22 +++ 5 files changed, 273 insertions(+), 242 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/8ef1b836/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java deleted file mode 100644 index c97c02c..0000000 --- a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.runtime.yarn.driver; - -import org.apache.reef.tang.Injector; -import org.apache.reef.tang.JavaConfigurationBuilder; -import org.apache.reef.tang.Tang; -import org.apache.reef.tang.exceptions.BindException; -import org.apache.reef.tang.exceptions.InjectionException; -import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeBegin; -import org.apache.reef.webserver.HttpHandlerConfiguration; -import org.apache.reef.webserver.HttpServer; -import org.apache.reef.webserver.HttpServerImpl; -import org.junit.Assert; -import org.junit.Test; - -import java.net.UnknownHostException; - -/** - * Tracking Uri test. - */ -public class TestTrackingUri { - /** - * Get Default Tracking URI. - * - * @throws InjectionException - * @throws UnknownHostException - */ - @Test - public void testDefaultTrackingUri() throws InjectionException, UnknownHostException { - final String uri = Tang.Factory.getTang().newInjector().getInstance(TrackingURLProvider.class).getTrackingUrl(); - Assert.assertEquals(uri, ""); - } - - /** - * Get Tracking URI with specified port number and HttpTrackingURLProvider. - * - * @throws InjectionException - * @throws UnknownHostException - * @throws BindException - */ - @Test - public void testHttpTrackingUri() throws InjectionException, UnknownHostException, BindException { - final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder() - .bindNamedParameter(TcpPortRangeBegin.class, "8888") - .bindImplementation(TrackingURLProvider.class, HttpTrackingURLProvider.class) - .bindImplementation(HttpServer.class, HttpServerImpl.class); - - final Injector injector = Tang.Factory.getTang().newInjector(cb.build()); - final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); - final int port = injector.getInstance(HttpServer.class).getPort(); - verifyUri(uri, port); - } - - /** - * Get Tracking URI with HttpTrackingURLProvider and defualt port number. - * - * @throws InjectionException - * @throws UnknownHostException - * @throws BindException - */ - @Test - public void testHttpTrackingUriDefaultPort() throws InjectionException, UnknownHostException, BindException { - final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder() - .bindImplementation(HttpServer.class, HttpServerImpl.class) - .bindImplementation(TrackingURLProvider.class, HttpTrackingURLProvider.class); - - final Injector injector = Tang.Factory.getTang().newInjector(cb.build()); - final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); - final int port = injector.getInstance(HttpServer.class).getPort(); - verifyUri(uri, port); - } - - /** - * Http Tracking URI using default binding test. - * - * @throws InjectionException - * @throws UnknownHostException - * @throws BindException - */ - @Test - public void testHttpTrackingUriDefaultBinding() throws InjectionException, UnknownHostException, BindException { - final Injector injector = Tang.Factory.getTang().newInjector(HttpHandlerConfiguration.CONF.build()); - final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); - final int port = injector.getInstance(HttpServer.class).getPort(); - verifyUri(uri, port); - } - - /** - * Verify if URI is correct. - * - * @param uri - * @param port - */ - private void verifyUri(final String uri, final int port) { - final String[] parts = uri.split(":"); - Assert.assertTrue(parts.length == 2); - Assert.assertEquals(port, Integer.parseInt(parts[1])); - } -} http://git-wip-us.apache.org/repos/asf/reef/blob/8ef1b836/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java deleted file mode 100644 index a7a9d59..0000000 --- a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.runtime.yarn.driver; - -import org.apache.reef.driver.catalog.ResourceCatalog; -import org.apache.reef.driver.evaluator.EvaluatorRequest; -import org.apache.reef.driver.evaluator.EvaluatorRequestor; -import org.apache.reef.runtime.common.driver.EvaluatorRequestorImpl; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.reef.tang.Tang; -import org.apache.reef.tang.exceptions.InjectionException; -import org.apache.reef.util.logging.LoggingScopeFactory; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -/** - * Tests for YarnResourceRequestHandler. - */ -public final class YarnResourceRequestHandlerTest { - private final ApplicationMasterRegistration applicationMasterRegistration = new ApplicationMasterRegistration(); - private final MockContainerRequestHandler containerRequestHandler = new MockContainerRequestHandler(); - private final YarnResourceRequestHandler resourceRequestHandler = new YarnResourceRequestHandler(containerRequestHandler, applicationMasterRegistration); - private final ResourceCatalog resourceCatalog = Mockito.mock(ResourceCatalog.class); - - private class MockContainerRequestHandler implements YarnContainerRequestHandler { - private AMRMClient.ContainerRequest[] requests; - - @Override - public void onContainerRequest(AMRMClient.ContainerRequest... containerRequests) { - this.requests = containerRequests; - } - - public AMRMClient.ContainerRequest[] getRequests() { - return requests; - } - } - - /** - * Tests whether the amount of memory is transferred correctly. - */ - @Test - public void testDifferentMemory() throws InjectionException { - final LoggingScopeFactory loggingScopeFactory = Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class); - final EvaluatorRequestor evaluatorRequestor = new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory); - - final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder() - .setNumber(1) - .setMemory(64) - .setNumberOfCores(1) - .build(); - final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder() - .setNumber(1) - .setMemory(128) - .setNumberOfCores(2) - .build(); - - evaluatorRequestor.submit(requestOne); - Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory", - requestOne.getMegaBytes(), - containerRequestHandler.getRequests()[0].getCapability().getMemory() - ); - - evaluatorRequestor.submit(requestTwo); - Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory", - requestTwo.getMegaBytes(), - containerRequestHandler.getRequests()[0].getCapability().getMemory() - ); - - evaluatorRequestor.submit(requestOne); - Assert.assertNotEquals("Request in REEF and YARN form should have the same amount of memory", - requestTwo.getMegaBytes(), - containerRequestHandler.getRequests()[0].getCapability().getMemory() - ); - } - - @Test - public void testEvaluatorCount() throws InjectionException { - final LoggingScopeFactory loggingScopeFactory = Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class); - final EvaluatorRequestor evaluatorRequestor = new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory); - final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder() - .setNumber(1) - .setMemory(64) - .setNumberOfCores(1) - .build(); - final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder() - .setNumber(2) - .setMemory(128) - .setNumberOfCores(2) - .build(); - - evaluatorRequestor.submit(requestOne); - Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators", - requestOne.getNumber(), - containerRequestHandler.getRequests().length - ); - - evaluatorRequestor.submit(requestTwo); - Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators", - requestTwo.getNumber(), - containerRequestHandler.getRequests().length - ); - - evaluatorRequestor.submit(requestTwo); - Assert.assertNotEquals("Request in REEF and YARN form should have the same number of Evaluators", - requestOne.getNumber(), - containerRequestHandler.getRequests().length - ); - } -} http://git-wip-us.apache.org/repos/asf/reef/blob/8ef1b836/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java new file mode 100644 index 0000000..c97c02c --- /dev/null +++ b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.yarn.driver; + +import org.apache.reef.tang.Injector; +import org.apache.reef.tang.JavaConfigurationBuilder; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.BindException; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeBegin; +import org.apache.reef.webserver.HttpHandlerConfiguration; +import org.apache.reef.webserver.HttpServer; +import org.apache.reef.webserver.HttpServerImpl; +import org.junit.Assert; +import org.junit.Test; + +import java.net.UnknownHostException; + +/** + * Tracking Uri test. + */ +public class TestTrackingUri { + /** + * Get Default Tracking URI. + * + * @throws InjectionException + * @throws UnknownHostException + */ + @Test + public void testDefaultTrackingUri() throws InjectionException, UnknownHostException { + final String uri = Tang.Factory.getTang().newInjector().getInstance(TrackingURLProvider.class).getTrackingUrl(); + Assert.assertEquals(uri, ""); + } + + /** + * Get Tracking URI with specified port number and HttpTrackingURLProvider. + * + * @throws InjectionException + * @throws UnknownHostException + * @throws BindException + */ + @Test + public void testHttpTrackingUri() throws InjectionException, UnknownHostException, BindException { + final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder() + .bindNamedParameter(TcpPortRangeBegin.class, "8888") + .bindImplementation(TrackingURLProvider.class, HttpTrackingURLProvider.class) + .bindImplementation(HttpServer.class, HttpServerImpl.class); + + final Injector injector = Tang.Factory.getTang().newInjector(cb.build()); + final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); + final int port = injector.getInstance(HttpServer.class).getPort(); + verifyUri(uri, port); + } + + /** + * Get Tracking URI with HttpTrackingURLProvider and defualt port number. + * + * @throws InjectionException + * @throws UnknownHostException + * @throws BindException + */ + @Test + public void testHttpTrackingUriDefaultPort() throws InjectionException, UnknownHostException, BindException { + final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder() + .bindImplementation(HttpServer.class, HttpServerImpl.class) + .bindImplementation(TrackingURLProvider.class, HttpTrackingURLProvider.class); + + final Injector injector = Tang.Factory.getTang().newInjector(cb.build()); + final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); + final int port = injector.getInstance(HttpServer.class).getPort(); + verifyUri(uri, port); + } + + /** + * Http Tracking URI using default binding test. + * + * @throws InjectionException + * @throws UnknownHostException + * @throws BindException + */ + @Test + public void testHttpTrackingUriDefaultBinding() throws InjectionException, UnknownHostException, BindException { + final Injector injector = Tang.Factory.getTang().newInjector(HttpHandlerConfiguration.CONF.build()); + final String uri = injector.getInstance(TrackingURLProvider.class).getTrackingUrl(); + final int port = injector.getInstance(HttpServer.class).getPort(); + verifyUri(uri, port); + } + + /** + * Verify if URI is correct. + * + * @param uri + * @param port + */ + private void verifyUri(final String uri, final int port) { + final String[] parts = uri.split(":"); + Assert.assertTrue(parts.length == 2); + Assert.assertEquals(port, Integer.parseInt(parts[1])); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/8ef1b836/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java new file mode 100644 index 0000000..73f5c2c --- /dev/null +++ b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.yarn.driver; + +import org.apache.reef.driver.catalog.ResourceCatalog; +import org.apache.reef.driver.evaluator.EvaluatorRequest; +import org.apache.reef.driver.evaluator.EvaluatorRequestor; +import org.apache.reef.runtime.common.driver.EvaluatorRequestorImpl; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.util.logging.LoggingScopeFactory; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Tests for YarnResourceRequestHandler. + */ +public final class YarnResourceRequestHandlerTest { + + private final ApplicationMasterRegistration applicationMasterRegistration = new ApplicationMasterRegistration(); + private final MockContainerRequestHandler containerRequestHandler = new MockContainerRequestHandler(); + + private final ResourceCatalog resourceCatalog = Mockito.mock(ResourceCatalog.class); + + private final YarnResourceRequestHandler resourceRequestHandler = + new YarnResourceRequestHandler(containerRequestHandler, applicationMasterRegistration); + + private class MockContainerRequestHandler implements YarnContainerRequestHandler { + private AMRMClient.ContainerRequest[] requests; + + @Override + public void onContainerRequest(final AMRMClient.ContainerRequest... containerRequests) { + this.requests = containerRequests; + } + + AMRMClient.ContainerRequest[] getRequests() { + return requests; + } + } + + /** + * Tests whether the amount of memory is transferred correctly. + */ + @Test + public void testDifferentMemory() throws InjectionException { + + final LoggingScopeFactory loggingScopeFactory = + Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class); + + final EvaluatorRequestor evaluatorRequestor = + new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory); + + final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder() + .setNumber(1) + .setMemory(64) + .setNumberOfCores(1) + .build(); + + final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder() + .setNumber(1) + .setMemory(128) + .setNumberOfCores(2) + .build(); + + evaluatorRequestor.submit(requestOne); + Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory", + requestOne.getMegaBytes(), + containerRequestHandler.getRequests()[0].getCapability().getMemory()); + + evaluatorRequestor.submit(requestTwo); + Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory", + requestTwo.getMegaBytes(), + containerRequestHandler.getRequests()[0].getCapability().getMemory()); + + evaluatorRequestor.submit(requestOne); + Assert.assertNotEquals("Second YARN request should have different amount of memory", + requestTwo.getMegaBytes(), + containerRequestHandler.getRequests()[0].getCapability().getMemory()); + } + + @Test + public void testEvaluatorCount() throws InjectionException { + + final LoggingScopeFactory loggingScopeFactory = + Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class); + + final EvaluatorRequestor evaluatorRequestor = + new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory); + + final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder() + .setNumber(1) + .setMemory(64) + .setNumberOfCores(1) + .build(); + + final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder() + .setNumber(2) + .setMemory(128) + .setNumberOfCores(2) + .build(); + + evaluatorRequestor.submit(requestOne); + Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators", + requestOne.getNumber(), + containerRequestHandler.getRequests().length); + + evaluatorRequestor.submit(requestTwo); + Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators", + requestTwo.getNumber(), + containerRequestHandler.getRequests().length); + + evaluatorRequestor.submit(requestTwo); + Assert.assertNotEquals("Second YARN request should have different number of Evaluators", + requestOne.getNumber(), + containerRequestHandler.getRequests().length); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/8ef1b836/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/package-info.java b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/package-info.java new file mode 100644 index 0000000..84ea07d --- /dev/null +++ b/lang/java/reef-runtime-yarn/src/test/java/org/apache/reef/runtime/yarn/driver/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Unit tests for REEF Driver running under YARN runtime. + */ +package org.apache.reef.runtime.yarn.driver;
