Repository: reef Updated Branches: refs/heads/master 266918a1d -> d8fe048d1
[REEF-983] Add Runtime Identifier to the EvaluatorDescriptor This addressed the issue by: * Adding RuntimeName to the EvaluatorDescriptor * Populating this field in different runtimes with appropriate values * Serializing RuntimeName for the bridge invocation * Serializing RuntimeName for HttpEndpoint JIRA: [REEF-983](https://issues.apache.org/jira/browse/REEF-983) Pull Request: Closes #704 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/d8fe048d Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/d8fe048d Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/d8fe048d Branch: refs/heads/master Commit: d8fe048d1b8200fd13dc89ff77042e4bbcb33ab5 Parents: 266918a Author: Boris Shulman <[email protected]> Authored: Thu Dec 3 00:45:37 2015 -0800 Committer: Andrew Chung <[email protected]> Committed: Sat Dec 5 19:11:41 2015 -0800 ---------------------------------------------------------------------- .../org/apache/reef/javabridge/Utilities.java | 2 +- .../driver/evaluator/EvaluatorDescriptor.java | 5 ++ .../evaluator/EvaluatorDescriptorImpl.java | 10 ++- .../evaluator/EvaluatorManagerFactory.java | 7 +- .../driver/resourcemanager/ResourceEvent.java | 4 + .../resourcemanager/ResourceEventImpl.java | 16 ++++ .../resourcemanager/ResourceStatusEvent.java | 5 ++ .../ResourceStatusEventImpl.java | 15 ++++ .../runtime/local/driver/ResourceManager.java | 2 +- .../runtime/local/driver/RuntimeIdentifier.java | 31 ++++++++ .../runtime/mesos/driver/RuntimeIdentifier.java | 31 ++++++++ .../runtime/yarn/driver/RuntimeIdentifier.java | 31 ++++++++ .../yarn/driver/YarnContainerManager.java | 1 + .../driver/YarnDriverRuntimeRestartManager.java | 3 +- .../driver/RuntimeNameTestConfiguration.java | 38 ++++++++++ .../org/apache/reef/tests/AllTestsSuite.java | 4 +- .../apache/reef/tests/LocalTestEnvironment.java | 6 ++ .../apache/reef/tests/MesosTestEnvironment.java | 5 ++ .../org/apache/reef/tests/TestEnvironment.java | 5 ++ .../apache/reef/tests/YarnTestEnvironment.java | 5 ++ .../tests/runtimename/RuntimeNameDriver.java | 63 ++++++++++++++++ .../reef/tests/runtimename/RuntimeNameTest.java | 77 ++++++++++++++++++++ .../reef/tests/runtimename/package-info.java | 22 ++++++ .../src/main/avro/EvaluatorInfo.avsc | 3 +- .../webserver/AvroEvaluatorInfoSerializer.java | 3 + .../webserver/HttpServerReefEventHandler.java | 2 + .../webserver/TestAvroSerializerForHttp.java | 7 +- .../webserver/TestReefEventStateManager.java | 5 ++ 28 files changed, 398 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/Utilities.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/Utilities.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/Utilities.java index 1bfda31..b50131c 100644 --- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/Utilities.java +++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/Utilities.java @@ -63,7 +63,7 @@ public final class Utilities { final InetSocketAddress socketAddress = evaluatorDescriptor.getNodeDescriptor().getInetSocketAddress(); return "IP=" + socketAddress.getAddress() + ", Port=" + socketAddress.getPort() + ", HostName=" + socketAddress.getHostName() + ", Memory=" + evaluatorDescriptor.getMemory() + ", Core=" + - evaluatorDescriptor.getNumberOfCores(); + evaluatorDescriptor.getNumberOfCores() + ", RuntimeName=" + evaluatorDescriptor.getRuntimeName(); } /** http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorDescriptor.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorDescriptor.java b/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorDescriptor.java index 44c2439..5a045c4 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorDescriptor.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorDescriptor.java @@ -44,4 +44,9 @@ public interface EvaluatorDescriptor { * @return the number of virtual core allocated to this Evaluator. */ int getNumberOfCores(); + + /** + * @return name of the runtime that was used to allocate this Evaluator + */ + String getRuntimeName(); } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorDescriptorImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorDescriptorImpl.java index ed9fefc..f52cc7f 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorDescriptorImpl.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorDescriptorImpl.java @@ -35,15 +35,18 @@ final class EvaluatorDescriptorImpl implements EvaluatorDescriptor { private final int megaBytes; private final int numberOfCores; private EvaluatorProcess process; + private final String runtimeName; EvaluatorDescriptorImpl(final NodeDescriptor nodeDescriptor, final int megaBytes, final int numberOfCores, - final EvaluatorProcess process) { + final EvaluatorProcess process, + final String runtimeName) { this.nodeDescriptor = nodeDescriptor; this.megaBytes = megaBytes; this.numberOfCores = numberOfCores; this.process = process; + this.runtimeName = runtimeName; } @Override @@ -72,4 +75,9 @@ final class EvaluatorDescriptorImpl implements EvaluatorDescriptor { public int getNumberOfCores() { return this.numberOfCores; } + + @Override + public String getRuntimeName() { + return this.runtimeName; + } } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManagerFactory.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManagerFactory.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManagerFactory.java index 12a6455..bd3d691 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManagerFactory.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/evaluator/EvaluatorManagerFactory.java @@ -35,7 +35,7 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * Helper class that creates new EvaluatorManager instances from alloations. + * Helper class that creates new EvaluatorManager instances from allocations. */ @Private @DriverSide @@ -74,7 +74,7 @@ public final class EvaluatorManagerFactory { } final EvaluatorDescriptorImpl evaluatorDescriptor = new EvaluatorDescriptorImpl(nodeDescriptor, resourceEvent.getResourceMemory(), resourceEvent.getVirtualCores().get(), - processFactory.newEvaluatorProcess()); + processFactory.newEvaluatorProcess(), resourceEvent.getRuntimeName()); LOG.log(Level.FINEST, "Resource allocation: new evaluator id[{0}]", resourceEvent.getIdentifier()); final EvaluatorManager evaluatorManager = @@ -134,7 +134,8 @@ public final class EvaluatorManagerFactory { public EvaluatorManager getNewEvaluatorManagerForEvaluatorFailedDuringDriverRestart( final ResourceStatusEvent resourceStatusEvent) { return getNewEvaluatorManagerInstance(resourceStatusEvent.getIdentifier(), - new EvaluatorDescriptorImpl(null, 128, 1, processFactory.newEvaluatorProcess())); + new EvaluatorDescriptorImpl(null, 128, 1, processFactory.newEvaluatorProcess(), + resourceStatusEvent.getRuntimeName())); } /** http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEvent.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEvent.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEvent.java index baae87e..0922e57 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEvent.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEvent.java @@ -54,4 +54,8 @@ public interface ResourceEvent { */ Optional<String> getRackName(); + /** + * @return Runtime name of the resource + */ + String getRuntimeName(); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEventImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEventImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEventImpl.java index 8d43be6..0f1757e 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEventImpl.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceEventImpl.java @@ -32,6 +32,7 @@ public final class ResourceEventImpl implements ResourceAllocationEvent, Resourc private final String nodeId; private final Optional<Integer> virtualCores; private final Optional<String> rackName; + private final String runtimeName; private ResourceEventImpl(final Builder builder) { @@ -40,6 +41,7 @@ public final class ResourceEventImpl implements ResourceAllocationEvent, Resourc this.nodeId = builder.recovery ? builder.nodeId : BuilderUtils.notNull(builder.nodeId); this.virtualCores = Optional.ofNullable(builder.virtualCores); this.rackName = Optional.ofNullable(builder.rackName); + this.runtimeName = BuilderUtils.notNull(builder.runtimeName); } @Override @@ -67,6 +69,11 @@ public final class ResourceEventImpl implements ResourceAllocationEvent, Resourc return rackName; } + @Override + public String getRuntimeName() { + return runtimeName; + } + public static Builder newAllocationBuilder() { return new Builder(false); } @@ -86,6 +93,7 @@ public final class ResourceEventImpl implements ResourceAllocationEvent, Resourc private String nodeId; private Integer virtualCores; private String rackName; + private String runtimeName; private Builder(final boolean recovery){ this.recovery = recovery; @@ -131,6 +139,14 @@ public final class ResourceEventImpl implements ResourceAllocationEvent, Resourc return this; } + /** + * @see ResourceAllocationEvent#getRuntimeName() + */ + public Builder setRuntimeName(final String runtimeName) { + this.runtimeName = runtimeName; + return this; + } + @Override public ResourceEventImpl build() { return new ResourceEventImpl(this); http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEvent.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEvent.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEvent.java index 089ea7c..c669d95 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEvent.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEvent.java @@ -38,6 +38,11 @@ public interface ResourceStatusEvent { String getIdentifier(); /** + * @return Runtime name + */ + String getRuntimeName(); + + /** * @return State of the resource */ ReefServiceProtos.State getState(); http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEventImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEventImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEventImpl.java index 5db541c..ee01879 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEventImpl.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/resourcemanager/ResourceStatusEventImpl.java @@ -31,12 +31,14 @@ public final class ResourceStatusEventImpl implements ResourceStatusEvent { private final ReefServiceProtos.State state; private final Optional<String> diagnostics; private final Optional<Integer> exitCode; + private final String runtimeName; private ResourceStatusEventImpl(final Builder builder) { this.identifier = BuilderUtils.notNull(builder.identifier); this.state = BuilderUtils.notNull(builder.state); this.diagnostics = Optional.ofNullable(builder.diagnostics); this.exitCode = Optional.ofNullable(builder.exitCode); + this.runtimeName = BuilderUtils.notNull(builder.identifier); } @Override @@ -45,6 +47,11 @@ public final class ResourceStatusEventImpl implements ResourceStatusEvent { } @Override + public String getRuntimeName() { + return runtimeName; + } + + @Override public ReefServiceProtos.State getState() { return state; } @@ -68,6 +75,7 @@ public final class ResourceStatusEventImpl implements ResourceStatusEvent { */ public static final class Builder implements org.apache.reef.util.Builder<ResourceStatusEvent> { private String identifier; + private String runtimeName; private ReefServiceProtos.State state; private String diagnostics; private Integer exitCode; @@ -81,6 +89,13 @@ public final class ResourceStatusEventImpl implements ResourceStatusEvent { } /** + * @see ResourceStatusEvent#getIdentifier() + */ + public Builder setRuntimeName(final String runtimeName) { + this.runtimeName = runtimeName; + return this; + } + /** * @see ResourceStatusEvent#getState() */ public Builder setState(final ReefServiceProtos.State state) { http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/ResourceManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/ResourceManager.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/ResourceManager.java index fb51de0..76d107a 100644 --- a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/ResourceManager.java +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/ResourceManager.java @@ -220,7 +220,7 @@ public final class ResourceManager { final ResourceAllocationEvent alloc = ResourceEventImpl.newAllocationBuilder() .setIdentifier(container.getContainerID()).setNodeId(container.getNodeID()) .setResourceMemory(container.getMemory()).setVirtualCores(container.getNumberOfCores()) - .setRackName(container.getRackName()).build(); + .setRackName(container.getRackName()).setRuntimeName(RuntimeIdentifier.RUNTIME_NAME).build(); LOG.log(Level.FINEST, "Allocating container: {0}", container); this.allocationHandler.onNext(alloc); http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/RuntimeIdentifier.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/RuntimeIdentifier.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/RuntimeIdentifier.java new file mode 100644 index 0000000..1d0456f --- /dev/null +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/driver/RuntimeIdentifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.local.driver; + +import org.apache.reef.annotations.audience.Private; + +/** + * Runtime Identifier Implementation. + */ +@Private +public final class RuntimeIdentifier { + public static final String RUNTIME_NAME = "Local"; + + private RuntimeIdentifier() { } +} http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/RuntimeIdentifier.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/RuntimeIdentifier.java b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/RuntimeIdentifier.java new file mode 100644 index 0000000..af82b64 --- /dev/null +++ b/lang/java/reef-runtime-mesos/src/main/java/org/apache/reef/runtime/mesos/driver/RuntimeIdentifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.mesos.driver; + +import org.apache.reef.annotations.audience.Private; + +/** + * Runtime Identifier Implementation. + */ +@Private +public final class RuntimeIdentifier { + public static final String RUNTIME_NAME = "Mesos"; + + private RuntimeIdentifier() { } +} http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/RuntimeIdentifier.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/RuntimeIdentifier.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/RuntimeIdentifier.java new file mode 100644 index 0000000..4ba8545 --- /dev/null +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/RuntimeIdentifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.yarn.driver; + +import org.apache.reef.annotations.audience.Private; + +/** + * Runtime Identifier Implementation. + */ +@Private +public final class RuntimeIdentifier { + public static final String RUNTIME_NAME = "Yarn"; + + private RuntimeIdentifier() { } +} http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java index a056983..a069367 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java @@ -445,6 +445,7 @@ final class YarnContainerManager .setResourceMemory(container.getResource().getMemory()) .setVirtualCores(container.getResource().getVirtualCores()) .setRackName(rackNameFormatter.getRackName(container)) + .setRuntimeName(RuntimeIdentifier.RUNTIME_NAME) .build()); this.updateRuntimeStatus(); } else { http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java index 713b132..d2a00f4 100644 --- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java +++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java @@ -234,7 +234,8 @@ public final class YarnDriverRuntimeRestartManager implements DriverRuntimeResta ResourceEventImpl.newRecoveryBuilder().setIdentifier(container.getId().toString()) .setNodeId(container.getNodeId().toString()).setRackName(rackNameFormatter.getRackName(container)) .setResourceMemory(container.getResource().getMemory()) - .setVirtualCores(container.getResource().getVirtualCores()).build())); + .setVirtualCores(container.getResource().getVirtualCores()) + .setRuntimeName(RuntimeIdentifier.RUNTIME_NAME).build())); } } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/main/java/org/apache/reef/tests/driver/RuntimeNameTestConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/main/java/org/apache/reef/tests/driver/RuntimeNameTestConfiguration.java b/lang/java/reef-tests/src/main/java/org/apache/reef/tests/driver/RuntimeNameTestConfiguration.java new file mode 100644 index 0000000..127b54a --- /dev/null +++ b/lang/java/reef-tests/src/main/java/org/apache/reef/tests/driver/RuntimeNameTestConfiguration.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.tests.driver; + +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; +import org.apache.reef.tang.formats.ConfigurationModule; +import org.apache.reef.tang.formats.ConfigurationModuleBuilder; +import org.apache.reef.tang.formats.RequiredParameter; + +public final class RuntimeNameTestConfiguration extends ConfigurationModuleBuilder { + + public static final RequiredParameter<String> RUNTIME_NAME = new RequiredParameter<>(); + public static final ConfigurationModule CONF = new RuntimeNameTestConfiguration() + .bindNamedParameter(RuntimeName.class, RUNTIME_NAME) + .build(); + + @NamedParameter(doc = "The runtime name") + public static class RuntimeName implements Name<String> { + } + +} http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AllTestsSuite.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AllTestsSuite.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AllTestsSuite.java index 82d1829..baa2c74 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AllTestsSuite.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/AllTestsSuite.java @@ -22,6 +22,7 @@ import org.apache.reef.tests.applications.ApplicationTestSuite; import org.apache.reef.tests.close_eval.CloseEvaluatorTest; import org.apache.reef.tests.configurationproviders.ConfigurationProviderTest; import org.apache.reef.tests.driver.DriverTest; +import org.apache.reef.tests.runtimename.RuntimeNameTest; import org.apache.reef.tests.evaluatorfailure.EvaluatorFailureTest; import org.apache.reef.tests.evaluatorreuse.EvaluatorReuseTest; import org.apache.reef.tests.evaluatorsize.EvaluatorSizeTest; @@ -52,7 +53,8 @@ import org.junit.runners.Suite; EvaluatorFailureTest.class, ExamplesTestSuite.class, ConfigurationProviderTest.class, - ApplicationTestSuite.class + ApplicationTestSuite.class, + RuntimeNameTest.class }) public final class AllTestsSuite { } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/LocalTestEnvironment.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/LocalTestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/LocalTestEnvironment.java index 3ac2e8c..775b7b1 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/LocalTestEnvironment.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/LocalTestEnvironment.java @@ -22,6 +22,7 @@ import org.apache.reef.io.ConfigurableDirectoryTempFileCreator; import org.apache.reef.io.TempFileCreator; import org.apache.reef.io.parameters.TempFileRootFolder; import org.apache.reef.runtime.local.client.LocalRuntimeConfiguration; +import org.apache.reef.runtime.local.driver.RuntimeIdentifier; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.Configurations; import org.apache.reef.tang.JavaConfigurationBuilder; @@ -74,4 +75,9 @@ public final class LocalTestEnvironment extends TestEnvironmentBase implements T public int getTestTimeout() { return 60000; // 1 min. } + + @Override + public String getRuntimeName() { + return RuntimeIdentifier.RUNTIME_NAME; + } } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/MesosTestEnvironment.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/MesosTestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/MesosTestEnvironment.java index 51b67bf..a4e2739 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/MesosTestEnvironment.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/MesosTestEnvironment.java @@ -19,6 +19,7 @@ package org.apache.reef.tests; import org.apache.reef.runtime.mesos.client.MesosClientConfiguration; +import org.apache.reef.runtime.mesos.driver.RuntimeIdentifier; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.exceptions.BindException; @@ -63,5 +64,9 @@ public final class MesosTestEnvironment extends TestEnvironmentBase implements T return 300000; // 5 minutes } + @Override + public String getRuntimeName() { + return RuntimeIdentifier.RUNTIME_NAME; + } } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironment.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironment.java index e6fa832..a7e149e 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironment.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/TestEnvironment.java @@ -56,4 +56,9 @@ public interface TestEnvironment { LauncherStatus run(final Configuration driverConfiguration); + /** + * Returns the runtimeName for the environment. + * @return runtimeName + */ + String getRuntimeName(); } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/YarnTestEnvironment.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/YarnTestEnvironment.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/YarnTestEnvironment.java index aa401e8..d534909 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/YarnTestEnvironment.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/YarnTestEnvironment.java @@ -19,6 +19,7 @@ package org.apache.reef.tests; import org.apache.reef.runtime.yarn.client.YarnClientConfiguration; +import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.exceptions.BindException; @@ -56,5 +57,9 @@ public final class YarnTestEnvironment extends TestEnvironmentBase implements Te return 300000; // 5 minutes } + @Override + public String getRuntimeName() { + return RuntimeIdentifier.RUNTIME_NAME; + } } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameDriver.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameDriver.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameDriver.java new file mode 100644 index 0000000..a57ffe2 --- /dev/null +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameDriver.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.tests.runtimename; + +import org.apache.reef.driver.evaluator.AllocatedEvaluator; +import org.apache.reef.tang.annotations.Parameter; +import org.apache.reef.tang.annotations.Unit; +import org.apache.reef.tests.driver.RuntimeNameTestConfiguration; +import org.apache.reef.tests.library.exceptions.DriverSideFailure; +import org.apache.reef.wake.EventHandler; + +import javax.inject.Inject; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Unit +final class RuntimeNameDriver { + private static final Logger LOG = Logger.getLogger(RuntimeNameDriver.class.getName()); + private final String runtimeName; + + @Inject + RuntimeNameDriver(@Parameter(RuntimeNameTestConfiguration.RuntimeName.class) final String name) { + runtimeName = name; + } + + /** + * Handles AllocatedEvaluator: Submit the HelloTask. + */ + public final class EvaluatorAllocatedHandler implements EventHandler<AllocatedEvaluator> { + + @Override + public void onNext(final AllocatedEvaluator allocatedEvaluator) { + LOG.log(Level.INFO, "Evaluator allocated with runtime: {0}", + allocatedEvaluator.getEvaluatorDescriptor().getRuntimeName()); + LOG.log(Level.INFO, "Evaluator expected runtime runtime: {0}", RuntimeNameDriver.this.runtimeName); + if (!RuntimeNameDriver.this.runtimeName.equals( + allocatedEvaluator.getEvaluatorDescriptor().getRuntimeName())) { + throw new DriverSideFailure( + "Got an Evaluator with different runtime name then expected. Expected " + + RuntimeNameDriver.this.runtimeName + + ", but got " + allocatedEvaluator.getEvaluatorDescriptor().getRuntimeName()); + } + + allocatedEvaluator.close(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameTest.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameTest.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameTest.java new file mode 100644 index 0000000..f1bf0c8 --- /dev/null +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/RuntimeNameTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.tests.runtimename; + +import org.apache.reef.client.DriverConfiguration; +import org.apache.reef.client.DriverLauncher; +import org.apache.reef.client.LauncherStatus; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; +import org.apache.reef.tang.exceptions.BindException; +import org.apache.reef.tang.exceptions.InjectionException; +import org.apache.reef.tests.TestEnvironment; +import org.apache.reef.tests.TestEnvironmentFactory; +import org.apache.reef.tests.driver.RuntimeNameTestConfiguration; +import org.apache.reef.tests.library.driver.OnDriverStartedAllocateOne; +import org.apache.reef.util.EnvironmentUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * This tests whether we receive correct runtimeName in the evaluator descriptor. + */ +public class RuntimeNameTest { + + private final TestEnvironment testEnvironment = TestEnvironmentFactory.getNewTestEnvironment(); + + @Before + public void setUp() throws Exception { + testEnvironment.setUp(); + } + + @Test + public void testRuntimeName() throws BindException, InjectionException { + final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration(); + final Configuration testConfiguration = RuntimeNameTestConfiguration.CONF + .set(RuntimeNameTestConfiguration.RUNTIME_NAME, this.testEnvironment.getRuntimeName()) + .build(); + + final Configuration driverConfiguration = DriverConfiguration.CONF + .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(OnDriverStartedAllocateOne.class)) + .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_DriverTest") + .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class) + .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RuntimeNameDriver.EvaluatorAllocatedHandler.class) + .build(); + + final Configuration mergedDriverConfiguration = Tang.Factory.getTang() + .newConfigurationBuilder(driverConfiguration, testConfiguration).build(); + + final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration) + .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout()); + + Assert.assertTrue("Job state after execution: " + status, status.isSuccess()); + } + + @After + public void tearDown() throws Exception { + this.testEnvironment.tearDown(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/package-info.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/package-info.java new file mode 100644 index 0000000..5f3645b --- /dev/null +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/runtimename/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Test for Driver start mechanism by launching noop driver. + */ +package org.apache.reef.tests.runtimename; http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-webserver/src/main/avro/EvaluatorInfo.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/main/avro/EvaluatorInfo.avsc b/lang/java/reef-webserver/src/main/avro/EvaluatorInfo.avsc index a8f4939..1b094bc 100644 --- a/lang/java/reef-webserver/src/main/avro/EvaluatorInfo.avsc +++ b/lang/java/reef-webserver/src/main/avro/EvaluatorInfo.avsc @@ -27,7 +27,8 @@ { "name": "nodeName", "type": "string" }, { "name": "memory", "type": "int" }, { "name": "type", "type": "string" }, - { "name": "internetAddress", "type": "string" } + { "name": "internetAddress", "type": "string" }, + { "name": "runtimeName", "type": "string" } ] }, { http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/AvroEvaluatorInfoSerializer.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/AvroEvaluatorInfoSerializer.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/AvroEvaluatorInfoSerializer.java index cce88c3..73e0354 100644 --- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/AvroEvaluatorInfoSerializer.java +++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/AvroEvaluatorInfoSerializer.java @@ -59,6 +59,7 @@ public class AvroEvaluatorInfoSerializer implements EvaluatorInfoSerializer { InetSocketAddress address = null; int memory = 0; String type = null; + String runtimeName = null; if (evaluatorDescriptor != null) { nodeId = evaluatorDescriptor.getNodeDescriptor().getId(); @@ -66,6 +67,7 @@ public class AvroEvaluatorInfoSerializer implements EvaluatorInfoSerializer { address = evaluatorDescriptor.getNodeDescriptor().getInetSocketAddress(); memory = evaluatorDescriptor.getMemory(); type = evaluatorDescriptor.getProcess().getType().toString(); + runtimeName = evaluatorDescriptor.getRuntimeName(); } evaluatorsInfo.add(AvroEvaluatorInfo.newBuilder() @@ -75,6 +77,7 @@ public class AvroEvaluatorInfoSerializer implements EvaluatorInfoSerializer { .setInternetAddress(address != null ? address.toString() : "") .setMemory(memory) .setType(type != null ? type : "") + .setRuntimeName(runtimeName != null ? runtimeName : "") .build()); } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java index 7f68277..acbc0f1 100644 --- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java +++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java @@ -280,6 +280,8 @@ public final class HttpServerReefEventHandler implements HttpHandler { writer.write("<br/>"); writer.println("Evaluator Type: " + evaluatorDescriptor.getProcess()); writer.write("<br/>"); + writer.println("Evaluator Runtime Name: " + evaluatorDescriptor.getRuntimeName()); + writer.write("<br/>"); } else { writer.println("Incorrect Evaluator Id: " + id); } http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestAvroSerializerForHttp.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestAvroSerializerForHttp.java b/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestAvroSerializerForHttp.java index 095851d..d2cb897 100644 --- a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestAvroSerializerForHttp.java +++ b/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestAvroSerializerForHttp.java @@ -79,7 +79,7 @@ public class TestAvroSerializerForHttp { final AvroEvaluatorsInfo evaluatorInfo = serializer.toAvro(ids, data); final String evaluatorInfoString = serializer.toString(evaluatorInfo); Assert.assertEquals(evaluatorInfoString, "{\"evaluatorsInfo\":[{\"evaluatorId\":\"abc\",\"nodeId\":\"\"," + - "\"nodeName\":\"mock\",\"memory\":64,\"type\":\"CLR\",\"internetAddress\":\"\"}]}"); + "\"nodeName\":\"mock\",\"memory\":64,\"type\":\"CLR\",\"internetAddress\":\"\",\"runtimeName\":\"Local\"}]}"); } catch (final InjectionException e) { Assert.fail("Not able to inject EvaluatorInfoSerializer"); } @@ -157,6 +157,11 @@ public class TestAvroSerializerForHttp { public int getNumberOfCores() { return 1; } + + @Override + public String getRuntimeName() { + return "Local"; + } } static class NodeDescriptorMock implements NodeDescriptor { http://git-wip-us.apache.org/repos/asf/reef/blob/d8fe048d/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestReefEventStateManager.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestReefEventStateManager.java b/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestReefEventStateManager.java index c64f5be..de20af9 100644 --- a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestReefEventStateManager.java +++ b/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestReefEventStateManager.java @@ -123,6 +123,11 @@ final class MockEvaluatorDescriptor implements EvaluatorDescriptor { public int getNumberOfCores() { return 1; } + + @Override + public String getRuntimeName() { + return "Local"; + } } final class MockNodeDescriptor implements NodeDescriptor {
