Repository: reef Updated Branches: refs/heads/master 040a6b366 -> 913c66cc7
[REEF-1375] Fixing Java multiruntime Addresses the issue by: 1) Fixing the multiruntime configuration builder by adding additioanl configurations that are required 2) Fixing test to actually use multiruntime JIRA: [REEF-1375](https://issues.apache.org/jira/browse/REEF-1375) Pull Request: Closes #983 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/913c66cc Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/913c66cc Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/913c66cc Branch: refs/heads/master Commit: 913c66cc70b20691bf45c1a3bcb5e035c45e6af2 Parents: 040a6b3 Author: Boris Shulman <[email protected]> Authored: Mon May 2 00:56:56 2016 -0700 Committer: Andrew Chung <[email protected]> Committed: Mon May 9 13:17:53 2016 -0700 ---------------------------------------------------------------------- .../hellomultiruntime/HelloREEFMultiYarn.java | 2 +- .../src/main/avro/RuntimeDefinition.avsc | 6 +-- ...tiRuntimeMainConfigurationGeneratorImpl.java | 46 +++++++++++++++++ .../MultiRuntimeConfigurationBuilder.java | 30 ++++++----- .../client/MultiRuntimeDefinitionBuilder.java | 18 ++++--- .../client/MultiRuntimeDefinitionGenerator.java | 14 ++++-- .../MultiRuntimeDefinitionGeneratorImpl.java | 18 ++++--- ...MultiRuntimeDriverConfigurationProvider.java | 21 +++++--- .../client/MultiRuntimeHelperConfiguration.java | 28 ----------- .../MultiRuntimeMainConfigurationGenerator.java | 46 +++++++++++++++++ ...tiRuntimeMainConfigurationGeneratorImpl.java | 53 ++++++++++++++++++++ .../reef/runtime/multi/driver/RuntimesHost.java | 8 +-- .../utils/MultiRuntimeDefinitionSerializer.java | 17 ++++--- .../runtime/multi/driver/RuntimesHostTest.java | 4 +- .../MultiRuntimeDefinitionSerializerTests.java | 12 ++--- .../reef/tests/examples/ExamplesTestSuite.java | 3 +- .../examples/TestHelloREEFMultiRuntime.java | 21 ++++++-- 17 files changed, 252 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-examples/src/main/java/org/apache/reef/examples/hellomultiruntime/HelloREEFMultiYarn.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/hellomultiruntime/HelloREEFMultiYarn.java b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/hellomultiruntime/HelloREEFMultiYarn.java index 5df7b25..8d8a95c 100644 --- a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/hellomultiruntime/HelloREEFMultiYarn.java +++ b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/hellomultiruntime/HelloREEFMultiYarn.java @@ -50,7 +50,7 @@ public final class HelloREEFMultiYarn { return DriverConfiguration.CONF .set(DriverConfiguration.GLOBAL_LIBRARIES, HelloREEFMultiYarn.class.getProtectionDomain().getCodeSource().getLocation().getFile()) - .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloREEF") + .set(DriverConfiguration.DRIVER_IDENTIFIER, "HelloREEFMultiYarn") .set(DriverConfiguration.ON_DRIVER_STARTED, HelloMultiRuntimeDriver.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloMultiRuntimeDriver.EvaluatorAllocatedHandler.class) .build(); http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/avro/RuntimeDefinition.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/avro/RuntimeDefinition.avsc b/lang/java/reef-runtime-multi/src/main/avro/RuntimeDefinition.avsc index 2e81ffa..ceb65e2 100644 --- a/lang/java/reef-runtime-multi/src/main/avro/RuntimeDefinition.avsc +++ b/lang/java/reef-runtime-multi/src/main/avro/RuntimeDefinition.avsc @@ -24,7 +24,7 @@ { "namespace":"org.apache.reef.runtime.multi.utils.avro", "type":"record", - "name":"RuntimeDefinition", + "name":"AvroRuntimeDefinition", "doc":"Defines the schema for runtime definition. This avro object is used to pass runtimes definitions to the runtimes host", "fields":[ { @@ -46,7 +46,7 @@ { "namespace":"org.apache.reef.runtime.multi.utils.avro", "type":"record", - "name":"MultiRuntimeDefinition", + "name":"AvroMultiRuntimeDefinition", "doc":"Defines the schema for multi runtime definition. This avro object is used to pass multi runtime definition to the runtimes host", "fields":[ { @@ -56,7 +56,7 @@ }, { "name":"runtimes", - "type":{"type":"array", "items":"RuntimeDefinition"}, + "type":{"type":"array", "items":"AvroRuntimeDefinition"}, "doc":"defined runtimes" } ] http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/EmptyMultiRuntimeMainConfigurationGeneratorImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/EmptyMultiRuntimeMainConfigurationGeneratorImpl.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/EmptyMultiRuntimeMainConfigurationGeneratorImpl.java new file mode 100644 index 0000000..b79a7da --- /dev/null +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/EmptyMultiRuntimeMainConfigurationGeneratorImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.reef.runtime.multi.client; + +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; + +import javax.inject.Inject; + +/** + * Default implementation for MultiRuntimeMainConfigurationGenerator that returns an empty configuration. + */ +final class EmptyMultiRuntimeMainConfigurationGeneratorImpl + implements MultiRuntimeMainConfigurationGenerator { + + @Inject + private EmptyMultiRuntimeMainConfigurationGeneratorImpl() { + } + + /** + * Generates needed driver configuration such as class path provider. + * + * @return Instance of <code>Configuration</code> + */ + @Override + public Configuration getMainConfiguration() { + return Tang.Factory.getTang().newConfigurationBuilder().build(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeConfigurationBuilder.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeConfigurationBuilder.java index 9851cc9..b9cc1cc 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeConfigurationBuilder.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeConfigurationBuilder.java @@ -19,6 +19,7 @@ package org.apache.reef.runtime.multi.client; import org.apache.commons.lang.Validate; +import org.apache.reef.annotations.Unstable; import org.apache.reef.runtime.local.client.parameters.MaxNumberOfEvaluators; import org.apache.reef.runtime.multi.client.parameters.DefaultRuntimeName; import org.apache.reef.runtime.multi.client.parameters.RuntimeNames; @@ -26,7 +27,8 @@ import org.apache.reef.runtime.yarn.client.ExtensibleYarnClientConfiguration; import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.Configurations; -import org.apache.reef.tang.formats.ConfigurationModuleBuilder; +import org.apache.reef.tang.JavaConfigurationBuilder; +import org.apache.reef.tang.Tang; import org.apache.reef.util.Optional; import java.util.*; @@ -34,6 +36,7 @@ import java.util.*; /** * A builder for Multi Runtime Configuration. */ +@Unstable public final class MultiRuntimeConfigurationBuilder { private static final Set<String> SUPPORTED_RUNTIMES = new HashSet<>(Arrays.asList( org.apache.reef.runtime.yarn.driver.RuntimeIdentifier.RUNTIME_NAME, @@ -45,7 +48,7 @@ public final class MultiRuntimeConfigurationBuilder { private Set<String> runtimeNames = new HashSet<>(); private Optional<String> defaultRuntime = Optional.empty(); - private String submissionRunitme; + private String submissionRuntime; private void addNamedParameter(final Class namedParameter, final Object namedParameterValue) { @@ -85,11 +88,12 @@ public final class MultiRuntimeConfigurationBuilder { * @return The builder instance */ public MultiRuntimeConfigurationBuilder setSubmissionRuntime(final String runtimeName) { - Validate.isTrue(SUPPORTED_SUBMISSION_RUNTIMES.contains(runtimeName), "Unsupported submission runtime " + + Validate.isTrue( + SUPPORTED_SUBMISSION_RUNTIMES.contains(runtimeName), "Unsupported submission runtime " + runtimeName); - Validate.isTrue(this.submissionRunitme == null, "Submission runtime was already added"); + Validate.isTrue(this.submissionRuntime == null, "Submission runtime was already added"); - this.submissionRunitme = runtimeName; + this.submissionRuntime = runtimeName; return this; } @@ -110,7 +114,7 @@ public final class MultiRuntimeConfigurationBuilder { * @return The built configuration */ public Configuration build() { - Validate.notNull(this.submissionRunitme, "Default Runtime was not defined"); + Validate.notNull(this.submissionRuntime, "Default Runtime was not defined"); if(!this.defaultRuntime.isPresent() || this.runtimeNames.size() == 1){ this.defaultRuntime = Optional.of(this.runtimeNames.toArray(new String[0])[0]); @@ -123,7 +127,7 @@ public final class MultiRuntimeConfigurationBuilder { this.runtimeNames.add(this.defaultRuntime.get()); } - ConfigurationModuleBuilder conf = new MultiRuntimeHelperConfiguration(); + JavaConfigurationBuilder conf = Tang.Factory.getTang().newConfigurationBuilder(); for(Map.Entry<Class, Object> entry: this.namedParameters.entrySet()){ conf = conf.bindNamedParameter(entry.getKey(), entry.getValue().toString()); @@ -135,15 +139,15 @@ public final class MultiRuntimeConfigurationBuilder { conf = conf.bindSetEntry(RuntimeNames.class, runtimeName); } - conf = conf.bindImplementation( - MultiRuntimeDefinitionGenerator.class, MultiRuntimeDefinitionGeneratorImpl.class); - - if(!this.submissionRunitme.equalsIgnoreCase(RuntimeIdentifier.RUNTIME_NAME)){ - throw new RuntimeException("Unsupported submission runtime " + this.submissionRunitme); + if(!this.submissionRuntime.equalsIgnoreCase(RuntimeIdentifier.RUNTIME_NAME)){ + throw new RuntimeException("Unsupported submission runtime " + this.submissionRuntime); } + conf = conf.bindImplementation(MultiRuntimeMainConfigurationGenerator.class, + YarnMultiRuntimeMainConfigurationGeneratorImpl.class); + // Currently only local runtime is supported as a secondary runtime - return Configurations.merge(conf.build().build(), + return Configurations.merge(conf.build(), ExtensibleYarnClientConfiguration.CONF .set(ExtensibleYarnClientConfiguration.DRIVER_CONFIGURATION_PROVIDER, MultiRuntimeDriverConfigurationProvider.class).build()); http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionBuilder.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionBuilder.java index edb0667..7d60e92 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionBuilder.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionBuilder.java @@ -20,8 +20,9 @@ package org.apache.reef.runtime.multi.client; import org.apache.commons.lang.Validate; import org.apache.commons.lang.StringUtils; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; -import org.apache.reef.runtime.multi.utils.avro.RuntimeDefinition; +import org.apache.reef.annotations.Unstable; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroRuntimeDefinition; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.formats.AvroConfigurationSerializer; @@ -32,16 +33,17 @@ import java.util.Map; /** * Builder for multi runtime definition. */ +@Unstable public final class MultiRuntimeDefinitionBuilder { - private Map<String, RuntimeDefinition> runtimes = new HashMap<>(); + private Map<String, AvroRuntimeDefinition> runtimes = new HashMap<>(); private String defaultRuntime; - private static RuntimeDefinition createRuntimeDefinition(final Configuration configModule, + private static AvroRuntimeDefinition createRuntimeDefinition(final Configuration configModule, final String runtimeName) { final Configuration localDriverConfiguration = configModule; final AvroConfigurationSerializer serializer = new AvroConfigurationSerializer(); final String serializedConfig = serializer.toString(localDriverConfiguration); - return new RuntimeDefinition(runtimeName, serializedConfig); + return new AvroRuntimeDefinition(runtimeName, serializedConfig); } /** @@ -54,7 +56,7 @@ public final class MultiRuntimeDefinitionBuilder { Validate.notNull(config, "runtime configuration module should not be null"); Validate.isTrue(StringUtils.isNotBlank(runtimeName), "runtimeName should be non empty and non blank string"); - final RuntimeDefinition rd = createRuntimeDefinition(config, runtimeName); + final AvroRuntimeDefinition rd = createRuntimeDefinition(config, runtimeName); this.runtimes.put(runtimeName, rd); return this; } @@ -75,7 +77,7 @@ public final class MultiRuntimeDefinitionBuilder { * Builds multi runtime definition. * @return The populated definition object */ - public MultiRuntimeDefinition build(){ + public AvroMultiRuntimeDefinition build(){ Validate.isTrue(this.runtimes.size() == 1 || !StringUtils.isEmpty(this.defaultRuntime), "Default runtime " + "should be set if more than a single runtime provided"); @@ -85,6 +87,6 @@ public final class MultiRuntimeDefinitionBuilder { } Validate.isTrue(this.runtimes.containsKey(this.defaultRuntime), "Default runtime should be configured"); - return new MultiRuntimeDefinition(defaultRuntime, new ArrayList<>(this.runtimes.values())); + return new AvroMultiRuntimeDefinition(defaultRuntime, new ArrayList<>(this.runtimes.values())); } } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGenerator.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGenerator.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGenerator.java index faf4901..0deea08 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGenerator.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGenerator.java @@ -19,13 +19,19 @@ package org.apache.reef.runtime.multi.client; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; +import org.apache.reef.annotations.Unstable; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; +import org.apache.reef.tang.annotations.DefaultImplementation; import java.net.URI; /** * Defines a contract for a multi runtime definition generator. + * The multi runtime avro definition contains all the needed information to instantiate + * runtimes inside runtime host. */ +@DefaultImplementation(MultiRuntimeDefinitionGeneratorImpl.class) +@Unstable public interface MultiRuntimeDefinitionGenerator { /** * Generates needed driver configuration modules. @@ -35,8 +41,8 @@ public interface MultiRuntimeDefinitionGenerator { * @param jobId the job id * @return Instance of <code>MultiRuntimeDefinition</code> */ - MultiRuntimeDefinition getMultiRuntimeDefinition(final URI jobFolder, - final String clientRemoteId, - final String jobId); + AvroMultiRuntimeDefinition getMultiRuntimeDefinition(final URI jobFolder, + final String clientRemoteId, + final String jobId); } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGeneratorImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGeneratorImpl.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGeneratorImpl.java index b44c72d..7217da7 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGeneratorImpl.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDefinitionGeneratorImpl.java @@ -28,7 +28,7 @@ import org.apache.reef.runtime.local.client.parameters.RackNames; import org.apache.reef.runtime.local.driver.LocalDriverConfiguration; import org.apache.reef.runtime.multi.client.parameters.DefaultRuntimeName; import org.apache.reef.runtime.multi.client.parameters.RuntimeNames; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier; import org.apache.reef.runtime.yarn.driver.YarnDriverConfiguration; import org.apache.reef.tang.Configuration; @@ -42,7 +42,7 @@ import java.util.Map; import java.util.Set; /** - * MultiRuntime configuration provider. + * MultiRuntime definition provider. Creates avro definition for the multi runtime environment. */ @Private @RuntimeAuthor @@ -107,7 +107,9 @@ final class MultiRuntimeDefinitionGeneratorImpl implements MultiRuntimeDefinitio .set(YarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolder.toString()) .set(YarnDriverConfiguration.JOB_IDENTIFIER, jobId) .set(YarnDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) - .set(YarnDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack).build(); + .set(YarnDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) + .set(YarnDriverConfiguration.RUNTIME_NAMES, RuntimeIdentifier.RUNTIME_NAME) + .build(); } @@ -121,7 +123,9 @@ final class MultiRuntimeDefinitionGeneratorImpl implements MultiRuntimeDefinitio .set(LocalDriverConfiguration.ROOT_FOLDER, ".") .set(LocalDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .set(LocalDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) - .set(LocalDriverConfiguration.JOB_IDENTIFIER, jobId); + .set(LocalDriverConfiguration.JOB_IDENTIFIER, jobId) + .set(LocalDriverConfiguration.RUNTIME_NAMES, + org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME); for (final String rackName : rackNames) { localModule = localModule.set(LocalDriverConfiguration.RACK_NAMES, rackName); } @@ -130,9 +134,9 @@ final class MultiRuntimeDefinitionGeneratorImpl implements MultiRuntimeDefinitio } - public MultiRuntimeDefinition getMultiRuntimeDefinition(final URI jobFolder, - final String clientRemoteId, - final String jobId) { + public AvroMultiRuntimeDefinition getMultiRuntimeDefinition(final URI jobFolder, + final String clientRemoteId, + final String jobId) { MultiRuntimeDefinitionBuilder builder = new MultiRuntimeDefinitionBuilder(); http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDriverConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDriverConfigurationProvider.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDriverConfigurationProvider.java index 3e9b82e..6dac3e5 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDriverConfigurationProvider.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeDriverConfigurationProvider.java @@ -21,8 +21,8 @@ package org.apache.reef.runtime.multi.client; import org.apache.reef.runtime.common.client.DriverConfigurationProvider; import org.apache.reef.runtime.multi.driver.MultiRuntimeDriverConfiguration; import org.apache.reef.runtime.multi.utils.MultiRuntimeDefinitionSerializer; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; -import org.apache.reef.runtime.multi.utils.avro.RuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroRuntimeDefinition; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.Configurations; import org.apache.reef.tang.formats.ConfigurationModule; @@ -35,11 +35,15 @@ import java.net.URI; */ final class MultiRuntimeDriverConfigurationProvider implements DriverConfigurationProvider { private final MultiRuntimeDefinitionSerializer runtimeDefinitionSerializer = new MultiRuntimeDefinitionSerializer(); + private final MultiRuntimeMainConfigurationGenerator mainRuntimeConfigGenerator; private MultiRuntimeDefinitionGenerator definitionGenerator; @Inject - MultiRuntimeDriverConfigurationProvider(final MultiRuntimeDefinitionGenerator definitionGenerator) { + MultiRuntimeDriverConfigurationProvider( + final MultiRuntimeDefinitionGenerator definitionGenerator, + final MultiRuntimeMainConfigurationGenerator mainRuntimeConfigGenerator) { this.definitionGenerator = definitionGenerator; + this.mainRuntimeConfigGenerator = mainRuntimeConfigGenerator; } /** @@ -57,21 +61,24 @@ final class MultiRuntimeDriverConfigurationProvider implements DriverConfigurati final String clientRemoteId, final String jobId, final Configuration applicationConfiguration) { - MultiRuntimeDefinition runtimeDefinitions = this.definitionGenerator.getMultiRuntimeDefinition( + AvroMultiRuntimeDefinition runtimeDefinitions = this.definitionGenerator.getMultiRuntimeDefinition( jobFolder, clientRemoteId, jobId); ConfigurationModule conf = MultiRuntimeDriverConfiguration.CONF; - for(RuntimeDefinition runtimeDefinition : runtimeDefinitions.getRuntimes()){ + for(AvroRuntimeDefinition runtimeDefinition : runtimeDefinitions.getRuntimes()){ conf = conf.set(MultiRuntimeDriverConfiguration.RUNTIME_NAMES, runtimeDefinition.getRuntimeName().toString()); } - return Configurations.merge(applicationConfiguration, + final Configuration mainConfiguration = this.mainRuntimeConfigGenerator.getMainConfiguration(); + + return Configurations.merge(mainConfiguration, applicationConfiguration, conf .set(MultiRuntimeDriverConfiguration.JOB_IDENTIFIER, jobId) .set(MultiRuntimeDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId) .set(MultiRuntimeDriverConfiguration.SERIALIZED_RUNTIME_DEFINITION, - this.runtimeDefinitionSerializer.toString(runtimeDefinitions)).build()); + this.runtimeDefinitionSerializer.toString(runtimeDefinitions)) + .build()); } } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeHelperConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeHelperConfiguration.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeHelperConfiguration.java deleted file mode 100644 index 1a5ff69..0000000 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeHelperConfiguration.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.reef.runtime.multi.client; - -import org.apache.reef.tang.formats.ConfigurationModuleBuilder; - -/** - * A ConfigurationModule to bind arbitrary named parameters for runtimes. - */ -final class MultiRuntimeHelperConfiguration extends ConfigurationModuleBuilder { -} - http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeMainConfigurationGenerator.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeMainConfigurationGenerator.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeMainConfigurationGenerator.java new file mode 100644 index 0000000..bd9698e --- /dev/null +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/MultiRuntimeMainConfigurationGenerator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.reef.runtime.multi.client; + +import org.apache.reef.annotations.Unstable; +import org.apache.reef.annotations.audience.RuntimeAuthor; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.annotations.DefaultImplementation; + +/** + * Defines a contract for a multi runtime main configuration. + * Main configuration is the configuration that allows multi runtime to be initialized in the certain concrete + * environment. For example, when operating in the yarn runtime, multi runtime needs to receive the proper + * classpath provider as well as the constructor for the YarnConfiguration. + * classpath provider as well as the constructor for the YarnConfiguration. + * Main configuration is provided in the context of the main runtime. Main runtime is the runtime that actually runs + * multi runtime host that hosts the actual runtimes. + *The main configuration is generated on the client and is merged into the driver configuration. + */ +@DefaultImplementation(EmptyMultiRuntimeMainConfigurationGeneratorImpl.class) +@Unstable +@RuntimeAuthor +interface MultiRuntimeMainConfigurationGenerator { + /** + * Generates needed driver configuration such as class path provider. + * @return Instance of <code>Configuration</code> + */ + Configuration getMainConfiguration(); +} http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/YarnMultiRuntimeMainConfigurationGeneratorImpl.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/YarnMultiRuntimeMainConfigurationGeneratorImpl.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/YarnMultiRuntimeMainConfigurationGeneratorImpl.java new file mode 100644 index 0000000..e9e7fdf --- /dev/null +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/client/YarnMultiRuntimeMainConfigurationGeneratorImpl.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.reef.runtime.multi.client; + +import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; +import org.apache.reef.runtime.yarn.YarnClasspathProvider; +import org.apache.reef.runtime.yarn.util.YarnConfigurationConstructor; +import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.Tang; + +import javax.inject.Inject; + +/** + * Yarn implementation for MultiRuntimeMainConfigurationGenerator. + */ +final class YarnMultiRuntimeMainConfigurationGeneratorImpl + implements MultiRuntimeMainConfigurationGenerator { + + @Inject + private YarnMultiRuntimeMainConfigurationGeneratorImpl() { + } + + /** + * Generates configuration that allows multi runtime to run on Yarn. + * MultiRuntimeMainConfigurationGenerator. + * @return Instance of <code>Configuration</code> + */ + @Override + public Configuration getMainConfiguration() { + + return Tang.Factory.getTang().newConfigurationBuilder() + .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class) + .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java index 00d3a62..bfbc037 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java @@ -28,8 +28,8 @@ import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEvent; import org.apache.reef.runtime.multi.client.parameters.SerializedRuntimeDefinition; import org.apache.reef.runtime.multi.driver.parameters.RuntimeName; import org.apache.reef.runtime.multi.utils.MultiRuntimeDefinitionSerializer; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; -import org.apache.reef.runtime.multi.utils.avro.RuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroRuntimeDefinition; import org.apache.reef.tang.Configuration; import org.apache.reef.tang.Injector; import org.apache.reef.tang.JavaConfigurationBuilder; @@ -50,7 +50,7 @@ import java.util.Map; * Hosts the actual runtime implementations and delegates invocations to them. */ final class RuntimesHost { - private final MultiRuntimeDefinition runtimeDefinition; + private final AvroMultiRuntimeDefinition runtimeDefinition; private final Injector originalInjector; private final String defaultRuntimeName; private final MultiRuntimeDefinitionSerializer runtimeDefinitionSerializer = new MultiRuntimeDefinitionSerializer(); @@ -79,7 +79,7 @@ final class RuntimesHost { this.runtimes = new HashMap<>(); - for (final RuntimeDefinition rd : runtimeDefinition.getRuntimes()) { + for (final AvroRuntimeDefinition rd : runtimeDefinition.getRuntimes()) { try { // We need to create different injector for each runtime as they define conflicting bindings. Also we cannot http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializer.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializer.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializer.java index ba836c1..f9ad2dc 100644 --- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializer.java +++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializer.java @@ -22,7 +22,7 @@ package org.apache.reef.runtime.multi.utils; import org.apache.avro.io.*; import org.apache.avro.specific.SpecificDatumReader; import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -39,9 +39,9 @@ public final class MultiRuntimeDefinitionSerializer { * @param runtimeDefinition the Avro object to toString * @return Serialized avro string */ - public String toString(final MultiRuntimeDefinition runtimeDefinition){ - final DatumWriter<MultiRuntimeDefinition> configurationWriter = - new SpecificDatumWriter<>(MultiRuntimeDefinition.class); + public String toString(final AvroMultiRuntimeDefinition runtimeDefinition){ + final DatumWriter<AvroMultiRuntimeDefinition> configurationWriter = + new SpecificDatumWriter<>(AvroMultiRuntimeDefinition.class); final String serializedConfiguration; try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) { final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(runtimeDefinition.getSchema(), out); @@ -62,12 +62,13 @@ public final class MultiRuntimeDefinitionSerializer { * @return Avro object * @throws IOException */ - public MultiRuntimeDefinition fromString(final String serializedRuntimeDefinition) throws + public AvroMultiRuntimeDefinition fromString(final String serializedRuntimeDefinition) throws IOException{ final JsonDecoder decoder = DecoderFactory.get(). - jsonDecoder(MultiRuntimeDefinition.getClassSchema(), serializedRuntimeDefinition); - final SpecificDatumReader<MultiRuntimeDefinition> reader = new SpecificDatumReader<>(MultiRuntimeDefinition.class); - final MultiRuntimeDefinition rd = reader.read(null, decoder); + jsonDecoder(AvroMultiRuntimeDefinition.getClassSchema(), serializedRuntimeDefinition); + final SpecificDatumReader<AvroMultiRuntimeDefinition> reader = new SpecificDatumReader<>(AvroMultiRuntimeDefinition + .class); + final AvroMultiRuntimeDefinition rd = reader.read(null, decoder); return rd; } } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/driver/RuntimesHostTest.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/driver/RuntimesHostTest.java b/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/driver/RuntimesHostTest.java index 5f855f1..05c5bda 100644 --- a/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/driver/RuntimesHostTest.java +++ b/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/driver/RuntimesHostTest.java @@ -27,7 +27,7 @@ import org.apache.reef.runtime.local.driver.*; import org.apache.reef.runtime.multi.client.MultiRuntimeDefinitionBuilder; import org.apache.reef.runtime.multi.client.parameters.SerializedRuntimeDefinition; import org.apache.reef.runtime.multi.utils.MultiRuntimeDefinitionSerializer; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; import org.apache.reef.tang.*; import org.apache.reef.tang.exceptions.InjectionException; import org.apache.reef.tang.formats.ConfigurationModule; @@ -245,7 +245,7 @@ public class RuntimesHostTest { Assert.assertTrue(obj instanceof RuntimeStart); } - private String getRuntimeDefinition(final MultiRuntimeDefinition rd) { + private String getRuntimeDefinition(final AvroMultiRuntimeDefinition rd) { return new MultiRuntimeDefinitionSerializer().toString(rd); } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializerTests.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializerTests.java b/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializerTests.java index b596880..38bca11 100644 --- a/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializerTests.java +++ b/lang/java/reef-runtime-multi/src/test/java/org/apache/reef/runtime/multi/utils/MultiRuntimeDefinitionSerializerTests.java @@ -18,8 +18,8 @@ */ package org.apache.reef.runtime.multi.utils; -import org.apache.reef.runtime.multi.utils.avro.MultiRuntimeDefinition; -import org.apache.reef.runtime.multi.utils.avro.RuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroMultiRuntimeDefinition; +import org.apache.reef.runtime.multi.utils.avro.AvroRuntimeDefinition; import org.junit.Assert; import org.junit.Test; @@ -33,14 +33,14 @@ public class MultiRuntimeDefinitionSerializerTests { @Test public void testSerailze() throws IOException { final MultiRuntimeDefinitionSerializer serializer = new MultiRuntimeDefinitionSerializer(); - final MultiRuntimeDefinition def = new MultiRuntimeDefinition(); - final RuntimeDefinition rd = new RuntimeDefinition(); + final AvroMultiRuntimeDefinition def = new AvroMultiRuntimeDefinition(); + final AvroRuntimeDefinition rd = new AvroRuntimeDefinition(); rd.setRuntimeName("default"); rd.setSerializedConfiguration("config"); def.setDefaultRuntimeName("default"); - def.setRuntimes(Arrays.asList(new RuntimeDefinition[]{rd})); + def.setRuntimes(Arrays.asList(new AvroRuntimeDefinition[]{rd})); final String serialized = serializer.toString(def); - final MultiRuntimeDefinition fromStringdef = serializer.fromString(serialized); + final AvroMultiRuntimeDefinition fromStringdef = serializer.fromString(serialized); Assert.assertEquals(def, fromStringdef); } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/ExamplesTestSuite.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/ExamplesTestSuite.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/ExamplesTestSuite.java index 3ec0ec8..290d2c1 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/ExamplesTestSuite.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/ExamplesTestSuite.java @@ -26,7 +26,8 @@ import org.junit.runners.Suite; */ @RunWith(Suite.class) @Suite.SuiteClasses({ - TestHelloREEF.class + TestHelloREEF.class, + TestHelloREEFMultiRuntime.class }) public final class ExamplesTestSuite { } http://git-wip-us.apache.org/repos/asf/reef/blob/913c66cc/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/TestHelloREEFMultiRuntime.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/TestHelloREEFMultiRuntime.java b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/TestHelloREEFMultiRuntime.java index ca7491d..5ede109 100644 --- a/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/TestHelloREEFMultiRuntime.java +++ b/lang/java/reef-tests/src/test/java/org/apache/reef/tests/examples/TestHelloREEFMultiRuntime.java @@ -19,9 +19,13 @@ package org.apache.reef.tests.examples; import org.apache.reef.client.DriverConfiguration; +import org.apache.reef.client.DriverLauncher; import org.apache.reef.client.LauncherStatus; import org.apache.reef.examples.hellomultiruntime.HelloMultiRuntimeDriver; +import org.apache.reef.runtime.multi.client.MultiRuntimeConfigurationBuilder; +import org.apache.reef.runtime.yarn.driver.RuntimeIdentifier; import org.apache.reef.tang.Configuration; +import org.apache.reef.tang.exceptions.InjectionException; import org.apache.reef.tests.TestEnvironment; import org.apache.reef.tests.TestEnvironmentFactory; import org.apache.reef.tests.YarnTestEnvironment; @@ -48,16 +52,27 @@ public class TestHelloREEFMultiRuntime { } @Test - public void testHelloREEFMultiRuntime() { + public void testHelloREEFMultiRuntime() throws InjectionException { if(this.testEnvironment instanceof YarnTestEnvironment){ // multi runtime can be tested on yarn only final Configuration driverConf = DriverConfiguration.CONF .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass())) - .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_HelloREEF") + .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_HelloREEFMultiRunitme") .set(DriverConfiguration.ON_DRIVER_STARTED, HelloMultiRuntimeDriver.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, HelloMultiRuntimeDriver.EvaluatorAllocatedHandler.class) .build(); - final LauncherStatus state = this.testEnvironment.run(driverConf); + + // create the multi runtime environment + Configuration multiruntimeConfig = new MultiRuntimeConfigurationBuilder() + .addRuntime(RuntimeIdentifier.RUNTIME_NAME) + .addRuntime(org.apache.reef.runtime.local.driver.RuntimeIdentifier.RUNTIME_NAME) + .setDefaultRuntime(RuntimeIdentifier.RUNTIME_NAME) + .setMaxEvaluatorsNumberForLocalRuntime(1) + .setSubmissionRuntime(RuntimeIdentifier.RUNTIME_NAME) + .build(); + + final LauncherStatus state = DriverLauncher.getLauncher(multiruntimeConfig).run(driverConf, this + .testEnvironment.getTestTimeout()); Assert.assertTrue("Job state after execution: " + state, state.isSuccess()); } }
