Repository: reef Updated Branches: refs/heads/master a56569520 -> 1cf486957
[REEF-1136] Implementing standalone runtime high-level interface This addressed the issue by * Implementing basic high-level interface for standalone runtime. * `NodeListFilePath` is a required parameter. * The file contains lines of ssh info. (e.g. reefuser@localhost) * Upon resource request, the runtime selects a node from it. JIRA: [REEF-1136] (https://issues.apache.org/jira/browse/REEF-1136) Pull Request: This closes #819 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/1cf48695 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/1cf48695 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/1cf48695 Branch: refs/heads/master Commit: 1cf486957fe6af308c8e26abb65c68807dca571f Parents: a565695 Author: LastOne817 <[email protected]> Authored: Wed Feb 3 15:31:28 2016 +0900 Committer: Markus Weimer <[email protected]> Committed: Sat Feb 6 15:56:32 2016 -0800 ---------------------------------------------------------------------- lang/java/reef-runtime-standalone/pom.xml | 70 ++++++++++++++++++++ .../client/StandaloneRuntimeConfiguration.java | 69 +++++++++++++++++++ .../runtime/standalone/client/package-info.java | 22 ++++++ .../client/parameters/NodeListFilePath.java | 36 ++++++++++ .../client/parameters/RootFolder.java | 38 +++++++++++ .../client/parameters/package-info.java | 22 ++++++ .../reef/runtime/standalone/package-info.java | 22 ++++++ pom.xml | 2 + 8 files changed, 281 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/pom.xml ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/pom.xml b/lang/java/reef-runtime-standalone/pom.xml new file mode 100644 index 0000000..43062f9 --- /dev/null +++ b/lang/java/reef-runtime-standalone/pom.xml @@ -0,0 +1,70 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.reef</groupId> + <artifactId>reef-project</artifactId> + <version>0.14.0-SNAPSHOT</version> + <relativePath>../../..</relativePath> + </parent> + + <artifactId>reef-runtime-standalone</artifactId> + <name>REEF Runtime Standalone</name> + <description>A ssh-based standalone distributed runtime.</description> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>reef-common</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <resources> + <resource> + <targetPath>META-INF/</targetPath> + <filtering>false</filtering> + <directory>${basedir}/conf</directory> + <includes> + <include>*.xml</include> + <include>*.properties</include> + </includes> + <excludes> + </excludes> + </resource> + </resources> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <configLocation>lang/java/reef-common/src/main/resources/checkstyle-strict.xml</configLocation> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java new file mode 100644 index 0000000..cb3ca3a --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.standalone.client; + +import org.apache.reef.annotations.Unstable; +import org.apache.reef.client.parameters.DriverConfigurationProviders; +import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration; +import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; +import org.apache.reef.runtime.standalone.client.parameters.NodeListFilePath; +import org.apache.reef.runtime.standalone.client.parameters.RootFolder; +import org.apache.reef.tang.ConfigurationProvider; +import org.apache.reef.tang.formats.*; +import org.apache.reef.wake.time.Clock; + +/** + * A ConfigurationModule to configure the standalone resourcemanager. + */ +@Unstable +public final class StandaloneRuntimeConfiguration extends ConfigurationModuleBuilder { + + /** + * The folder in which the sub-folders, one per Node, will be created. Those will contain one folder per + * Evaluator instantiated on the virtual node. Those inner folders will be named by the time when the Evaluator was + * launched. + * <p> + * If none is given, a folder "REEF_STANDALONE_RUNTIME" will be created in the local directory. + */ + public static final OptionalParameter<String> RUNTIME_ROOT_FOLDER = new OptionalParameter<>(); + + /** + * Configuration provides whose Configuration will be merged into all Driver Configuration. + */ + public static final OptionalImpl<ConfigurationProvider> DRIVER_CONFIGURATION_PROVIDERS = new OptionalImpl<>(); + + /** + * The file which will contain information of remote nodes. + */ + public static final RequiredParameter<String> NODE_LIST_FILE_PATH = new RequiredParameter<>(); + + /** + * The ConfigurationModule for the standalone resourcemanager. + */ + public static final ConfigurationModule CONF = new StandaloneRuntimeConfiguration() + .merge(CommonRuntimeConfiguration.CONF) + // Bind parameters of the standalone runtime + .bindNamedParameter(RootFolder.class, RUNTIME_ROOT_FOLDER) + .bindNamedParameter(NodeListFilePath.class, NODE_LIST_FILE_PATH) + .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS) + .bindSetEntry(Clock.StartHandler.class, PIDStoreStartHandler.class) + .build(); + + +} http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/package-info.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/package-info.java new file mode 100644 index 0000000..4cd7895 --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Client-side event handlers for the standalone resourcemanager. + */ +package org.apache.reef.runtime.standalone.client; http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/NodeListFilePath.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/NodeListFilePath.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/NodeListFilePath.java new file mode 100644 index 0000000..b3809f0 --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/NodeListFilePath.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.standalone.client.parameters; + +import org.apache.reef.annotations.Unstable; +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The file which will contain information of remote nodes. + */ +@Unstable +@NamedParameter(doc = "The file contains lines of ssh info of remote nodes", short_name = "node_list_file_path") +public final class NodeListFilePath implements Name<String> { + /** + * Empty private constructor to prohibit instantiation of utility class. + */ + private NodeListFilePath() { + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/RootFolder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/RootFolder.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/RootFolder.java new file mode 100644 index 0000000..85a349a --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/RootFolder.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.reef.runtime.standalone.client.parameters; + +import org.apache.reef.annotations.Unstable; +import org.apache.reef.tang.annotations.Name; +import org.apache.reef.tang.annotations.NamedParameter; + +/** + * The folder where logs etc. shall be stored. + */ +@Unstable +@NamedParameter(default_value = RootFolder.DEFAULT_VALUE, doc = "The folder where logs etc. shall be stored.") +public final class RootFolder implements Name<String> { + public static final String DEFAULT_VALUE = "REEF_STANDALONE_RUNTIME"; + + /** + * Empty private constructor to prohibit instantiation of utility class. + */ + private RootFolder() { + } +} http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/package-info.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/package-info.java new file mode 100644 index 0000000..3dda6a6 --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/parameters/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Parameters of the standalone runtime. + */ +package org.apache.reef.runtime.standalone.client.parameters; http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/package-info.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/package-info.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/package-info.java new file mode 100644 index 0000000..9f6e777 --- /dev/null +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * A standalone implementation of REEF that uses remote JVMs for execution. + */ +package org.apache.reef.runtime.standalone; http://git-wip-us.apache.org/repos/asf/reef/blob/1cf48695/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 0b70d55..4f2b13c 100644 --- a/pom.xml +++ b/pom.xml @@ -275,6 +275,7 @@ under the License. <!-- The below are sometimes created during tests --> <exclude>REEF_LOCAL_RUNTIME/**</exclude> <exclude>REEF_MESOS_RUNTIME/**</exclude> + <exclude>REEF_STANDALONE_RUNTIME/**</exclude> <!-- JVM error logs, especially troublesome on CI servers --> <exclude>**/hs_err_*.log</exclude> <!-- The Visual Studio and Nuget build files --> @@ -814,6 +815,7 @@ under the License. <module>lang/java/reef-runtime-yarn</module> <module>lang/java/reef-runtime-mesos</module> <module>lang/java/reef-runtime-multi</module> + <module>lang/java/reef-runtime-standalone</module> <module>lang/java/reef-tang</module> <module>lang/java/reef-tests</module> <module>lang/java/reef-wake</module>
