This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new d8f682f SUBMARINE-467. Simplify the submitter configuration
d8f682f is described below
commit d8f682fd6e8552e4467d012be6190412f22dc499
Author: Wanqiang Ji <[email protected]>
AuthorDate: Fri Apr 17 23:56:11 2020 +0800
SUBMARINE-467. Simplify the submitter configuration
### What is this PR for?
At now the submarine server supports config more submitters, but it's not
necessary in current stage. At the same time it also cause trouble to users. So
we'd better simplify the configuration to keep only one submitter in server.
Considering the project vision, we set the K8s submitter as the default.
### What type of PR is it?
[Improvement]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-467
### How should this be tested?
https://travis-ci.com/github/jiwq/submarine/builds/160762675
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Wanqiang Ji <[email protected]>
Closes #266 from jiwq/SUBMARINE-467 and squashes the following commits:
6586d0c [Wanqiang Ji] SUBMARINE-467. Simplify the submitter configuration
---
conf/submarine-site.xml | 22 +---
conf/submarine-site.xml.template | 29 +----
docs/submarine-server/README.md | 50 ++-------
.../submarine/commons/utils/SubmarineConfVars.java | 5 +-
.../commons/utils/SubmarineConfiguration.java | 41 -------
.../apache/submarine/server/api/spec/JobSpec.java | 28 ++---
.../server/api/spec/JobSubmitterSpec.java | 118 ---------------------
.../apache/submarine/server/SubmitterManager.java | 74 +++++++------
.../apache/submarine/server/job/JobManager.java | 27 ++---
.../server/submitter/k8s/K8sJobSubmitter.java | 42 +++-----
.../server/submitter/k8s/parser/JobSpecParser.java | 2 +-
.../server/submitter/k8s/JobSpecParserTest.java | 4 +-
.../server/submitter/k8s/K8SJobSubmitterTest.java | 34 +-----
.../src/test/resources/pytorch_job_req.json | 5 +-
.../src/test/resources/tf_mnist_req.json | 5 +-
.../apache/submarine/rest/JobManagerRestApiIT.java | 14 ---
.../test/resources/pytorch/pt-mnist-patch-req.json | 5 +-
.../test/resources/pytorch/pt-mnist-patch-req.yaml | 4 +-
.../src/test/resources/pytorch/pt-mnist-req.json | 5 +-
.../src/test/resources/pytorch/pt-mnist-req.yaml | 4 +-
.../resources/tensorflow/tf-mnist-patch-req.json | 5 +-
.../resources/tensorflow/tf-mnist-patch-req.yaml | 7 +-
.../test/resources/tensorflow/tf-mnist-req.json | 5 +-
.../test/resources/tensorflow/tf-mnist-req.yaml | 6 +-
24 files changed, 106 insertions(+), 435 deletions(-)
diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml
index 302e240..c241d08 100755
--- a/conf/submarine-site.xml
+++ b/conf/submarine-site.xml
@@ -155,28 +155,10 @@
<description>Rpc server port</description>
</property>
- <!-- Submarine Submitters Configuration -->
<property>
- <name>submarine.submitters</name>
+ <name>submarine.submitter</name>
<value>k8s</value>
- <description>Submitter list for the server</description>
- </property>
- <property>
- <name>submarine.submitters.k8s.class</name>
- <value>org.apache.submarine.server.submitter.k8s.K8sJobSubmitter</value>
- <description>The entry class for the specified submitter</description>
- </property>
- <property>
- <name>submarine.submitters.k8s.classpath</name>
- <value>../lib/submitter/k8s/</value>
- <description>The libs for Kubernetes submitter</description>
- </property>
-
- <!-- K8s Configuration -->
- <property>
- <name>submarine.k8s.kube.config</name>
- <value>../conf/k8s/config</value>
- <description>Kube config for kubernetes, you should get the config from
cluster</description>
+ <description>The submitter which you want used in the server. Build-in
k8s</description>
</property>
</configuration>
diff --git a/conf/submarine-site.xml.template b/conf/submarine-site.xml.template
index e4caa2d..887a408 100755
--- a/conf/submarine-site.xml.template
+++ b/conf/submarine-site.xml.template
@@ -155,28 +155,9 @@
<description>Rpc server port</description>
</property>
- <!-- Submarine Submitters Configuration -->
- <property>
- <name>submarine.submitters</name>
- <value>k8s</value>
- <description>Submitter list for the server</description>
- </property>
- <property>
- <name>submarine.submitters.k8s.class</name>
- <value>org.apache.submarine.server.submitter.k8s.K8sJobSubmitter</value>
- <description>The entry class for the specified submitter</description>
- </property>
- <property>
- <name>submarine.submitters.k8s.classpath</name>
- <value>../lib/submitter/k8s/</value>
- <description>The libs for Kubernetes submitter</description>
- </property>
-
- <!-- K8s Configuration -->
- <property>
- <name>submarine.k8s.kube.config</name>
- <value>../conf/k8s/config</value>
- <description>Kube config for kubernetes, you should get the config from
cluster</description>
- </property>
-
+ <property>
+ <name>submarine.submitter</name>
+ <value>k8s</value>
+ <description>The submitter which you want used in the server. Build-in
k8s</description>
+ </property>
</configuration>
diff --git a/docs/submarine-server/README.md b/docs/submarine-server/README.md
index c1fc875..9afc3cf 100644
--- a/docs/submarine-server/README.md
+++ b/docs/submarine-server/README.md
@@ -37,6 +37,7 @@ Job spec consists of `librarySpec`, `submitterSpec` and
`taskSpecs`. Below are e
### Sample Tensorflow Spec
```yaml
name: "mnist"
+namespace: "submarine"
librarySpec:
name: "TensorFlow"
version: "2.1.0"
@@ -44,10 +45,6 @@ librarySpec:
cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150"
envVars:
ENV_1: "ENV1"
-submitterSpec:
- type: "k8s"
- configPath:
- namespace: "submarine"
taskSpecs:
Ps:
name: tensorflow
@@ -62,6 +59,7 @@ or
```json
{
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -71,10 +69,6 @@ or
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Ps": {
"name": "tensorflow",
@@ -95,6 +89,7 @@ or
```json
{
"name": "pytorch-dist-mnist-gloo",
+ "namespace": "submarine",
"librarySpec": {
"name": "pytorch",
"version": "2.1.0",
@@ -104,10 +99,6 @@ or
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Master": {
"name": "master",
@@ -134,6 +125,7 @@ For more info about the spec definition see
[here](../design/submarine-server/jo
curl -X POST -H "Content-Type: application/json" -d '
{
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -143,10 +135,6 @@ curl -X POST -H "Content-Type: application/json" -d '
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -171,6 +159,7 @@ curl -X POST -H "Content-Type: application/json" -d '
"acceptedTime": "2020-04-06T14:59:29.000+08:00",
"spec": {
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -180,10 +169,6 @@ curl -X POST -H "Content-Type: application/json" -d '
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -223,6 +208,7 @@ curl -X GET http://127.0.0.1/api/v1/jobs
"createdTime": "2020-04-06T14:59:29.000+08:00",
"spec": {
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -232,10 +218,6 @@ curl -X GET http://127.0.0.1/api/v1/jobs
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -275,6 +257,7 @@ curl -X GET
http://127.0.0.1/api/v1/jobs/job_1586156073228_0005
"createdTime": "2020-04-06T14:59:29.000+08:00",
"spec": {
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -284,10 +267,6 @@ curl -X GET
http://127.0.0.1/api/v1/jobs/job_1586156073228_0005
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -312,6 +291,7 @@ curl -X GET
http://127.0.0.1/api/v1/jobs/job_1586156073228_0005
curl -X PATCH -H "Content-Type: application/json" -d '
{
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -321,10 +301,6 @@ curl -X PATCH -H "Content-Type: application/json" -d '
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -351,6 +327,7 @@ curl -X PATCH -H "Content-Type: application/json" -d '
"createdTime": "2020-04-06T14:59:29.000+08:00",
"spec": {
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -360,10 +337,6 @@ curl -X PATCH -H "Content-Type: application/json" -d '
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
@@ -402,6 +375,7 @@ curl -X DELETE http://127.0.0.1/api/v1/jobs/job_123_01
"createdTime": "2020-04-06T14:59:29.000+08:00",
"spec": {
"name": "mnist",
+ "namespace": "submarine",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -411,10 +385,6 @@ curl -X DELETE http://127.0.0.1/api/v1/jobs/job_123_01
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
index 99e624c..de57152 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
@@ -68,10 +68,7 @@ public class SubmarineConfVars {
WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
SUBMARINE_RUNTIME_CLASS("submarine.runtime.class",
"org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
- SUBMARINE_SUBMITTERS("submarine.submitters", ""),
- SUBMARINE_SUBMITTERS_CLASS("submarine.submitters.%s.class", ""),
- SUBMARINE_SUBMITTERS_CLASSPATH("submarine.submitters.%s.classpath", ""),
- SUBMARINE_K8S_KUBE_CONFIG("submarine.k8s.kube.config", "");
+ SUBMARINE_SUBMITTER("submarine.submitter", "k8s");
private String varName;
@SuppressWarnings("rawtypes")
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 6ed4fb1..1060310 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -28,11 +28,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.StringTokenizer;
public class SubmarineConfiguration extends XMLConfiguration {
private static final Logger LOG =
LoggerFactory.getLogger(SubmarineConfiguration.class);
@@ -313,45 +311,6 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return
getString(SubmarineConfVars.ConfVars.WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
- /**
- * Get all submitters from configuration file
- * @return list
- */
- public List<String> listSubmitter() {
- List<String> values = new ArrayList<>();
-
- String submitters =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS);
- if (submitters != null) {
- final String delim = ",";
- StringTokenizer tokenizer = new StringTokenizer(submitters, delim);
- while (tokenizer.hasMoreTokens()) {
- values.add(tokenizer.nextToken());
- }
- }
-
- return values;
- }
-
- /**
- * Get the submitter class name by the specified name
- * @param name the submitter's name
- * @return class name
- */
- public String getSubmitterClass(String name) {
- return
getStringValue(String.format(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS_CLASS.getVarName(),
- name), "");
- }
-
- /**
- * Get the submitter's classpath by the specified name
- * @param name the submitter's name
- * @return classpath
- */
- public String getSubmitterClassPath(String name) {
- return
getStringValue(String.format(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS_CLASSPATH.
- getVarName(), name), "");
- }
-
private String getStringValue(String name, String d) {
String value = this.properties.get(name);
if (value != null) {
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
index f792825..876662c 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
@@ -31,17 +31,14 @@ public class JobSpec {
*/
private String name;
+ private String namespace;
+
/**
* The user-specified ML framework spec.
*/
private JobLibrarySpec librarySpec;
/**
- * The user-specified submitter spec.
- */
- private JobSubmitterSpec submitterSpec;
-
- /**
* The tasks of the job.
* Such as:
* TensorFlow: Chief, Ps, Worker, Evaluator
@@ -65,6 +62,14 @@ public class JobSpec {
this.name = name;
}
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
/**
* Get the library spec.
* @return JobLibrarySpec
@@ -78,18 +83,6 @@ public class JobSpec {
}
/**
- * Get the submitter spec.
- * @return JobSubmitterSpec
- */
- public JobSubmitterSpec getSubmitterSpec() {
- return submitterSpec;
- }
-
- public void setSubmitterSpec(JobSubmitterSpec submitterSpec) {
- this.submitterSpec = submitterSpec;
- }
-
- /**
* Get all tasks spec
* @return Map of JobTaskSpec
*/
@@ -103,7 +96,6 @@ public class JobSpec {
public boolean validate() {
return librarySpec != null && librarySpec.validate()
- && submitterSpec != null && submitterSpec.validate()
&& taskSpecs != null;
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
deleted file mode 100644
index 5dd7066..0000000
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.api.spec;
-
-/**
- * The spec of the specified JobSubmitter
- */
-public class JobSubmitterSpec {
- /**
- * The type of JobSubmitter which will be selected to submit job. Such as:
yarn/yarnservice/k8s
- */
- private String type;
-
- /**
- * The config path of the specified Resource Manager.
- */
- private String configPath;
-
- /**
- * It known as queue in Apache Hadoop YARN and namespace in Kubernetes.
- */
- private String namespace;
-
- private String kind;
-
- private String apiVersion;
-
- public JobSubmitterSpec() {
-
- }
-
- /**
- * Get the submitter type, which used to select the specified submitter to
submit this job
- * @return submitter type, such as yarn/yarnservice/k8s
- */
- public String getType() {
- return type;
- }
-
- /**
- * Set the submitter type, now supports the yarn/yarnservice/k8s.
- * @param type yarn/yarnservice/k8s
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- * Get the config path to initialize the submitter. In Apache Hadoop YARN
cluster it maybe the
- * path of xxx-site.xml and the kube config for Kubernetes.
- * @return config path
- */
- public String getConfigPath() {
- return configPath;
- }
-
- public void setConfigPath(String configPath) {
- this.configPath = configPath;
- }
-
- /**
- * Get the namespace to submit the job.
- * It known as queue in Apache Hadoop YARN and namespace in Kubernetes.
- * @return namespace
- */
- public String getNamespace() {
- return namespace;
- }
-
- public void setNamespace(String namespace) {
- this.namespace = namespace;
- }
-
- /**
- * (K8s) Get the CRD kind, which will accept the job
- * @return CRD kind, such as: TFJob/PyTorchJob
- */
- public String getKind() {
- return kind;
- }
-
- public void setKind(String kind) {
- this.kind = kind;
- }
-
- /**
- * (K8s) Get the CRD api version
- * @return version, such as: apache.org/submarine/v1
- */
- public String getApiVersion() {
- return apiVersion;
- }
-
- public void setApiVersion(String apiVersion) {
- this.apiVersion = apiVersion;
- }
-
- public boolean validate() {
- return type != null && namespace != null;
- }
-}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
index 77cc524..d3385b2 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
@@ -19,6 +19,7 @@
package org.apache.submarine.server;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.server.api.job.JobSubmitter;
import org.slf4j.Logger;
@@ -30,44 +31,49 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.Map;
/**
- * Submitter Manager, load all the submitter plugin, configured by
- * {@link SubmarineConfiguration.ConfVars#SUBMARINE_SUBMITTERS} and related
key.
+ * Submitter Manager is responsible for load {@link JobSubmitter} implements
class.
*/
public class SubmitterManager {
private static final Logger LOG =
LoggerFactory.getLogger(SubmitterManager.class);
- private SubmarineConfiguration conf;
- private ConcurrentMap<String, JobSubmitter> submitterMap = new
ConcurrentHashMap<>();
+ private static final SubmitterManager INSTANCE = new SubmitterManager();
- public SubmitterManager(SubmarineConfiguration conf) {
- this.conf = conf;
- loadSubmitters();
+ private final Map<String, JobSubmitterConfig> SUBMITTER_CONFIG_MAP = new
HashMap<>();
+
+ private JobSubmitter submitter;
+
+ {
+ String home = System.getenv("SUBMARINE_HOME");
+ LOG.info("Submarine Home: {}", home);
+ SUBMITTER_CONFIG_MAP.put("k8s", new JobSubmitterConfig(
+ "org.apache.submarine.server.submitter.k8s.K8sJobSubmitter",
+ home + "/lib/submitter/k8s/"));
}
- private void loadSubmitters() {
- LOG.info("Start load submitter plugins...");
- List<String> list = conf.listSubmitter();
- for (String name : list) {
- String clazzName = conf.getSubmitterClass(name);
- String classpath = conf.getSubmitterClassPath(name);
- try {
- ClassLoader classLoader = new
URLClassLoader(constructUrlsFromClasspath(classpath));
- Class<?> clazz = Class.forName(clazzName, true, classLoader);
- Class<? extends JobSubmitter> sClass =
clazz.asSubclass(JobSubmitter.class);
- Constructor<? extends JobSubmitter> method =
sClass.getDeclaredConstructor();
- JobSubmitter submitter = method.newInstance();
- submitter.initialize(conf);
- submitterMap.put(submitter.getSubmitterType(), submitter);
- } catch (Exception e) {
- LOG.error(e.toString(), e);
- }
+ public static JobSubmitter loadSubmitter() {
+ return INSTANCE.submitter;
+ }
+
+ private SubmitterManager() {
+ SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
+ String type =
conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTER);
+ String clazzName = SUBMITTER_CONFIG_MAP.get(type).clazzName;
+ String classpath = SUBMITTER_CONFIG_MAP.get(type).classpath;
+ try {
+ ClassLoader classLoader = new
URLClassLoader(constructUrlsFromClasspath(classpath));
+ Class<?> clazz = Class.forName(clazzName, true, classLoader);
+ Class<? extends JobSubmitter> sClass =
clazz.asSubclass(JobSubmitter.class);
+ Constructor<? extends JobSubmitter> method =
sClass.getDeclaredConstructor();
+ submitter = method.newInstance();
+ submitter.initialize(conf);
+ } catch (Exception e) {
+ LOG.error("Initialize the submitter failed. " + e.getMessage(), e);
}
- LOG.info("Success loaded {} submitters.", submitterMap.size());
}
private URL[] constructUrlsFromClasspath(String classpath) throws
MalformedURLException {
@@ -92,11 +98,13 @@ public class SubmitterManager {
return urls.toArray(new URL[0]);
}
- /**
- * Get the specified submitter by submitter type
- * @return submitter
- */
- public JobSubmitter getSubmitterByType(String type) {
- return submitterMap.get(type);
+ private static class JobSubmitterConfig {
+ private final String clazzName;
+ private final String classpath;
+
+ JobSubmitterConfig(String clazzName, String classpath) {
+ this.clazzName = clazzName;
+ this.classpath = classpath;
+ }
}
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/job/JobManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/job/JobManager.java
index df346a0..df95c68 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/job/JobManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/job/JobManager.java
@@ -28,7 +28,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.core.Response.Status;
-import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.SubmarineServer;
import org.apache.submarine.server.SubmitterManager;
@@ -56,7 +55,7 @@ public class JobManager {
*/
private final ConcurrentMap<String, Job> cachedJobMap = new
ConcurrentHashMap<>();
- private SubmitterManager submitterManager;
+ private final JobSubmitter submitter;
/**
* Get the singleton instance
@@ -66,17 +65,15 @@ public class JobManager {
if (manager == null) {
synchronized (JobManager.class) {
if (manager == null) {
- SubmarineConfiguration conf = SubmarineConfiguration.getInstance();
- SubmitterManager submitterManager = new SubmitterManager(conf);
- manager = new JobManager(submitterManager);
+ manager = new JobManager(SubmitterManager.loadSubmitter());
}
}
}
return manager;
}
- private JobManager(SubmitterManager submitterManager) {
- this.submitterManager = submitterManager;
+ private JobManager(JobSubmitter submitter) {
+ this.submitter = submitter;
}
/**
@@ -87,7 +84,6 @@ public class JobManager {
*/
public Job createJob(JobSpec spec) throws SubmarineRuntimeException {
checkSpec(spec);
- JobSubmitter submitter = getSubmitter(spec.getSubmitterSpec().getType());
Job job = submitter.createJob(spec);
job.setJobId(generateJobId());
job.setSpec(spec);
@@ -109,7 +105,7 @@ public class JobManager {
checkJobId(id);
Job job = cachedJobMap.get(id);
JobSpec spec = job.getSpec();
- Job patchJob =
getSubmitter(spec.getSubmitterSpec().getType()).findJob(spec);
+ Job patchJob = submitter.findJob(spec);
job.rebuild(patchJob);
return job;
}
@@ -125,7 +121,6 @@ public class JobManager {
for (Map.Entry<String, Job> entry : cachedJobMap.entrySet()) {
Job job = entry.getValue();
JobSpec spec = job.getSpec();
- JobSubmitter submitter = getSubmitter(spec.getSubmitterSpec().getType());
Job patchJob = submitter.findJob(spec);
LOG.info("Found job: {}", patchJob.getStatus());
if (status == null ||
status.toLowerCase().equals(patchJob.getStatus().toLowerCase())) {
@@ -148,7 +143,6 @@ public class JobManager {
checkJobId(id);
checkSpec(spec);
Job job = cachedJobMap.get(id);
- JobSubmitter submitter = getSubmitter(spec.getSubmitterSpec().getType());
Job patchJob = submitter.patchJob(spec);
job.setSpec(spec);
job.rebuild(patchJob);
@@ -165,20 +159,11 @@ public class JobManager {
checkJobId(id);
Job job = cachedJobMap.remove(id);
JobSpec spec = job.getSpec();
- Job patchJob =
getSubmitter(spec.getSubmitterSpec().getType()).deleteJob(spec);
+ Job patchJob = submitter.deleteJob(spec);
job.rebuild(patchJob);
return job;
}
- private JobSubmitter getSubmitter(String type) throws
SubmarineRuntimeException {
- JobSubmitter submitter = submitterManager.getSubmitterByType(type);
- if (submitter == null) {
- throw new
SubmarineRuntimeException(Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- "Invalid submitter.");
- }
- return submitter;
- }
-
private void checkSpec(JobSpec spec) throws SubmarineRuntimeException {
if (spec == null) {
throw new SubmarineRuntimeException(Status.OK.getStatusCode(), "Invalid
job spec.");
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
index 37e0e86..557ebaa 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
@@ -22,7 +22,6 @@ package org.apache.submarine.server.submitter.k8s;
import java.io.FileReader;
import java.io.IOException;
-import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import io.kubernetes.client.ApiClient;
@@ -33,7 +32,6 @@ import io.kubernetes.client.apis.CustomObjectsApi;
import io.kubernetes.client.models.V1Status;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
-import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.exception.InvalidSpecException;
@@ -50,50 +48,36 @@ import org.slf4j.LoggerFactory;
* JobSubmitter for Kubernetes Cluster.
*/
public class K8sJobSubmitter implements JobSubmitter {
- private final Logger LOG = LoggerFactory.getLogger(K8sJobSubmitter.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(K8sJobSubmitter.class);
- private String confPath;
+ private static final String KUBECONFIG_ENV = "KUBECONFIG";
// K8s API client for CRD
private CustomObjectsApi api;
public K8sJobSubmitter() {}
- @VisibleForTesting
- public K8sJobSubmitter(String confPath) {
- this.confPath = confPath;
- }
-
@Override
public void initialize(SubmarineConfiguration conf) {
- if (confPath == null || confPath.trim().isEmpty()) {
- confPath = conf.getString(
- SubmarineConfVars.ConfVars.SUBMARINE_K8S_KUBE_CONFIG);
- }
- loadClientConfiguration(confPath);
- if (api == null) {
- api = new CustomObjectsApi();
- }
- }
-
- private void loadClientConfiguration(String path) {
+ ApiClient client = null;
try {
+ String path = System.getenv(KUBECONFIG_ENV);
KubeConfig config = KubeConfig.loadKubeConfig(new FileReader(path));
- ApiClient client = ClientBuilder.kubeconfig(config).build();
- Configuration.setDefaultApiClient(client);
+ client = ClientBuilder.kubeconfig(config).build();
} catch (Exception e) {
- LOG.warn("Failed to load the configured K8s kubeconfig file: " +
- e.getMessage(), e);
-
- LOG.info("Assume running in the k8s cluster, " +
- "try to load in-cluster config");
+ LOG.info("Maybe in cluster mode, try to initialize the client again.");
try {
- ApiClient client = ClientBuilder.cluster().build();
- Configuration.setDefaultApiClient(client);
+ client = ClientBuilder.cluster().build();
} catch (IOException e1) {
LOG.error("Initialize K8s submitter failed. " + e.getMessage(), e1);
throw new SubmarineRuntimeException(500, "Initialize K8s submitter
failed.");
}
+ } finally {
+ Configuration.setDefaultApiClient(client);
+ }
+
+ if (api == null) {
+ api = new CustomObjectsApi();
}
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/JobSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/JobSpecParser.java
index 8ed64c4..162bca5 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/JobSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/JobSpecParser.java
@@ -112,7 +112,7 @@ public class JobSpecParser {
private static V1ObjectMeta parseMetadata(JobSpec jobSpec) {
V1ObjectMeta meta = new V1ObjectMeta();
- meta.setNamespace(jobSpec.getSubmitterSpec().getNamespace());
+ meta.setNamespace(jobSpec.getNamespace());
meta.setName(jobSpec.getName());
return meta;
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/JobSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/JobSpecParserTest.java
index 36bb28a..e9d38a4 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/JobSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/JobSpecParserTest.java
@@ -44,7 +44,7 @@ public class JobSpecParserTest extends SpecBuilder {
JobSpec jobSpec = buildFromJsonFile(tfJobReqFile);
TFJob tfJob = (TFJob) JobSpecParser.parseJob(jobSpec);
// namespace
- String expectedNS = jobSpec.getSubmitterSpec().getNamespace();
+ String expectedNS = jobSpec.getNamespace();
String actualNS = tfJob.getMetadata().getNamespace();
Assert.assertEquals(expectedNS, actualNS);
// job name
@@ -91,7 +91,7 @@ public class JobSpecParserTest extends SpecBuilder {
JobSpec jobSpec = buildFromJsonFile(pytorchJobReqFile);
PyTorchJob pyTorchJob = (PyTorchJob) JobSpecParser.parseJob(jobSpec);
// namespace
- String expectedNS = jobSpec.getSubmitterSpec().getNamespace();
+ String expectedNS = jobSpec.getNamespace();
String actualNS = pyTorchJob.getMetadata().getNamespace();
Assert.assertEquals(expectedNS, actualNS);
// job name
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 3191f4e..a98fb09 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -19,14 +19,10 @@
package org.apache.submarine.server.submitter.k8s;
-import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import io.kubernetes.client.ApiException;
-import io.kubernetes.client.apis.CoreV1Api;
-import io.kubernetes.client.models.V1Namespace;
-import io.kubernetes.client.models.V1NamespaceList;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.job.Job;
import org.apache.submarine.server.api.spec.JobSpec;
@@ -38,10 +34,9 @@ import org.junit.Test;
* We have two ways to test submitter for K8s cluster, local and travis CI.
* <p>
* For running the tests locally, ensure that:
- * 1. There's a k8s cluster running somewhere
- * 2. There's a "config" file as ~/.kube/config targeting the cluster
- * 3. A namespace submarine was created
- * 3. The CRDs was created beforehand. The operator doesn't needs to be
running.
+ * 1. There's a K8s cluster running somewhere
+ * 2. Had set the env KUBECONFIG variable
+ * 3. The CRDs was created in default namespace. The operator doesn't needs to
be running.
* <p>
* Use "kubectl -n submarine get tfjob" or "kubectl -n submarine get
pytorchjob"
* to check the status if you comment the deletion job code in method "after()"
@@ -56,28 +51,9 @@ public class K8SJobSubmitterTest extends SpecBuilder {
private K8sJobSubmitter submitter;
@Before
- public void before() throws IOException, ApiException {
- String confPath = System.getProperty("user.home") + "/.kube/config";
- if (!new File(confPath).exists()) {
- throw new IOException("Get kube config file failed.");
- }
- submitter = new K8sJobSubmitter(confPath);
+ public void before() throws ApiException {
+ submitter = new K8sJobSubmitter();
submitter.initialize(null);
- String ns = "submarine";
- if (!isEnvReady()) {
- throw new ApiException("Please create a namespace 'submarine'.");
- }
- }
-
- private boolean isEnvReady() throws ApiException {
- CoreV1Api api = new CoreV1Api();
- V1NamespaceList list = api.listNamespace(null, null, null, null, null,
null, null, null);
- for (V1Namespace item : list.getItems()) {
- if (item.getMetadata().getName().equals("submarine")) {
- return true;
- }
- }
- return false;
}
@Test
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req.json
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req.json
index a8be643..d7288ea 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req.json
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/pytorch_job_req.json
@@ -1,5 +1,6 @@
{
"name": "pytorch-dist-mnist-gloo",
+ "namespace": "default",
"librarySpec": {
"name": "pytorch",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Master": {
"name": "master",
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_mnist_req.json
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_mnist_req.json
index 13fc934..137b03a 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_mnist_req.json
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_mnist_req.json
@@ -1,5 +1,6 @@
{
"name": "mnist",
+ "namespace": "default",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "submarine"
- },
"taskSpecs": {
"Ps": {
"name": "tensorflow",
diff --git
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
index f9a0fb0..35aff5b 100644
---
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
+++
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/JobManagerRestApiIT.java
@@ -243,20 +243,6 @@ public class JobManagerRestApiIT extends
AbstractSubmarineServerTest {
}
@Test
- public void testCreateJobWithInvalidSubmitter() throws Exception {
- String body = loadContent("tensorflow/tf-mnist-req.json");
- body = body.replace("k8s", "InvalidSubmitter");
- PostMethod postMethod = httpPost(JOB_PATH, body,
MediaType.APPLICATION_JSON);
- Assert.assertEquals(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- postMethod.getStatusCode());
-
- String json = postMethod.getResponseBodyAsString();
- JsonResponse jsonResponse = gson.fromJson(json, JsonResponse.class);
- Assert.assertEquals(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- jsonResponse.getCode());
- }
-
- @Test
public void testNotFoundJob() throws Exception {
GetMethod getMethod = httpGet(JOB_PATH + "/" + "job_123456789_0001");
Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(),
getMethod.getStatusCode());
diff --git
a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.json
b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.json
index dd4467c..15878ad 100644
--- a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.json
+++ b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.json
@@ -1,5 +1,6 @@
{
"name": "mnist",
+ "namespace": "default",
"librarySpec": {
"name": "pytorch",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "default"
- },
"taskSpecs": {
"Master": {
"name": "master",
diff --git
a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.yaml
b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.yaml
index 8b47741..d10463a 100644
--- a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.yaml
+++ b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-patch-req.yaml
@@ -1,5 +1,6 @@
---
name: mnist
+namespace: default
librarySpec:
name: pytorch
version: 2.1.0
@@ -7,9 +8,6 @@ librarySpec:
cmd: python /var/mnist.py --backend gloo
envVars:
ENV_1: ENV1
-submitterSpec:
- type: k8s
- namespace: default
taskSpecs:
Master:
name: master
diff --git
a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.json
b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.json
index cb884bf..0f1c3fa 100644
--- a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.json
+++ b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.json
@@ -1,5 +1,6 @@
{
"name": "mnist",
+ "namespace": "default",
"librarySpec": {
"name": "pytorch",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "default"
- },
"taskSpecs": {
"Master": {
"name": "master",
diff --git
a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.yaml
b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.yaml
index 414d6ef..8e18fc8 100644
--- a/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.yaml
+++ b/submarine-test/test-k8s/src/test/resources/pytorch/pt-mnist-req.yaml
@@ -1,5 +1,6 @@
---
name: mnist
+namespace: default
librarySpec:
name: pytorch
version: 2.1.0
@@ -7,9 +8,6 @@ librarySpec:
cmd: python /var/mnist.py --backend gloo
envVars:
ENV_1: ENV1
-submitterSpec:
- type: k8s
- namespace: default
taskSpecs:
Master:
name: master
diff --git
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.json
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.json
index d8df243..b85aca7 100644
---
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.json
+++
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.json
@@ -1,5 +1,6 @@
{
"name": "mnist",
+ "namespace": "default",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "default"
- },
"taskSpecs": {
"Worker": {
"name": "tensorflow",
diff --git
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.yaml
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.yaml
index edae4cb..2bb4d0c 100644
---
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.yaml
+++
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-patch-req.yaml
@@ -1,4 +1,5 @@
name: "mnist"
+namespace: "default"
librarySpec:
name: "TensorFlow"
version: "2.1.0"
@@ -6,12 +7,6 @@ librarySpec:
cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150"
envVars:
ENV_1: "ENV1"
-submitterSpec:
- type: "k8s"
- configPath:
- namespace: "default"
- kind: "TFJob"
- apiVersion: "kubeflow.org/v1"
taskSpecs:
Worker:
name: tensorflow
diff --git
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.json
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.json
index d7f7473..9c1be42 100644
--- a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.json
+++ b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.json
@@ -1,5 +1,6 @@
{
"name": "mnist",
+ "namespace": "default",
"librarySpec": {
"name": "TensorFlow",
"version": "2.1.0",
@@ -9,10 +10,6 @@
"ENV_1": "ENV1"
}
},
- "submitterSpec": {
- "type": "k8s",
- "namespace": "default"
- },
"taskSpecs": {
"Ps": {
"name": "tensorflow",
diff --git
a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.yaml
b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.yaml
index bf729a6..74db4d3 100644
--- a/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.yaml
+++ b/submarine-test/test-k8s/src/test/resources/tensorflow/tf-mnist-req.yaml
@@ -1,4 +1,5 @@
name: "mnist"
+namespace: "default"
librarySpec:
name: "TensorFlow"
version: "2.1.0"
@@ -6,11 +7,6 @@ librarySpec:
cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150"
envVars:
ENV_1: "ENV1"
-submitterSpec:
- type: "k8s"
- namespace: "default"
- kind: "TFJob"
- apiVersion: "kubeflow.org/v1"
taskSpecs:
Ps:
name: tensorflow
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
