This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 7576a5e SUBMARINE-330. Pack the k8s submitter libs into
lib/submitter/k8s
7576a5e is described below
commit 7576a5e300b4659c3a8a36c035e125d7a5c7f44e
Author: Wanqiang Ji <[email protected]>
AuthorDate: Fri Jan 3 21:12:06 2020 +0800
SUBMARINE-330. Pack the k8s submitter libs into lib/submitter/k8s
### What is this PR for?
* add default configuration for k8s submitter
* pack the k8s submitter and dependencies to lib/submitter/k8s
* fix k8s submitter parser `JobSpec` used the error `plural` name
* remove redundant `"` in travis caused CI/CD error
### What type of PR is it?
Feature
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-330
### How should this be tested?
https://travis-ci.com/jiwq/submarine/builds/142970359
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Wanqiang Ji <[email protected]>
Closes #141 from jiwq/SUBMARINE-330 and squashes the following commits:
1657120 [Wanqiang Ji] SUBMARINE-330. Pack the k8s submitter libs into
lib/submitter/k8s
---
.travis.yml | 2 +-
conf/submarine-site.xml | 24 ++++++++++++++
.../commons/utils/SubmarineConfiguration.java | 11 ++++---
submarine-dist/src/assembly/distribution.xml | 9 +++++-
submarine-server/server-api/pom.xml | 7 ++++
.../apache/submarine/server/api/JobSubmitter.java | 4 ++-
.../apache/submarine/server/SubmitterManager.java | 10 +++---
.../server-submitter/submitter-k8s/pom.xml | 28 ++++++++++++++++
.../server/submitter/k8s/K8sJobSubmitter.java | 37 ++++++++++++++++------
.../server/submitter/k8s/K8SJobSubmitterTest.java | 1 +
10 files changed, 111 insertions(+), 22 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 1c94b8d..75000b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,7 +43,7 @@ env:
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add
`!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
-
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-core,!submarine-server/server-rpc"
-
EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-web-ng"
- -
EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter""
+ -
EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter"
-
EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli"
- EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
- EXCLUDE_COMMON_RPC="!submarine-commons/commons-rpc"
diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml
index f369659..da74af0 100755
--- a/conf/submarine-site.xml
+++ b/conf/submarine-site.xml
@@ -113,4 +113,28 @@
<description>RuntimeFactory for Submarine jobs</description>
</property>
+ <!-- Submarine Submitters Configuration -->
+ <property>
+ <name>submarine.submitters</name>
+ <value>k8s</value>
+ <description>Submitter list for the server</description>
+ </property>
+ <property>
+ <name>submarine.submitters.k8s.class</name>
+ <value>org.apache.submarine.server.submitter.k8s.K8sJobSubmitter</value>
+ <description>The entry class for the specified submitter</description>
+ </property>
+ <property>
+ <name>submarine.submitters.k8s.classpath</name>
+ <value>../lib/submitter/k8s/</value>
+ <description>The libs for Kubernetes submitter</description>
+ </property>
+
+ <!-- K8s Configuration -->
+ <property>
+ <name>submarine.k8s.kube.config</name>
+ <value>../conf/k8s/config</value>
+ <description>Kube config for kubernetes, you should get the config from
cluster</description>
+ </property>
+
</configuration>
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 396f088..4e63a09 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -333,12 +333,12 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
/**
- * Get the entry class name by the specified name
+ * Get the submitter class name by the specified name
* @param name the submitter's name
* @return class name
*/
- public String getSubmitterEntry(String name) {
- return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_ENTRY.getVarName(),
name), "");
+ public String getSubmitterClass(String name) {
+ return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_CLASS.getVarName(),
name), "");
}
/**
@@ -526,8 +526,9 @@ public class SubmarineConfiguration extends
XMLConfiguration {
SUBMARINE_RUNTIME_CLASS("submarine.runtime.class",
"org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
SUBMARINE_SUBMITTERS("submarine.submitters", ""),
- SUBMARINE_SUBMITTERS_ENTRY("submarine.submitters.%s.class", ""),
- SUBMARINE_SUBMITTERS_CLASSPATH("submarine.submitters.%s.classpath", "");
+ SUBMARINE_SUBMITTERS_CLASS("submarine.submitters.%s.class", ""),
+ SUBMARINE_SUBMITTERS_CLASSPATH("submarine.submitters.%s.classpath", ""),
+ SUBMARINE_K8S_KUBE_CONFIG("submarine.k8s.kube.config", "");
private String varName;
@SuppressWarnings("rawtypes")
diff --git a/submarine-dist/src/assembly/distribution.xml
b/submarine-dist/src/assembly/distribution.xml
index cd48f29..703ed5a 100644
--- a/submarine-dist/src/assembly/distribution.xml
+++ b/submarine-dist/src/assembly/distribution.xml
@@ -101,14 +101,21 @@
<include>submarine-commons-rpc-${project.version}.jar</include>
</includes>
</fileSet>
+
+ <!-- Kubernetes Submitter -->
<fileSet>
<directory>../submarine-server/server-submitter/submitter-k8s/target</directory>
- <outputDirectory>/lib/submitter</outputDirectory>
+ <outputDirectory>/lib/submitter/k8s</outputDirectory>
<includes>
<include>submarine-submitter-k8s-${project.version}.jar</include>
</includes>
</fileSet>
<fileSet>
+
<directory>../submarine-server/server-submitter/submitter-k8s/target/dependency</directory>
+ <outputDirectory>/lib/submitter/k8s</outputDirectory>
+ </fileSet>
+
+ <fileSet>
<directory>../submarine-server/server-submitter/submitter-yarn/target</directory>
<outputDirectory>/lib/submitter</outputDirectory>
<includes>
diff --git a/submarine-server/server-api/pom.xml
b/submarine-server/server-api/pom.xml
index a786965..e4e2d94 100644
--- a/submarine-server/server-api/pom.xml
+++ b/submarine-server/server-api/pom.xml
@@ -31,4 +31,11 @@
<artifactId>server-api</artifactId>
<name>Submarine: Server API</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>commons-utils</artifactId>
+ </dependency>
+ </dependencies>
+
</project>
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/JobSubmitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/JobSubmitter.java
index 87f90d4..0fd4563 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/JobSubmitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/JobSubmitter.java
@@ -19,6 +19,8 @@
package org.apache.submarine.server.api;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
+
/**
* The submitter should implement this interface.
*/
@@ -27,7 +29,7 @@ public interface JobSubmitter extends JobHandler {
/**
* Initialize the submitter related code
*/
- void initialize();
+ void initialize(SubmarineConfiguration conf);
/**
* Get the submitter type which is the unique identifier.
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
index 07069c8..36e3d6c 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmitterManager.java
@@ -53,7 +53,7 @@ public class SubmitterManager {
LOG.info("Start load submitter plugins...");
List<String> list = conf.listSubmitter();
for (String name : list) {
- String clazzName = conf.getSubmitterEntry(name);
+ String clazzName = conf.getSubmitterClass(name);
String classpath = conf.getSubmitterClassPath(name);
try {
ClassLoader classLoader = new
URLClassLoader(constructUrlsFromClasspath(classpath));
@@ -61,7 +61,7 @@ public class SubmitterManager {
Class<? extends JobSubmitter> sClass =
clazz.asSubclass(JobSubmitter.class);
Constructor<? extends JobSubmitter> method =
sClass.getDeclaredConstructor();
JobSubmitter submitter = method.newInstance();
- submitter.initialize();
+ submitter.initialize(conf);
submitterMap.put(submitter.getSubmitterType(), submitter);
} catch (Exception e) {
LOG.error(e.toString(), e);
@@ -79,10 +79,10 @@ public class SubmitterManager {
File file = new File(path);
if (file.isDirectory()) {
- String[] items = file.list();
+ File[] items = file.listFiles();
if (items != null) {
- for (String item : items) {
- urls.add(new File(item).toURI().toURL());
+ for (File item : items) {
+ urls.add(item.toURI().toURL());
}
}
} else {
diff --git a/submarine-server/server-submitter/submitter-k8s/pom.xml
b/submarine-server/server-submitter/submitter-k8s/pom.xml
index 96288b9..f79b5d9 100644
--- a/submarine-server/server-submitter/submitter-k8s/pom.xml
+++ b/submarine-server/server-submitter/submitter-k8s/pom.xml
@@ -78,6 +78,34 @@
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependencies-runtime</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-dependencies-system</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>system</includeScope>
+ <excludeTransitive>true</excludeTransitive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
index de946ff..afc187a 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
@@ -20,7 +20,6 @@
package org.apache.submarine.server.submitter.k8s;
import java.io.FileReader;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -34,6 +33,7 @@ import io.kubernetes.client.models.V1DeleteOptions;
import io.kubernetes.client.models.V1DeleteOptionsBuilder;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.server.api.JobSubmitter;
import org.apache.submarine.server.api.exception.UnsupportedJobTypeException;
import org.apache.submarine.server.api.job.Job;
@@ -51,22 +51,41 @@ import org.slf4j.LoggerFactory;
public class K8sJobSubmitter implements JobSubmitter {
private final Logger LOG = LoggerFactory.getLogger(K8sJobSubmitter.class);
+ private String confPath;
+
/**
* Key: kind of CRD, such as TFJob/PyTorchJob
* Value: the CRD api with version
*/
private Map<String, String> supportedCRDMap;
- public K8sJobSubmitter(String confPath) throws IOException {
- ApiClient client =
- ClientBuilder.kubeconfig(KubeConfig.loadKubeConfig(new
FileReader(confPath))).build();
- Configuration.setDefaultApiClient(client);
+ public K8sJobSubmitter() {
+
+ }
+
+ public K8sJobSubmitter(String confPath) {
+ this.confPath = confPath;
}
@Override
- public void initialize() {
+ public void initialize(SubmarineConfiguration conf) {
supportedCRDMap = new HashMap<>();
- supportedCRDMap.put("TFJob", "kubeflow.org/v1");
+ supportedCRDMap.put("TFJob", "tfjobs");
+
+ if (confPath == null || confPath.trim().isEmpty()) {
+ confPath =
conf.getString(SubmarineConfiguration.ConfVars.SUBMARINE_K8S_KUBE_CONFIG);
+ }
+ loadClientConfiguration(confPath);
+ }
+
+ private void loadClientConfiguration(String path) {
+ try {
+ KubeConfig config = KubeConfig.loadKubeConfig(new FileReader(path));
+ ApiClient client = ClientBuilder.kubeconfig(config).build();
+ Configuration.setDefaultApiClient(client);
+ } catch (Exception e){
+ LOG.error("Load the K8s client conf failed: " + e.getMessage(), e);
+ }
}
@Override
@@ -76,7 +95,7 @@ public class K8sJobSubmitter implements JobSubmitter {
@Override
public Job submitJob(JobSpec jobSpec) throws UnsupportedJobTypeException {
- if (!supportedCRDMap.containsKey(jobSpec.getSubmitterSpec().getType())) {
+ if (!supportedCRDMap.containsKey(jobSpec.getSubmitterSpec().getKind())) {
throw new UnsupportedJobTypeException();
}
@@ -91,7 +110,7 @@ public class K8sJobSubmitter implements JobSubmitter {
try {
CustomObjectsApi api = new CustomObjectsApi();
api.createNamespacedCustomObject(job.getGroup(), job.getVersion(),
- job.getMetadata().getNamespace(), job.getMetadata().getName(), job,
"true");
+ job.getMetadata().getNamespace(),
supportedCRDMap.get(job.getKind()), job, "true");
} catch (ApiException e) {
LOG.error("Create {} job: " + e.getMessage(), e);
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 136d30c..163a552 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -50,6 +50,7 @@ public class K8SJobSubmitterTest {
}
submitter = new K8sJobSubmitter(confPath);
+ submitter.initialize(null);
path = new K8sJobRequest.Path("kubeflow.org", "v1", "kubeflow", "tfjobs");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
