This is an automated email from the ASF dual-hosted git repository.
zhouquan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new a3ebce1 SUBMARINE-174. Add travis support for K8s submitter
a3ebce1 is described below
commit a3ebce18da951e69045dd54014beb1c93b13a4a0
Author: Wanqiang Ji <[email protected]>
AuthorDate: Thu Nov 21 23:48:56 2019 +0800
SUBMARINE-174. Add travis support for K8s submitter
### What is this PR for?
K8s submitter is ready for POC, but cannot run the UTs in travis.
### What type of PR is it?
[ Improvement ]
### Todos
https://issues.apache.org/jira/browse/SUBMARINE-300
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-174
### How should this be tested?
https://travis-ci.org/jiwq/submarine/builds/616270819
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Wanqiang Ji <[email protected]>
Closes #100 from jiwq/SUBMARINE-174 and squashes the following commits:
ffb806e [Wanqiang Ji] SUBMARINE-174. Add travis support for K8s submitter
---
.travis.yml | 48 +++++++++++++++++-----
dev-support/travis/tf-operator/crd_v1.yaml | 42 +++++++++++++++++++
.../tf-operator}/tfevent-volume/tfevent-pv.yaml | 0
.../tf-operator}/tfevent-volume/tfevent-pvc.yaml | 0
.../server/submitter/k8s/K8sJobSubmitter.java | 1 +
.../server/submitter/k8s/K8SJobSubmitterTest.java | 39 +++++++++++-------
.../submitter-k8s/src/test/resources/config | 19 ---------
7 files changed, 104 insertions(+), 45 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index ac95101..0c0309d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-sudo: false
+sudo: required
before_cache:
- sudo chown -R travis:travis $HOME/.m2
@@ -41,10 +41,11 @@ services:
env:
global:
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add
`!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
- -
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s,!submarine-server/server-core"
+ -
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-core"
-
EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web"
-
EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter""
-
EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli"
+ - EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
before_install:
# maven 3.6.1 (3.6.2 build tony failed!!!)
@@ -73,25 +74,25 @@ matrix:
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Build hadoop-2.7" PROFILE="-Phadoop-2.7" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
+ env: NAME="Build hadoop-2.7" PROFILE="-Phadoop-2.7" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
\"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
# Build hadoop-2.9(default)
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Build hadoop-2.9" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
+ env: NAME="Build hadoop-2.9" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
\"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
# Build hadoop-3.1
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Build hadoop-3.1" PROFILE="-Phadoop-3.1" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
+ env: NAME="Build hadoop-3.1" PROFILE="-Phadoop-3.1" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
\"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
# Build hadoop-3.2
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Build hadoop-3.2" PROFILE="-Phadoop-3.2" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl \"${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
+ env: NAME="Build hadoop-3.2" PROFILE="-Phadoop-3.2" BUILD_FLAG="clean
package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
\"${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},!submarine-dist\""
TEST_PROJECTS=""
# Build workbench-web
- language: node_js
@@ -110,7 +111,7 @@ matrix:
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Test workbench-web" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean
package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl
${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl
submarine-workbench/workbench-web" TEST_PROJECTS=""
+ env: NAME="Test workbench-web" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean
package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl
${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl
submarine-workbench/workbench-web" TEST_PROJECTS=""
# Test test-e2e
- language: java
@@ -122,7 +123,7 @@ matrix:
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Test interpreter" PYTHON="3" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH},${EXCLUDE_SUBMODULE_TONY}"
TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')"
TEST_PROJECTS=""
+ env: NAME="Test interpreter" PYTHON="3" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl
${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH},${EXCLUDE_SUBMODULE_TONY}"
TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')"
TEST_PROJECTS=""
# Test submarine-sdk
- language: python
@@ -139,7 +140,7 @@ matrix:
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Test submarine distribution" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="" TEST_MODULES="" TEST_PROJECTS=""
+ env: NAME="Test submarine distribution" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl ${EXCLUDE_K8S}" TEST_MODULES="-pl ${EXCLUDE_K8S}" TEST_PROJECTS=""
# Test submarine web-ng
- language: node_js
@@ -160,12 +161,37 @@ matrix:
- npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
env: NAME="Build workbench-web-ng"
-
# Test submarine-server
- language: java
jdk: "openjdk8"
dist: xenial
- env: NAME="Test submarine-server" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl ${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl
submarine-server/server-core" TEST_PROJECTS=""
+ env: NAME="Test submarine-server" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}"
TEST_MODULES="-pl submarine-server/server-core" TEST_PROJECTS=""
+
+ - name: Submarine on Kubernetes
+ dist: xenial
+ services: docker
+ language: java
+ jdk: openjdk8
+ env: BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test
-am" MODULES="-pl org.apache.submarine:submitter-k8s" TEST_MODULES="-pl
org.apache.submarine:submitter-k8s" TEST_PROJECTS=""
+ before_install:
+ # deploy Kubernetes cluster
+ - curl -LO
https://storage.googleapis.com/kubernetes-release/release/$(curl -s
https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+ - chmod +x kubectl
+ - sudo mv kubectl /usr/bin
+ - docker run -d --name kube --privileged -p 8443:8443 -p 10080:10080
bsycorp/kind:latest-1.15
+ - until curl -s --fail http://127.0.0.1:10080/kubernetes-ready; do
+ sleep 1;
+ done
+ - echo "Kubernetes ready - run tests!"
+ - mkdir $HOME/.kube
+ - curl http://127.0.0.1:10080/config > $HOME/.kube/config
+ - export KUBECONFIG=$HOME/.kube/config
+ - echo $KUBECONFIG
+ - kubectl config set clusters.kubernetes.server https://127.0.0.1:8443
+ - kubectl get nodes
+ - kubectl create namespace kubeflow
+ - kubectl apply -f ./dev-support/travis/tf-operator/crd_v1.yaml
+ - kubectl apply -f ./dev-support/travis/tf-operator/tfevent-volume/.
install:
- mvn --version
diff --git a/dev-support/travis/tf-operator/crd_v1.yaml
b/dev-support/travis/tf-operator/crd_v1.yaml
new file mode 100644
index 0000000..546e66a
--- /dev/null
+++ b/dev-support/travis/tf-operator/crd_v1.yaml
@@ -0,0 +1,42 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: tfjobs.kubeflow.org
+spec:
+ group: kubeflow.org
+ scope: Namespaced
+ names:
+ kind: TFJob
+ singular: tfjob
+ plural: tfjobs
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ subresources:
+ status: {}
+ validation:
+ openAPIV3Schema:
+ properties:
+ spec:
+ properties:
+ tfReplicaSpecs:
+ properties:
+ # The validation works when the configuration contains
+ # `Worker`, `PS` or `Chief`. Otherwise it will not be
validated.
+ Worker:
+ properties:
+ replicas:
+ type: integer
+ minimum: 1
+ PS:
+ properties:
+ replicas:
+ type: integer
+ minimum: 1
+ Chief:
+ properties:
+ replicas:
+ type: integer
+ minimum: 1
+ maximum: 1
\ No newline at end of file
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml
b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pv.yaml
similarity index 100%
rename from
submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml
rename to dev-support/travis/tf-operator/tfevent-volume/tfevent-pv.yaml
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml
b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pvc.yaml
similarity index 100%
rename from
submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml
rename to dev-support/travis/tf-operator/tfevent-volume/tfevent-pvc.yaml
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
index 90ec873..0e6f0ad 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
@@ -85,6 +85,7 @@ public class K8sJobSubmitter {
return null;
}
+ @VisibleForTesting
CustomResourceJob deleteCustomResourceJob(K8sJobRequest request) {
try {
CustomObjectsApi api = new CustomObjectsApi();
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 27b41fa..136d30c 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -21,6 +21,7 @@ package org.apache.submarine.server.submitter.k8s;
import org.apache.submarine.server.submitter.k8s.model.CustomResourceJob;
import org.apache.submarine.server.submitter.k8s.model.CustomResourceJobList;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -34,9 +35,20 @@ public class K8SJobSubmitterTest {
private K8sJobSubmitter submitter;
private K8sJobRequest.Path path;
+ /**
+ * We have two ways to test submitter for K8s cluster, local and travis CI.
+ * For the travis CI, we use the kind to setup K8s, more info see
'.travis.yml' file.
+ * Local: docker run -it --privileged -p 8443:8443 -p 10080:10080
bsycorp/kind:latest-1.15
+ * Travis: See '.travis.yml'
+ * @throws IOException IO
+ */
@Before
public void before() throws IOException {
- String confPath = this.getClass().getResource("/config").getFile();
+ String confPath = System.getProperty("user.home") + "/.kube/config";
+ if (!new File(confPath).exists()) {
+ throw new IOException("Get kube config file failed.");
+ }
+
submitter = new K8sJobSubmitter(confPath);
path = new K8sJobRequest.Path("kubeflow.org", "v1", "kubeflow", "tfjobs");
}
@@ -46,12 +58,11 @@ public class K8SJobSubmitterTest {
if (getCustomJob() != null) {
K8sJobRequest request = new K8sJobRequest(path, null, jobName);
CustomResourceJob delJob = submitter.deleteCustomResourceJob(request);
- // Assert.assertNotNull(delJob);
+ Assert.assertNotNull(delJob);
}
- CustomResourceJob job = submitter.createCustomJob(new K8sJobRequest(path,
getCutomJobSpecFile()));
- // Assert.assertNotNull(job);
- System.out.println("Create job: " + job);
+ CustomResourceJob job = submitter.createCustomJob(new K8sJobRequest(path,
getCustomJobSpecFile()));
+ Assert.assertNotNull(job);
}
@Test
@@ -59,29 +70,27 @@ public class K8SJobSubmitterTest {
testCreateCustomJob();
CustomResourceJob job = getCustomJob();
- // Assert.assertNotNull(job);
- // Assert.assertEquals(job.getMetadata().getName(), jobName);
- System.out.println("Get Job: " + job);
+ Assert.assertNotNull(job);
+ Assert.assertEquals(job.getMetadata().getName(), jobName);
}
@Test
public void testListCustomJobs() throws URISyntaxException {
CustomResourceJobList list
- = submitter.listCustomResourceJobs(new K8sJobRequest(path,
getCutomJobSpecFile()));
- System.out.println("Job List: " + list);
+ = submitter.listCustomResourceJobs(new K8sJobRequest(path,
getCustomJobSpecFile()));
+ Assert.assertNotNull(list);
}
@Test
public void testDeleteCustomJob() throws URISyntaxException {
if (getCustomJob() == null) {
- CustomResourceJob job = submitter.createCustomJob(new
K8sJobRequest(path, getCutomJobSpecFile()));
- // Assert.assertNotNull(job);
+ CustomResourceJob job = submitter.createCustomJob(new
K8sJobRequest(path, getCustomJobSpecFile()));
+ Assert.assertNotNull(job);
}
K8sJobRequest request = new K8sJobRequest(path, null, jobName);
CustomResourceJob delJob = submitter.deleteCustomResourceJob(request);
- // Assert.assertNotNull(delJob);
- System.out.println("Delete job: " + delJob);
+ Assert.assertNotNull(delJob);
}
private CustomResourceJob getCustomJob() {
@@ -89,7 +98,7 @@ public class K8SJobSubmitterTest {
return submitter.getCustomResourceJob(request);
}
- private File getCutomJobSpecFile() throws URISyntaxException {
+ private File getCustomJobSpecFile() throws URISyntaxException {
URL fileUrl = this.getClass().getResource("/tf_job_mnist.json");
return new File(fileUrl.toURI());
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/resources/config
b/submarine-server/server-submitter/submitter-k8s/src/test/resources/config
deleted file mode 100644
index bc57012..0000000
--- a/submarine-server/server-submitter/submitter-k8s/src/test/resources/config
+++ /dev/null
@@ -1,19 +0,0 @@
-apiVersion: v1
-clusters:
-- cluster:
- certificate-authority-data:
LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1Ea3hNREl6TVRjeE5sb1hEVEk1TURrd09ESXpNVGN4Tmxvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTm8zCmJGbnllQmlhOUlzeHorVWg3cExGMWNaTkNiR0lkWXJxejFFUlRodXcwN2t1QnlDNWp4K2wxOEZCcGkvSVJ0OWYKaXU4MVVGZEVKSStKaEo2bC95Y3RtRitxaXhiMkUyM1ltL29VTGIxOVh1V3RuZXZHZkNNZFZiUVRv
[...]
- server: https://localhost:8443
- name: kind
-contexts:
-- context:
- cluster: kind
- user: kind-admin
- name: kind
-current-context: kind
-kind: Config
-preferences: {}
-users:
-- name: kind-admin
- user:
- client-certificate-data:
LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJVzVPRkVhcmhlSU13RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2JXbHVhV3QxWW1WRFFUQWVGdzB4T1RBNU1UQXlNekUzTVRaYUZ3MHlNREE1TVRBeU16RTNORFJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXUyRWJyaVMraDZ4MzdaeXYKMnpvZWlUalI5Zm4rYSt3ZTJ4TXFHaDJVMG1vRGV6Y251TkdXWUZXUWpob0VrOEhnQTJGMmFZbEhJeEM
[...]
- client-key-data:
LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBdTJFYnJpUytoNngzN1p5djJ6b2VpVGpSOWZuK2Erd2UyeE1xR2gyVTBtb0RlemNuCnVOR1dZRldRamhvRWs4SGdBMkYyYVlsSEl4QzBvTTNpejlHQnFhcjRjUWNFV2p6dlI5WkczaXBnWDg1OTRYQkgKUko4b1ZSdDl1ZW5hT01PTjBlUVZaWFhycE1OZXY2MCtIeWJPbk5NQWFsUnRGN1pxeHhFblU3RGVwZTdmSXJ1ZApWZDFvNVU3ZndsUEt1WmJZREVoYzdCL2puMjVtV1BJNURPK1kxQTNTREVwQklxZVVaUVFjZUJvbnVNZTF0amo0CkxNeUhZZE5pNFZyb1NoZlFpMVluQS9WM1NwV2pNaitGQUYraDhaNWg3MzlUOXNUMWxCbjJ2MGtmQUpqS3R
[...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
