This is an automated email from the ASF dual-hosted git repository.
zhouquan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 6f0b152 SUBMARINE-326. Hook REST and JobManager to accept TFJob
6f0b152 is described below
commit 6f0b152f3fb4d17f751be1e8b6ecd625a3e27203
Author: Wanqiang Ji <[email protected]>
AuthorDate: Wed Jan 1 12:33:16 2020 +0800
SUBMARINE-326. Hook REST and JobManager to accept TFJob
### What is this PR for?
Implements submit the TF job to K8s cluster by REST
### What type of PR is it?
Feature
### Todos
Implements the job RUD operates when 0.4.0 released
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-326
### How should this be tested?
unit test
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Wanqiang Ji <[email protected]>
Closes #138 from jiwq/SUBMARINE-326 and squashes the following commits:
c99ac2d [Wanqiang Ji] SUBMARINE-326. Hook REST and JobManager to accept
TFJob
---
.../commons/utils/SubmarineConfiguration.java | 7 +-
submarine-server/server-api/pom.xml | 2 +-
.../org/apache/submarine/server/api/job/JobId.java | 18 +++
.../submarine/server/api/spec/JobLibrarySpec.java | 4 +
.../apache/submarine/server/api/spec/JobSpec.java | 6 +
.../server/api/spec/JobSubmitterSpec.java | 4 +
.../submarine/server/api/spec/JobTaskSpec.java | 7 +-
.../org/apache/submarine/server/JobManager.java | 12 +-
.../server/jobserver/rest/JobServerRestApi.java | 93 ----------------
.../submarine/server/json/JobIdDeserializer.java | 37 +++++++
.../submarine/server/json/JobIdSerializer.java | 35 ++++++
.../submarine/server/response/JsonResponse.java | 8 +-
.../submarine/server/rest/JobManagerRestApi.java | 121 +++++++++++++++++++++
.../provider/YamlEntityProvider.java | 2 +-
.../JobManagerRestApiTest.java} | 46 +++++---
.../src/test/resources/tf-mnist-req.json | 31 ++++++
.../src/test/resources/tf-mnist-req.yaml | 23 ++++
17 files changed, 336 insertions(+), 120 deletions(-)
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 5d6b4c2..396f088 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -320,7 +320,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
public List<String> listSubmitter() {
List<String> values = new ArrayList<>();
- String submitters = getString(ConfVars.SUBMARINE_SUBMITTERS.getVarName());
+ String submitters = getString(ConfVars.SUBMARINE_SUBMITTERS);
if (submitters != null) {
final String delim = ",";
StringTokenizer tokenizer = new StringTokenizer(submitters, delim);
@@ -338,7 +338,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
* @return class name
*/
public String getSubmitterEntry(String name) {
- return
getString(String.format(ConfVars.SUBMARINE_SUBMITTERS_ENTRY.getVarName(),
name));
+ return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_ENTRY.getVarName(),
name), "");
}
/**
@@ -347,7 +347,8 @@ public class SubmarineConfiguration extends
XMLConfiguration {
* @return classpath
*/
public String getSubmitterClassPath(String name) {
- return
getString(String.format(ConfVars.SUBMARINE_SUBMITTERS_CLASSPATH.getVarName(),
name));
+ return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_CLASSPATH.getVarName(),
+ name), "");
}
private String getStringValue(String name, String d) {
diff --git a/submarine-server/server-api/pom.xml
b/submarine-server/server-api/pom.xml
index 84adc7f..a786965 100644
--- a/submarine-server/server-api/pom.xml
+++ b/submarine-server/server-api/pom.xml
@@ -31,4 +31,4 @@
<artifactId>server-api</artifactId>
<name>Submarine: Server API</name>
-</project>
\ No newline at end of file
+</project>
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/job/JobId.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/job/JobId.java
index bb0a4b6..7026ed0 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/job/JobId.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/job/JobId.java
@@ -29,9 +29,26 @@ public class JobId implements Comparable<JobId> {
private static final int JOB_ID_MIN_DIGITS = 4;
private int id;
+
private long serverTimestamp;
/**
+ * Get the object of JobId.
+ * @param jobId job id string
+ * @return object
+ */
+ public static JobId fromString(String jobId) {
+ if (jobId == null) {
+ return null;
+ }
+ String[] components = jobId.split("\\_");
+ if (components.length != 3) {
+ return null;
+ }
+ return JobId.newInstance(Long.parseLong(components[1]),
Integer.parseInt(components[2]));
+ }
+
+ /**
* Ge the object of JobId.
* @param serverTimestamp the timestamp when the server start
* @param id count
@@ -39,6 +56,7 @@ public class JobId implements Comparable<JobId> {
*/
public static JobId newInstance(long serverTimestamp, int id) {
JobId jobId = new JobId();
+ jobId.setServerTimestamp(serverTimestamp);
jobId.setId(id);
return jobId;
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobLibrarySpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobLibrarySpec.java
index 236c138..96dcdae 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobLibrarySpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobLibrarySpec.java
@@ -116,4 +116,8 @@ public class JobLibrarySpec {
public void setEnvVars(Map<String, String> envVars) {
this.envVars = envVars;
}
+
+ public boolean validate() {
+ return name != null && image != null && cmd != null;
+ }
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
index bb88004..f792825 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSpec.java
@@ -101,6 +101,12 @@ public class JobSpec {
this.taskSpecs = taskSpecs;
}
+ public boolean validate() {
+ return librarySpec != null && librarySpec.validate()
+ && submitterSpec != null && submitterSpec.validate()
+ && taskSpecs != null;
+ }
+
/**
* This could be file/directory which contains multiple python scripts.
* We should solve dependencies distribution in k8s or yarn.
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
index d012a26..5dd7066 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobSubmitterSpec.java
@@ -111,4 +111,8 @@ public class JobSubmitterSpec {
public void setApiVersion(String apiVersion) {
this.apiVersion = apiVersion;
}
+
+ public boolean validate() {
+ return type != null && namespace != null;
+ }
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobTaskSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobTaskSpec.java
index 586f3ee..f2964c0 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobTaskSpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/JobTaskSpec.java
@@ -27,7 +27,7 @@ import java.util.Map;
*/
public class JobTaskSpec {
/**
- * The job task name, the range is [Chief, Ps, Worker, Evaluator, Master]
+ * The job task name, if not specify using the library name
*/
private String name;
@@ -72,7 +72,7 @@ public class JobTaskSpec {
}
/**
- * Get the task name which range is [Chief, Ps, Worker, Evaluator, Master]
+ * Get the task name, if not specify using the library name
* @return task name
*/
public String getName() {
@@ -81,7 +81,7 @@ public class JobTaskSpec {
/**
* Set the task name
- * @param name task name which range is [Chief, Ps, Worker, Evaluator,
Master]
+ * @param name task name
*/
public void setName(String name) {
this.name = name;
@@ -216,4 +216,5 @@ public class JobTaskSpec {
public String getGpu() {
return resourceMap.get("nvidia.com/gpu");
}
+
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/JobManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/JobManager.java
index 03c386d..0311378 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/JobManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/JobManager.java
@@ -73,12 +73,20 @@ public class JobManager implements JobHandler {
@Override
public Job submitJob(JobSpec spec) throws UnsupportedJobTypeException {
+ if (!spec.validate()) {
+ return null;
+ }
+
+ JobSubmitter submitter = submitterManager.getSubmitterByType(
+ spec.getSubmitterSpec().getType());
+ if (submitter == null) {
+ throw new UnsupportedJobTypeException();
+ }
+
Job job = new Job();
job.setJobId(generateJobId());
executorService.submit(() -> {
try {
- JobSubmitter submitter = submitterManager.getSubmitterByType(
- spec.getSubmitterSpec().getType());
jobs.putIfAbsent(job.getJobId(), submitter.submitJob(spec));
} catch (UnsupportedJobTypeException e) {
LOG.warn(e.getMessage(), e);
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/JobServerRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/JobServerRestApi.java
deleted file mode 100644
index e853fef..0000000
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/JobServerRestApi.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.submarine.server.jobserver.rest;
-
-import org.apache.submarine.server.rest.RestConstants;
-import org.apache.submarine.server.api.spec.JobSpec;
-import org.apache.submarine.server.response.JsonResponse;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- * ML job rest API v1. It can accept MLJobSpec to create a job.
- * To create a job:
- * POST /api/v1/jobs
- *
- * To list the jobs
- * GET /api/v1/jobs
- *
- * To get a specific job
- * GET /api/v1/jobs/{id}
- *
- * To delete a job by id
- * DELETE /api/v1/jobs/{id}
- * */
-@Path(RestConstants.V1 + "/" + RestConstants.JOBS)
-@Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8})
-public class JobServerRestApi {
-
- // A ping test to verify the job server is up.
- @Path(RestConstants.PING)
- @GET
- @Consumes(MediaType.APPLICATION_JSON)
- public Response ping() {
- return new JsonResponse.Builder<String>(Response.Status.OK)
- .success(true).result("Pong").build();
- }
-
- @POST
- @Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON})
- public Response submitJob(JobSpec jobSpec) {
- // Submit the job spec through submitter
- return new JsonResponse.Builder<JobSpec>(Response.Status.ACCEPTED)
- .success(true).result(jobSpec).build();
- }
-
- @GET
- @Path("{" + RestConstants.JOB_ID + "}")
- public Response listJob(@PathParam(RestConstants.JOB_ID) String id) {
- // Query the job status though submitter
- return new JsonResponse.Builder<String>(Response.Status.OK)
- .success(true).result(id).build();
- }
-
- @GET
- public Response listAllJob() {
- // Query all the job status though submitter
- return new JsonResponse.Builder<JobSpec>(Response.Status.OK)
- .success(true).build();
- }
-
- @DELETE
- @Path("{" + RestConstants.JOB_ID + "}")
- public Response deleteJob(@PathParam(RestConstants.JOB_ID) String id) {
- // Delete the job though submitter
- return new JsonResponse.Builder<String>(Response.Status.OK)
- .success(true).result(id).build();
- }
-
-}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdDeserializer.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdDeserializer.java
new file mode 100644
index 0000000..0e1fdc4
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdDeserializer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.json;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import org.apache.submarine.server.api.job.JobId;
+
+import java.lang.reflect.Type;
+
+public class JobIdDeserializer implements JsonDeserializer<JobId> {
+
+ @Override
+ public JobId deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context)
+ throws JsonParseException {
+ return JobId.fromString(json.getAsJsonPrimitive().getAsString());
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdSerializer.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdSerializer.java
new file mode 100644
index 0000000..362bc47
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/json/JobIdSerializer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.json;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import org.apache.submarine.server.api.job.JobId;
+
+import java.lang.reflect.Type;
+
+public class JobIdSerializer implements JsonSerializer<JobId> {
+ @Override
+ public JsonElement serialize(JobId src, Type typeOfSrc,
JsonSerializationContext context) {
+ return new JsonPrimitive(src.toString());
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
index 304c324..72d4c2b 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/response/JsonResponse.java
@@ -23,6 +23,9 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.TypeAdapter;
+import org.apache.submarine.server.api.job.JobId;
+import org.apache.submarine.server.json.JobIdDeserializer;
+import org.apache.submarine.server.json.JobIdSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -161,7 +164,10 @@ public class JsonResponse<T> {
safeGson = new GsonBuilder()
.registerTypeAdapter(Date.class, safeDateTypeAdapter)
- .serializeNulls().create();
+ .registerTypeAdapter(JobId.class, new JobIdSerializer())
+ .registerTypeAdapter(JobId.class, new JobIdDeserializer())
+ .serializeNulls()
+ .create();
}
boolean haveDictAnnotation = false;
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/JobManagerRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/JobManagerRestApi.java
new file mode 100644
index 0000000..fae7be8
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/JobManagerRestApi.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.rest;
+
+import org.apache.submarine.server.JobManager;
+import org.apache.submarine.server.api.exception.UnsupportedJobTypeException;
+import org.apache.submarine.server.api.job.Job;
+import org.apache.submarine.server.api.job.JobId;
+import org.apache.submarine.server.api.spec.JobSpec;
+import org.apache.submarine.server.response.JsonResponse;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * {@link JobManager}'s REST API v1. It can accept {@link JobSpec} to create a
job.
+ */
+@Path(RestConstants.V1 + "/" + RestConstants.JOBS)
+@Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8})
+public class JobManagerRestApi {
+
+ /**
+ * Return the Pong message for test the connectivity
+ * @return Pong message
+ */
+ @GET
+ @Path(RestConstants.PING)
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response ping() {
+ return new JsonResponse.Builder<String>(Response.Status.OK)
+ .success(true).result("Pong").build();
+ }
+
+ /**
+ * Returns the contents of {@link Job} that submitted by user.
+ * @param jobSpec job spec
+ * @return the contents of job
+ */
+ @POST
+ @Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON})
+ public Response submitJob(JobSpec jobSpec) {
+ if (!jobSpec.validate()) {
+ return new JsonResponse.Builder<String>(Response.Status.ACCEPTED)
+ .success(true).result("Invalid params.").build();
+ }
+
+ try {
+ Job job = JobManager.getInstance().submitJob(jobSpec);
+ return new JsonResponse.Builder<Job>(Response.Status.ACCEPTED)
+ .success(true).result(job).build();
+ } catch (UnsupportedJobTypeException e) {
+ return new JsonResponse.Builder<String>(Response.Status.ACCEPTED)
+ .success(true).result(e.getMessage()).build();
+ }
+ }
+
+ /**
+ * List all job for the user
+ * @return job list
+ */
+ @GET
+ public Response listJob() {
+ // TODO(jiwq): Hook JobManager when 0.4.0 released
+ return new JsonResponse.Builder<List<Job>>(Response.Status.OK)
+ .success(true).result(new ArrayList<>()).build();
+ }
+
+ /**
+ * Returns the job detailed info by specified job id
+ * @param id job id
+ * @return the detailed info of job
+ */
+ @GET
+ @Path("{" + RestConstants.JOB_ID + "}")
+ public Response getJob(@PathParam(RestConstants.JOB_ID) String id) {
+ // TODO(jiwq): Hook JobManager when 0.4.0 released
+ Job job = new Job();
+ job.setJobId(JobId.fromString(id));
+ return new JsonResponse.Builder<Job>(Response.Status.OK)
+ .success(true).result(job).build();
+ }
+
+ /**
+ * Returns the job that deleted
+ * @param id job id
+ * @return the detailed info about deleted job
+ */
+ @DELETE
+ @Path("{" + RestConstants.JOB_ID + "}")
+ public Response deleteJob(@PathParam(RestConstants.JOB_ID) String id) {
+ // TODO(jiwq): Hook JobManager when 0.4.0 released
+ return new JsonResponse.Builder<Job>(Response.Status.OK)
+ .success(true).result(new Job()).build();
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/provider/YamlEntityProvider.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/provider/YamlEntityProvider.java
similarity index 98%
rename from
submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/provider/YamlEntityProvider.java
rename to
submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/provider/YamlEntityProvider.java
index 902dc03..bce9b7b 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/provider/YamlEntityProvider.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/provider/YamlEntityProvider.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.submarine.server.jobserver.provider;
+package org.apache.submarine.server.rest.provider;
import org.yaml.snakeyaml.Yaml;
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/jobserver/JobServerRestApiTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/JobManagerRestApiTest.java
similarity index 76%
rename from
submarine-server/server-core/src/test/java/org/apache/submarine/server/jobserver/JobServerRestApiTest.java
rename to
submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/JobManagerRestApiTest.java
index d580e9d..52d52b7 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/jobserver/JobServerRestApiTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/JobManagerRestApiTest.java
@@ -17,14 +17,20 @@
* under the License.
*/
-package org.apache.submarine.server.jobserver;
+package org.apache.submarine.server.rest;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.io.FileUtils;
import org.apache.submarine.server.AbstractSubmarineServerTest;
-import org.apache.submarine.server.rest.RestConstants;
+import org.apache.submarine.server.api.job.Job;
+import org.apache.submarine.server.api.job.JobId;
+import org.apache.submarine.server.json.JobIdDeserializer;
+import org.apache.submarine.server.json.JobIdSerializer;
import org.apache.submarine.server.response.JsonResponse;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -33,16 +39,19 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.core.Response;
+import java.io.File;
import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertEquals;
-public class JobServerRestApiTest extends AbstractSubmarineServerTest {
- private static final Logger LOG =
LoggerFactory.getLogger(JobServerRestApiTest.class);
+public class JobManagerRestApiTest extends AbstractSubmarineServerTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(JobManagerRestApiTest.class);
@BeforeClass
public static void init() throws Exception {
-
AbstractSubmarineServerTest.startUp(JobServerRestApiTest.class.getSimpleName());
+
AbstractSubmarineServerTest.startUp(JobManagerRestApiTest.class.getSimpleName());
}
@AfterClass
@@ -69,17 +78,18 @@ public class JobServerRestApiTest extends
AbstractSubmarineServerTest {
// Test job created with correct JSON input
@Test
- public void testCreateJobWhenJsonInputIsCorrectThenResponseCodeAccepted()
throws IOException {
- String jobSpec = "{\"name\": \"mnist\"}";
+ public void testCreateJobWhenJsonInputIsCorrectThenResponseCodeAccepted()
throws Exception {
+ URL fileUrl = this.getClass().getResource("/tf-mnist-req.json");
+ String jobSpec = FileUtils.readFileToString(new File(fileUrl.toURI()),
StandardCharsets.UTF_8);
PostMethod response = httpPost("/api/" + RestConstants.V1 + "/" +
RestConstants.JOBS, jobSpec);
LOG.info(response.toString());
- String requestBody = response.getResponseBodyAsString();
- LOG.info(requestBody);
+ String responseBodyAsString = response.getResponseBodyAsString();
+ LOG.info(responseBodyAsString);
Gson gson = new Gson();
- JsonResponse jsonResponse = gson.fromJson(requestBody, JsonResponse.class);
+ JsonResponse jsonResponse = gson.fromJson(responseBodyAsString,
JsonResponse.class);
assertEquals("Response code should be 202 ",
Response.Status.ACCEPTED.getStatusCode(), jsonResponse.getCode());
}
@@ -113,20 +123,24 @@ public class JobServerRestApiTest extends
AbstractSubmarineServerTest {
// Test get job by id
@Test
public void testGetJobById() throws IOException {
- String jobId = "job1";
+ String jobId = "job_1577810970_0001";
GetMethod response = httpGet("/api/" + RestConstants.V1 + "/"
+ RestConstants.JOBS + "/" + jobId);
LOG.info(response.toString());
- String requestBody = response.getResponseBodyAsString();
- LOG.info(requestBody);
+ String responseBodyAsString = response.getResponseBodyAsString();
+ LOG.info(responseBodyAsString);
- Gson gson = new Gson();
- JsonResponse jsonResponse = gson.fromJson(requestBody, JsonResponse.class);
+ Gson gson = new GsonBuilder()
+ .registerTypeAdapter(JobId.class, new JobIdSerializer())
+ .registerTypeAdapter(JobId.class, new JobIdDeserializer())
+ .create();
+ JsonResponse<Job> jsonResponse = gson.fromJson(responseBodyAsString,
+ new TypeToken<JsonResponse<Job>>(){}.getType());
assertEquals("Response code should be 200 ",
Response.Status.OK.getStatusCode(), jsonResponse.getCode());
assertEquals("Job id should be " + jobId, jobId,
- jsonResponse.getResult().toString());
+ jsonResponse.getResult().getJobId().toString());
}
// Test delete job by id
diff --git a/submarine-server/server-core/src/test/resources/tf-mnist-req.json
b/submarine-server/server-core/src/test/resources/tf-mnist-req.json
new file mode 100644
index 0000000..0850e54
--- /dev/null
+++ b/submarine-server/server-core/src/test/resources/tf-mnist-req.json
@@ -0,0 +1,31 @@
+{
+ "name": "mnist",
+ "librarySpec": {
+ "name": "TensorFlow",
+ "version": "2.1.0",
+ "image": "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0",
+ "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150",
+ "envVars": {
+ "ENV_1": "ENV1"
+ }
+ },
+ "submitterSpec": {
+ "type": "k8s",
+ "configPath": null,
+ "namespace": "submarine",
+ "kind": "TFJob",
+ "apiVersion": "kubeflow.org/v1"
+ },
+ "taskSpecs": {
+ "Ps": {
+ "name": "tensorflow",
+ "replicas": 2,
+ "resources": "cpu=4,memory=2048M,nvidia.com/gpu=1"
+ },
+ "Worker": {
+ "name": "tensorflow",
+ "replicas": 2,
+ "resources": "cpu=4,memory=2048M,nvidia.com/gpu=1"
+ }
+ }
+}
\ No newline at end of file
diff --git a/submarine-server/server-core/src/test/resources/tf-mnist-req.yaml
b/submarine-server/server-core/src/test/resources/tf-mnist-req.yaml
new file mode 100644
index 0000000..02e2125
--- /dev/null
+++ b/submarine-server/server-core/src/test/resources/tf-mnist-req.yaml
@@ -0,0 +1,23 @@
+name: "mnist"
+librarySpec:
+ name: "TensorFlow"
+ version: "2.1.0"
+ image: "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0"
+ cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log
--learning_rate=0.01 --batch_size=150"
+ envVars:
+ ENV_1: "ENV1"
+submitterSpec:
+ type: "k8s"
+ configPath:
+ namespace: "submarine"
+ kind: "TFJob"
+ apiVersion: "kubeflow.org/v1"
+taskSpecs:
+ Ps:
+ name: tensorflow
+ replicas: 2
+ resources: "cpu=4,memory=2048M,nvidia.com/gpu=1"
+ Worker:
+ name: tensorflow
+ replicas: 2
+ resources: "cpu=4,memory=2048M,nvidia.com/gpu=1"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
