This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 02c070a SUBMARINE-267. Initial job server which defines REST skeleton
02c070a is described below
commit 02c070a41d4af0027249e5ddf09642d0e0d492fe
Author: Zhankun Tang <[email protected]>
AuthorDate: Wed Nov 13 13:19:39 2019 +0800
SUBMARINE-267. Initial job server which defines REST skeleton
### What is this PR for?
The initial job server which defines REST skeleton
### What type of PR is it?
Feature
### Todos
* [1] - Fix the YAML unit test case issue
* [2] - Populate the REST API with integration to yarn-submitter or
k8s-submitter
* [3] - Add Swagger to easy access the API spec
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-267
### How should this be tested?
CI unit test is enough for the REST API before we integrate with submitter
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Are there breaking changes for older versions? No
* Does this needs documentation? No
Author: Zhankun Tang <[email protected]>
Closes #87 from tangzhankun/submarine-267 and squashes the following
commits:
5303d3f [Zhankun Tang] Add submarine server core test in Travis CI
a355ff0 [Zhankun Tang] fix dependency issue for hadoop 3.1 and 3.2
441317a [Zhankun Tang] Remove workbench server dependency from submarine
server-core
ae3cd5f [Zhankun Tang] Fix wrong test case name and inproper log method
d97cdb7 [Zhankun Tang] SUBMARINE-267. Initial version of job server which
defines job spec and REST skelon.
---
.travis.yml | 9 +-
pom.xml | 3 +-
.../commons/utils/SubmarineConfiguration.java | 34 +++-
submarine-server/pom.xml | 68 +++++++
submarine-server/server-core/pom.xml | 102 ++++++++++
.../org/apache/submarine/jobserver/JobServer.java | 165 ++++++++++++++++
.../submarine/jobserver/rest/api/JobApi.java | 93 +++++++++
.../submarine/jobserver/rest/dao/Component.java | 69 +++++++
.../submarine/jobserver/rest/dao/EnvVaraible.java | 53 +++++
.../jobserver/rest/dao/JsonExclusionStrategy.java | 33 ++++
.../submarine/jobserver/rest/dao/JsonResponse.java | 217 +++++++++++++++++++++
.../submarine/jobserver/rest/dao/MLJobSpec.java | 177 +++++++++++++++++
.../jobserver/rest/dao/RestConstants.java | 29 +++
.../rest/provider/YamlEntityProvider.java | 91 +++++++++
.../src/main/resources/log4j.properties | 17 ++
.../server-core/src/test/java/JobApiTest.java | 146 ++++++++++++++
16 files changed, 1300 insertions(+), 6 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 037c9cb..af2a24e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,7 @@ env:
global:
# submarine core does not required by workbench-server integration tests
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add
`!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
- -
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s"
+ -
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s,!submarine-server/server-core"
-
EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-server"
-
EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter""
-
EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli"
@@ -160,6 +160,13 @@ matrix:
- npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
env: NAME="Build workbench-web-ng"
+
+ # Test submarine-server
+ - language: java
+ jdk: "openjdk8"
+ dist: xenial
+ env: NAME="Test submarine-server" PROFILE="-Phadoop-2.9"
BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am"
MODULES="-pl ${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl
submarine-server/server-core" TEST_PROJECTS=""
+
install:
- mvn --version
- echo "[$NAME] > mvn $BUILD_FLAG $MODULES $PROFILE -B"
diff --git a/pom.xml b/pom.xml
index df70ae0..8936bae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,7 +92,7 @@
<snakeyaml.version>1.16</snakeyaml.version>
<httpcore.version>4.4.4</httpcore.version>
<httpclient.version>4.5.2</httpclient.version>
- <commons-lang.version>2.5</commons-lang.version>
+ <commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-io.version>2.5</commons-io.version>
<commons-codec.version>1.5</commons-codec.version>
@@ -124,6 +124,7 @@
<hive.version>2.1.1</hive.version>
<!-- Submarine on Kubernetes -->
<k8s.client-java.version>6.0.1</k8s.client-java.version>
+ <jersey.test-framework>2.27</jersey.test-framework>
</properties>
<modules>
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 2ddb50c..44cea26 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -113,11 +113,11 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
}
- LOG.info("Server Host: " + conf.getServerAddress());
+ LOG.info("Workbench server Host: " + conf.getServerAddress());
if (conf.useSsl() == false) {
- LOG.info("Server Port: " + conf.getServerPort());
+ LOG.info("Workbench server Port: " + conf.getServerPort());
} else {
- LOG.info("Server SSL Port: " + conf.getServerSslPort());
+ LOG.info("Workbench server SSL Port: " + conf.getServerSslPort());
}
return conf;
@@ -127,14 +127,26 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getString(ConfVars.SERVER_ADDR);
}
+ public String getJobServerAddress() {
+ return getString(ConfVars.JOB_SERVER_ADDR);
+ }
+
public boolean useSsl() {
return getBoolean(ConfVars.SERVER_SSL);
}
+ public boolean isJobServerSslEnabled() {
+ return getBoolean(ConfVars.JOB_SERVER_SSL);
+ }
+
public int getServerPort() {
return getInt(ConfVars.SERVER_PORT);
}
+ public int getJobServerPort() {
+ return getInt(ConfVars.JOB_SERVER_PORT);
+ }
+
@VisibleForTesting
public void setServerPort(int port) {
properties.put(ConfVars.SERVER_PORT.getVarName(), String.valueOf(port));
@@ -144,6 +156,14 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getInt(ConfVars.SERVER_SSL_PORT);
}
+ public int getJobServerSslPort() {
+ return getInt(ConfVars.JOB_SERVER_SSL_PORT);
+ }
+
+ public String getJobServerUrlPrefix() {
+ return getString(ConfVars.JOB_SERVER_REST_URL_PREFIX);
+ }
+
public String getKeyStorePath() {
String path = getString(ConfVars.SSL_KEYSTORE_PATH);
return path;
@@ -419,7 +439,13 @@ public class SubmarineConfiguration extends
XMLConfiguration {
JDBC_PASSWORD("jdbc.password", "password"),
WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
"workbench.websocket.max.text.message.size", "1024000"),
- WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist");
+ WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
+ // submarine job server settings
+ JOB_SERVER_SSL("job.server.ssl", false),
+ JOB_SERVER_SSL_PORT("job.server.ssl.port", 8443),
+ JOB_SERVER_ADDR("job.server.port", "0.0.0.0"),
+ JOB_SERVER_PORT("job.server.port", 8765),
+ JOB_SERVER_REST_URL_PREFIX("job.server.rest.prefix", "/*");
private String varName;
@SuppressWarnings("rawtypes")
diff --git a/submarine-server/pom.xml b/submarine-server/pom.xml
index 0e7629a..a367703 100644
--- a/submarine-server/pom.xml
+++ b/submarine-server/pom.xml
@@ -35,6 +35,7 @@
<modules>
<module>server-submitter</module>
+ <module>server-core</module>
</modules>
<dependencyManagement>
@@ -43,6 +44,73 @@
<groupId>org.apache.submarine</groupId>
<artifactId>commons-runtime</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-jaxb-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>commons-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ <version>${jersey.test-framework}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.test-framework.providers</groupId>
+ <artifactId>jersey-test-framework-provider-grizzly2</artifactId>
+ <version>${jersey.test-framework}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/submarine-server/server-core/pom.xml
b/submarine-server/server-core/pom.xml
new file mode 100644
index 0000000..84c075e
--- /dev/null
+++ b/submarine-server/server-core/pom.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>submarine-server</artifactId>
+ <groupId>org.apache.submarine</groupId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>server-core</artifactId>
+ <name>Submarine: Submarine Server Core</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet-core</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>commons-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.submarine</groupId>
+ <artifactId>commons-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.test-framework.providers</groupId>
+ <artifactId>jersey-test-framework-provider-grizzly2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
new file mode 100644
index 0000000..298081e
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.submarine.jobserver;
+
+
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ThreadPool;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
+
+
+/**
+ * The ml job server. It will load the classes in rest package when
+ * bootstrap with related configurable settings.
+ * */
+public class JobServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobServer.class);
+
+ private SubmarineConfiguration conf = SubmarineConfiguration.create();
+
+ private Server jobServer;
+
+ public void start() {
+ ServletContextHandler context = new
+ ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/");
+ setupServer(conf);;
+ jobServer.setHandler(context);
+
+ // Job API servlet
+ ServletHolder apiServlet = context.addServlet(ServletContainer.class,
+ conf.getJobServerUrlPrefix());
+ apiServlet.setInitOrder(1);
+ apiServlet.setInitParameter("jersey.config.server.provider.packages",
+ "org.apache.submarine.jobserver.rest");
+
+ try {
+ jobServer.start();
+ LOG.info("Submarine job server started");
+ jobServer.join();
+ } catch (Exception e) {
+ LOG.error("Submarine job server failed to start");
+ e.printStackTrace();
+ } finally {
+ jobServer.destroy();
+ LOG.info("Submarine job server stopped");
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ new JobServer().start();
+ }
+
+ private Server setupServer(SubmarineConfiguration conf) {
+ ThreadPool threadPool =
+ new QueuedThreadPool(
+ conf.getInt(SubmarineConfiguration
+ .ConfVars.SERVER_JETTY_THREAD_POOL_MAX),
+ conf.getInt(SubmarineConfiguration
+ .ConfVars.SERVER_JETTY_THREAD_POOL_MIN),
+ conf.getInt(SubmarineConfiguration
+ .ConfVars.SERVER_JETTY_THREAD_POOL_TIMEOUT));
+ jobServer = new Server(threadPool);
+ ServerConnector connector;
+
+ if (conf.isJobServerSslEnabled()) {
+ LOG.debug("Enabling SSL for submarine job server on port "
+ + conf.getServerSslPort());
+ HttpConfiguration httpConfig = new HttpConfiguration();
+ httpConfig.setSecureScheme("https");
+ httpConfig.setSecurePort(conf.getServerSslPort());
+ httpConfig.setOutputBufferSize(32768);
+ httpConfig.setResponseHeaderSize(8192);
+ httpConfig.setSendServerVersion(true);
+
+ HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
+ SecureRequestCustomizer src = new SecureRequestCustomizer();
+ httpsConfig.addCustomizer(src);
+
+ connector = new ServerConnector(
+ jobServer,
+ new SslConnectionFactory(getSslContextFactory(conf),
+ HttpVersion.HTTP_1_1.asString()),
+ new HttpConnectionFactory(httpsConfig));
+ } else {
+ connector = new ServerConnector(jobServer);
+ }
+
+ configureRequestHeaderSize(conf, connector);
+ // Set some timeout options to make debugging easier.
+ int timeout = 1000 * 30;
+ connector.setIdleTimeout(timeout);
+ connector.setSoLingerTime(-1);
+ connector.setHost(conf.getJobServerAddress());
+ if (conf.useSsl()) {
+ connector.setPort(conf.getJobServerSslPort());
+ } else {
+ connector.setPort(conf.getJobServerPort());
+ }
+
+ jobServer.addConnector(connector);
+ return jobServer;
+ }
+
+ private static SslContextFactory getSslContextFactory(
+ SubmarineConfiguration conf) {
+ SslContextFactory sslContextFactory = new SslContextFactory();
+
+ // Set keystore
+ sslContextFactory.setKeyStorePath(conf.getKeyStorePath());
+ sslContextFactory.setKeyStoreType(conf.getKeyStoreType());
+ sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword());
+ sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword());
+
+ if (conf.useClientAuth()) {
+ sslContextFactory.setNeedClientAuth(conf.useClientAuth());
+
+ // Set truststore
+ sslContextFactory.setTrustStorePath(conf.getTrustStorePath());
+ sslContextFactory.setTrustStoreType(conf.getTrustStoreType());
+ sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword());
+ }
+
+ return sslContextFactory;
+ }
+
+ private static void configureRequestHeaderSize(
+ SubmarineConfiguration conf, ServerConnector connector) {
+ HttpConnectionFactory cf =
+ (HttpConnectionFactory) connector
+ .getConnectionFactory(HttpVersion.HTTP_1_1.toString());
+ int requestHeaderSize = conf.getJettyRequestHeaderSize();
+ cf.getHttpConfiguration().setRequestHeaderSize(requestHeaderSize);
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java
new file mode 100644
index 0000000..8b842f4
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.api;
+
+import org.apache.submarine.jobserver.rest.dao.JsonResponse;
+import org.apache.submarine.jobserver.rest.dao.MLJobSpec;
+import org.apache.submarine.jobserver.rest.dao.RestConstants;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ * ML job rest API v1. It can accept MLJobSpec to create a job.
+ * To create a job:
+ * POST /api/v1/jobs
+ *
+ * To list the jobs
+ * GET /api/v1/jobs
+ *
+ * To get a specific job
+ * GET /api/v1/jobs/{id}
+ *
+ * To delete a job by id
+ * DELETE /api/v1/jobs/{id}
+ * */
+@Path(RestConstants.V1 + "/" + RestConstants.JOBS)
+@Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8})
+public class JobApi {
+
+ // A ping test to verify the job server is up.
+ @Path(RestConstants.PING)
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response ping() {
+ return new JsonResponse.Builder<String>(Response.Status.OK)
+ .success(true).result("Pong").build();
+ }
+
+ @POST
+ @Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON})
+ public Response submitJob(MLJobSpec jobSpec) {
+ // Submit the job spec through submitter
+ return new JsonResponse.Builder<MLJobSpec>(Response.Status.ACCEPTED)
+ .success(true).result(jobSpec).build();
+ }
+
+ @GET
+ @Path("{" + RestConstants.JOB_ID + "}")
+ public Response listJob(@PathParam(RestConstants.JOB_ID) String id) {
+ // Query the job status though submitter
+ return new JsonResponse.Builder<MLJobSpec>(Response.Status.OK)
+ .success(true).result(id).build();
+ }
+
+ @GET
+ public Response listAllJob() {
+ // Query all the job status though submitter
+ return new JsonResponse.Builder<MLJobSpec>(Response.Status.OK)
+ .success(true).build();
+ }
+
+ @DELETE
+ @Path("{" + RestConstants.JOB_ID + "}")
+ public Response deleteJob(@PathParam(RestConstants.JOB_ID) String id) {
+ // Delete the job though submitter
+ return new JsonResponse.Builder<MLJobSpec>(Response.Status.OK)
+ .success(true).result(id).build();
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java
new file mode 100644
index 0000000..a2c1458
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+
+/**
+ * One component contains role, count and resources.
+ * The role name could be tensorflow ps, Pytorch master or tensorflow worker
+ * The count is the count of the role instance
+ * The resource is the memory/vcore/gpu resource strings in the format:
+ * "memory=2048M,vcore=4,nvidia.com/gpu=1"
+ */
+
+public class Component {
+
+ public String getRole() {
+ return role;
+ }
+
+ public void setRole(String role) {
+ this.role = role;
+ }
+
+ public String getCount() {
+ return count;
+ }
+
+ public void setCount(String count) {
+ this.count = count;
+ }
+
+ public String getResources() {
+ return resources;
+ }
+
+ public void setResources(String resources) {
+ this.resources = resources;
+ }
+
+ String role;
+ String count;
+ String resources;
+
+ public Component() {}
+
+ public Component(String r, String ct, String res) {
+ this.count = ct;
+ this.role = r;
+ this.resources = res;
+ }
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java
new file mode 100644
index 0000000..24a4ae2
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+// A process level environment variable.
+public class EnvVaraible {
+
+ public String getKey() {
+ return key;
+ }
+
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ String key;
+ String value;
+
+ public EnvVaraible() {}
+
+ public EnvVaraible(String k, String v) {
+ this.key = k;
+ this.value = v;
+ }
+
+
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java
new file mode 100644
index 0000000..c24e413
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.FieldAttributes;
+
+public class JsonExclusionStrategy implements ExclusionStrategy {
+ public boolean shouldSkipClass(Class<?> arg0) {
+ return false;
+ }
+
+ public boolean shouldSkipField(FieldAttributes f) {
+ return false;
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java
new file mode 100644
index 0000000..46726cf
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.NewCookie;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Json response builder.
+ *
+ * @param <T> can be an object or a ListResult
+ */
+public class JsonResponse<T> {
+ private static final Logger LOG =
LoggerFactory.getLogger(JsonResponse.class);
+
+ private final javax.ws.rs.core.Response.Status status;
+ private final int code;
+ private final Boolean success;
+ private final String message;
+ private final T result;
+ private final transient ArrayList<NewCookie> cookies;
+ private final transient boolean pretty = false;
+ private final Map<String, Object> attributes;
+
+ private static Gson safeGson = null;
+
+ private static final String CGLIB_PROPERTY_PREFIX = "\\$cglib_prop_";
+
+ private JsonResponse(Builder builder) {
+ this.status = builder.status;
+ this.code = builder.code;
+ this.success = builder.success;
+ this.message = builder.message;
+ this.attributes = builder.attributes;
+ this.result = (T) builder.result;
+ this.cookies = builder.cookies;
+ }
+
+ public T getResult() {
+ return result;
+ }
+
+ public Boolean getSuccess() {
+ return success;
+ }
+
+ @VisibleForTesting
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+
+ public static class Builder<T> {
+ private javax.ws.rs.core.Response.Status status;
+ private int code;
+ private Boolean success;
+ private String message;
+ private T result;
+ private Map<String, Object> attributes = new HashMap<>();
+ private transient ArrayList<NewCookie> cookies;
+ private transient boolean pretty = false;
+
+ public Builder(javax.ws.rs.core.Response.Status status) {
+ this.status = status;
+ this.code = status.getStatusCode();
+ }
+
+ public Builder(int code) {
+ this.code = code;
+ }
+
+ public Builder attribute(String key, Object value) {
+ this.attributes.put(key, value);
+ return this;
+ }
+
+ public Builder success(Boolean success) {
+ this.success = success;
+ return this;
+ }
+
+ public Builder message(String message) {
+ this.message = message;
+ return this;
+ }
+
+ public Builder result(T result) {
+ this.result = result;
+ return this;
+ }
+
+ public Builder code(int code) {
+ this.code = code;
+ return this;
+ }
+
+ public Builder cookies(ArrayList<NewCookie> newCookies) {
+ if (cookies == null) {
+ cookies = new ArrayList<>();
+ }
+ cookies.addAll(newCookies);
+ return this;
+ }
+
+ public javax.ws.rs.core.Response build() {
+ JsonResponse jsonResponse = new JsonResponse(this);
+ return jsonResponse.build();
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (safeGson == null) {
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ if (pretty) {
+ gsonBuilder.setPrettyPrinting();
+ }
+ gsonBuilder.setExclusionStrategies(new JsonExclusionStrategy());
+
+ // Trick to get the DefaultDateTypeAdatpter instance
+ // Create a first instance a Gson
+ Gson gson = gsonBuilder.setDateFormat("yyyy-MM-dd HH:mm:ss").create();
+
+ // Get the date adapter
+ TypeAdapter<Date> dateTypeAdapter = gson.getAdapter(Date.class);
+
+ // Ensure the DateTypeAdapter is null safe
+ TypeAdapter<Date> safeDateTypeAdapter = dateTypeAdapter.nullSafe();
+
+ safeGson = new GsonBuilder()
+ .registerTypeAdapter(Date.class, safeDateTypeAdapter)
+ .serializeNulls().create();
+ }
+
+// boolean haveDictAnnotation = false;
+// try {
+// if (null != getResult()) {
+// haveDictAnnotation = DictAnnotation.parseDictAnnotation(getResult());
+// }
+// } catch (Exception e) {
+// LOG.error(e.getMessage(), e);
+// }
+
+ String json = safeGson.toJson(this);
+// if (haveDictAnnotation) {
+// json = json.replaceAll(CGLIB_PROPERTY_PREFIX, "");
+// }
+
+ return json;
+ }
+
+ private synchronized javax.ws.rs.core.Response build() {
+ Response.ResponseBuilder r =
javax.ws.rs.core.Response.status(status).entity(this.toString());
+ if (cookies != null) {
+ for (NewCookie nc : cookies) {
+ r.cookie(nc);
+ }
+ }
+ return r.build();
+ }
+
+ // list result type response
+ // Used to return a list of records
+ public static class ListResult<T> {
+ private List<T> records;
+ private long total;
+
+ public ListResult(List<T> records, long total) {
+ this.records = records;
+ this.total = total;
+ }
+
+ public List<T> getRecords() {
+ return records;
+ }
+
+ public void setRecords(List<T> records) {
+ this.records = records;
+ }
+
+ public long getTotal() {
+ return total;
+ }
+
+ public void setTotal(long total) {
+ this.total = total;
+ }
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java
new file mode 100644
index 0000000..26f7fa8
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+/**
+ * The machine learning job spec the submarine job server can accept.
+ * */
+public class MLJobSpec {
+
+ String apiVersion;
+ // The engine type the job will use, can be tensorflow or pytorch
+ String type;
+ // The engine version, not image version/tag. For instance, tensorflow v1.13
+ String version;
+ /**
+ * Advanced property. The RM this job will submit to, k8s or yarn.
+ * Could be the path of yarn’s config file or k8s kubeconfig file
+ * This should be settled when deploy submarine job server.
+ */
+ String rmConfig;
+
+ /**
+ * Advanced property. The image should be inferred from type and version.
+ * The normal user should not set this.
+ * */
+ String dockerImage;
+
+ // The process aware environment variable
+ EnvVaraible[] envVars;
+
+ // The components this cluster will consists.
+ Component[] components;
+
+ // The user id who submit job
+ String uid;
+
+ /**
+ * The queue this job will submitted to.
+ * In YARN, we call it queue. In k8s, no such concept.
+ * It could be namespace’s name.
+ */
+ String queue;
+
+ /**
+ * The user-specified job name for easy search
+ * */
+ String name;
+
+ /**
+ * This could be file/directory which contains multiple python scripts.
+ * We should solve dependencies distribution in k8s or yarn.
+ * Or we could build images for each projects before submitting the job
+ * */
+ String projects;
+
+ /**
+ * The command uses to start the job. This is very job-specific.
+ * We assume the cmd is the same for all containers in a cluster
+ * */
+ String cmd;
+
+ public MLJobSpec() {}
+
+ public String getUid() {
+ return uid;
+ }
+
+ public void setUid(String uid) {
+ this.uid = uid;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getProjects() {
+ return projects;
+ }
+
+ public void setProjects(String projects) {
+ this.projects = projects;
+ }
+ public String getCmd() {
+ return cmd;
+ }
+
+ public void setCmd(String cmd) {
+ this.cmd = cmd;
+ }
+
+ public Component[] getComponents() {
+ return components;
+ }
+
+ public void setComponents(
+ Component[] components) {
+ this.components = components;
+ }
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public String getRmConfig() {
+ return rmConfig;
+ }
+
+ public void setRmConfig(String rmConfig) {
+ this.rmConfig = rmConfig;
+ }
+
+ public String getDockerImage() {
+ return dockerImage;
+ }
+
+ public void setDockerImage(String dockerImage) {
+ this.dockerImage = dockerImage;
+ }
+
+ public EnvVaraible[] getEnvVars() {
+ return envVars;
+ }
+
+ public void setEnvVars(EnvVaraible[] envVars) {
+ this.envVars = envVars;
+ }
+
+ public String getApiVersion() {
+ return apiVersion;
+ }
+
+ public void setApiVersion(String apiVersion) {
+ this.apiVersion = apiVersion;
+ }
+
+
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java
new file mode 100644
index 0000000..a33a931
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.dao;
+
+public class RestConstants {
+ public final static String V1 = "v1";
+ public final static String JOBS = "jobs";
+ public final static String JOB_ID = "id";
+ public final static String PING = "ping";
+ public final static String MEDIA_TYPE_YAML = "application/yaml";
+ public final static String CHARSET_UTF8 = "charset=utf-8";
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java
new file mode 100644
index 0000000..3343cc8
--- /dev/null
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.jobserver.rest.provider;
+
+import org.yaml.snakeyaml.Yaml;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.MessageBodyWriter;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Type;
+import java.util.Scanner;
+
+@Provider
+@Consumes({"application/yaml", MediaType.TEXT_PLAIN})
+@Produces({"application/yaml", MediaType.TEXT_PLAIN})
+public class YamlEntityProvider<T> implements MessageBodyWriter<T>,
MessageBodyReader<T> {
+
+ @Override
+ public boolean isReadable(Class<?> type, Type genericType,
+ Annotation[] annotations,
+ MediaType mediaType) {
+ return true;
+ }
+
+ @Override
+ public T readFrom(Class<T> type, Type genericType, Annotation[] annotations,
+ MediaType mediaType,
+ MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
+ throws WebApplicationException {
+ Yaml yaml = new Yaml();
+ T t = yaml.loadAs(toString(entityStream), type);
+ return t;
+ }
+
+ public static String toString(InputStream inputStream) {
+ return new Scanner(inputStream, "UTF-8")
+ .useDelimiter("\\A").next();
+ }
+
+ @Override
+ public boolean isWriteable(Class<?> type, Type genericType,
+ Annotation[] annotations,
+ MediaType mediaType) {
+ return true;
+ }
+
+ @Override
+ public long getSize(T t, Class<?> type, Type genericType,
+ Annotation[] annotations,
+ MediaType mediaType) {
+ return -1;
+ }
+
+ @Override
+ public void writeTo(T t, Class<?> type, Type genericType,
+ Annotation[] annotations,
+ MediaType mediaType, MultivaluedMap<String, Object> httpHeaders,
+ OutputStream entityStream) throws IOException, WebApplicationException {
+ Yaml yaml = new Yaml();
+ OutputStreamWriter writer = new OutputStreamWriter(entityStream);
+ yaml.dump(t, writer);
+ writer.close();
+ }
+}
diff --git a/submarine-server/server-core/src/main/resources/log4j.properties
b/submarine-server/server-core/src/main/resources/log4j.properties
new file mode 100644
index 0000000..55e02b6
--- /dev/null
+++ b/submarine-server/server-core/src/main/resources/log4j.properties
@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+log4j.rootLogger = info, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target = System.out
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd
HH:mm:ss,SSS} method:%l%n%m%n
diff --git a/submarine-server/server-core/src/test/java/JobApiTest.java
b/submarine-server/server-core/src/test/java/JobApiTest.java
new file mode 100644
index 0000000..6a7ef12
--- /dev/null
+++ b/submarine-server/server-core/src/test/java/JobApiTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.google.gson.Gson;
+import org.apache.submarine.jobserver.rest.dao.JsonResponse;
+import org.apache.submarine.jobserver.rest.dao.RestConstants;
+import org.apache.submarine.jobserver.rest.api.JobApi;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.test.JerseyTest;
+import org.junit.Test;
+
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import static org.junit.Assert.assertEquals;
+
+public class JobApiTest extends JerseyTest {
+
+ @Override
+ protected Application configure() {
+ return new ResourceConfig(JobApi.class);
+ }
+
+ @Test
+ public void testJobServerPing() {
+ String str = "Pong";
+ Response response = target(RestConstants.V1 + "/"
+ + RestConstants.JOBS + "/" + RestConstants.PING)
+ .request()
+ .get();
+ Gson gson = new Gson();
+ JsonResponse r = gson.fromJson(response.readEntity(String.class),
+ JsonResponse.class);
+ assertEquals("Response code should be 200 ",
+ Response.Status.OK.getStatusCode(), response.getStatus());
+ assertEquals("Response message should be " + str,
+ str, r.getResult().toString());
+ }
+
+ // Test job created with correct JSON input
+ @Test
+ public void testCreateJobWhenJsonInputIsCorrectThenResponseCodeAccepted() {
+ String jobSpec = "{\"type\": \"tensorflow\", \"version\":\"v1.13\"}";
+ Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS)
+ .request()
+ .post(Entity.entity(jobSpec, MediaType.APPLICATION_JSON));
+
+ assertEquals("Http Response should be 202 ",
+ Response.Status.ACCEPTED.getStatusCode(), response.getStatus());
+ }
+
+ // Test job created with incorrect JSON input
+ @Test
+ public void testCreateJobWhenJsonInputIsWrongThenResponseCodeBadRequest() {
+ String jobSpec = "{\"ttype\": \"tensorflow\", \"version\":\"v1.13\"}";
+ Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS)
+ .request()
+ .post(Entity.entity(jobSpec, MediaType.APPLICATION_JSON));
+
+ assertEquals("Http Response should be 400 ",
+ Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+ }
+
+ // Test get job list
+ @Test
+ public void testGetJobList() {
+ Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS)
+ .request()
+ .get();
+
+ assertEquals("Http Response should be 200 ",
+ Response.Status.OK.getStatusCode(), response.getStatus());
+ }
+
+ // Test get job by id
+ @Test
+ public void testGetJobById() {
+ String jobId = "job1";
+ Response response = target(RestConstants.V1 + "/"
+ + RestConstants.JOBS + "/" + jobId)
+ .request()
+ .get();
+ Gson gson = new Gson();
+ JsonResponse r = gson.fromJson(response.readEntity(String.class),
+ JsonResponse.class);
+ assertEquals("Http Response should be 200 ",
+ Response.Status.OK.getStatusCode(), response.getStatus());
+ assertEquals("Job id should be " + jobId,
+ jobId, r.getResult().toString());
+ }
+
+ // Test delete job by id
+ @Test
+ public void testDeleteJobById() {
+ String jobId = "job1";
+ Response response = target(RestConstants.V1 + "/"
+ + RestConstants.JOBS + "/" + jobId)
+ .request()
+ .delete();
+ Gson gson = new Gson();
+ JsonResponse r = gson.fromJson(response.readEntity(String.class),
+ JsonResponse.class);
+ assertEquals("Http Response should be 200 ",
+ Response.Status.OK.getStatusCode(), response.getStatus());
+ assertEquals("Deleted job id should be " + jobId,
+ jobId, r.getResult().toString());
+ }
+
+ /**
+ * FiXME. The manual YAML test with postman works but failed here.
+ * We need to figure out why the YAML entity provider not work in this test.
+ * */
+ @Test
+ public void testCreateJobWhenYamlInputIsCorrectThenResponseCodeAccepted() {
+// Client client = ClientBuilder.newBuilder()
+// .register(new YamlEntityProvider<>()).build();
+// this.setClient(client);
+// String jobSpec = "type: tf";
+// Response response = target(RestConstants.V1 + "/"
+// + RestConstants.JOBS + "/" + "test")
+// .request()
+// .put(Entity.entity(jobSpec, "application/yaml"));
+//
+// assertEquals("Http Response should be 202 ",
+// Response.Status.ACCEPTED.getStatusCode(), response.getStatus());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
