This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new fa58973 SUBMARINE-1187. Refine the interface of internal service and
keep more details into db
fa58973 is described below
commit fa58973f5f36067ab7db7a1b6d2e27ea8a71874b
Author: FatalLin <fatal...@gmail.com>
AuthorDate: Thu Feb 17 00:40:32 2022 +0800
SUBMARINE-1187. Refine the interface of internal service and keep more
details into db
### What is this PR for?
per discussion with pingsutw, we would like to persist more detail data
into database which allow us to provide to users when they check the detail,
therefore I refine the logic when handler receiving the events, it will align
the same logic in k8ssubmitter to rebuild the corresponding object, and pass to
internal service, then the internal service persist it into database.
### What type of PR is it?
Improvement
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1187
### How should this be tested?
existed unit tests.
### Screenshots (if appropriate)
same of: https://github.com/apache/submarine/pull/877
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: FatalLin <fatal...@gmail.com>
Signed-off-by: Kevin <pings...@apache.org>
Closes #878 from FatalLin/SUBMARINE-1187 and squashes the following commits:
01fdea4e [FatalLin] add missing dependency
282cffc2 [FatalLin] code polish
89565e1d [FatalLin] change jdbc type from date to timestamp
fcf0fd26 [FatalLin] use the same time from k8s instead of mysql as
initialize value
d1c0eb27 [FatalLin] persist createTime as well
8f8a047b [FatalLin] add timezone to extract correct time
e2d11f2a [FatalLin] fix test case
bf504dbd [FatalLin] align datetime formate
82c4de06 [FatalLin] merge
53893d95 [FatalLin] polish the interface of internal service
f1cc552d [FatalLin] rollback codes to fit testing behavior
ba2fdd5e [FatalLin] justify init status of notebook creation
6d797ed8 [FatalLin] fix
c5fa8a3d [FatalLin] format
87d98614 [FatalLin] remove unsed code section
71647660 [FatalLin] merge
225e06d6 [FatalLin] refactory of notebook crud opertion
f09dd69e [FatalLin] commit for pair programming
---
dev-support/database/submarine.sql | 3 ++
dev-support/docker-images/agent/build.sh | 2 +-
pom.xml | 6 +++
submarine-server/server-core/pom.xml | 5 +++
.../database/entity/ExperimentEntity.java | 2 +-
.../server/internal/InternalServiceManager.java | 41 +++++++++++++++----
.../notebook/database/entity/NotebookEntity.java | 39 ++++++++++++++++++
.../notebook/database/service/NotebookService.java | 17 ++++++++
.../server/rest/InternalServiceRestApi.java | 9 +++--
.../submarine/database/mappers/NotebookMapper.xml | 17 ++++++--
.../internal/InternalServiceManagerTest.java | 27 +++++++++++--
.../server/rest/InternalServiceRestApiTest.java | 7 +++-
.../server/k8s/agent/handler/NotebookHandler.java | 47 +++++++++++++++++++---
.../server/k8s/agent/util/RestClient.java | 20 ++++-----
.../submitter/k8s/parser/NotebookSpecParser.java | 2 +-
15 files changed, 202 insertions(+), 42 deletions(-)
diff --git a/dev-support/database/submarine.sql
b/dev-support/database/submarine.sql
index 9dba21a..41f3446 100644
--- a/dev-support/database/submarine.sql
+++ b/dev-support/database/submarine.sql
@@ -263,6 +263,9 @@ CREATE TABLE `notebook` (
`update_by` varchar(32) DEFAULT NULL COMMENT 'last update user',
`update_time` datetime DEFAULT NULL COMMENT 'last update time',
`notebook_status` varchar(20) DEFAULT NULL COMMENT 'notebook status',
+ `notebook_url` varchar(256) DEFAULT NULL COMMENT 'notebook url',
+ `reason` varchar(512) DEFAULT NULL COMMENT 'reason of state',
+ `deleted_time` datetime DEFAULT NULL COMMENT 'delete time',
PRIMARY KEY `id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/dev-support/docker-images/agent/build.sh
b/dev-support/docker-images/agent/build.sh
index 5ef4a3e..dd3b3c4 100755
--- a/dev-support/docker-images/agent/build.sh
+++ b/dev-support/docker-images/agent/build.sh
@@ -17,7 +17,7 @@
set -euxo pipefail
SUBMARINE_VERSION=0.7.0-SNAPSHOT
-SUBMARINE_IMAGE_NAME="apache/submarine:sidecar-agent-${SUBMARINE_VERSION}"
+SUBMARINE_IMAGE_NAME="apache/submarine:agent-${SUBMARINE_VERSION}"
if [ -L ${BASH_SOURCE-$0} ]; then
PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
diff --git a/pom.xml b/pom.xml
index ffd3e47..3d91af1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -146,6 +146,7 @@
<guice.version>3.0</guice.version>
<!-- server API -->
<protobuf-java.version>3.14.0</protobuf-java.version>
+ <joda-time.version>2.10.8</joda-time.version>
</properties>
<modules>
@@ -323,6 +324,11 @@
<artifactId>byte-buddy</artifactId>
<version>${bytebuddy.version}</version>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda-time.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/submarine-server/server-core/pom.xml
b/submarine-server/server-core/pom.xml
index ad552e5..8ad3203 100644
--- a/submarine-server/server-core/pom.xml
+++ b/submarine-server/server-core/pom.xml
@@ -438,6 +438,11 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/database/entity/ExperimentEntity.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/database/entity/ExperimentEntity.java
index 2324c52..3bad368 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/database/entity/ExperimentEntity.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/database/entity/ExperimentEntity.java
@@ -28,7 +28,7 @@ public class ExperimentEntity extends BaseEntity {
private String experimentSpec;
private String experimentStatus;
-
+
public ExperimentEntity() {}
public String getExperimentSpec() {
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/internal/InternalServiceManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/internal/InternalServiceManager.java
index 9e5cb1f..7451fce 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/internal/InternalServiceManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/internal/InternalServiceManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.submarine.server.internal;
+import java.util.Map;
+
import javax.ws.rs.core.Response.Status;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
@@ -26,12 +28,14 @@ import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.experiment.database.entity.ExperimentEntity;
import
org.apache.submarine.server.experiment.database.service.ExperimentService;
import org.apache.submarine.server.notebook.database.service.NotebookService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
public class InternalServiceManager {
private static volatile InternalServiceManager internalServiceManager;
-
+ private static final Logger LOG =
LoggerFactory.getLogger(InternalServiceManager.class);
private final ExperimentService experimentService;
private final NotebookService notebookService;
@@ -48,32 +52,53 @@ public class InternalServiceManager {
this.notebookService = notebookService;
}
- public boolean updateCRStatus(CustomResourceType crType, String resourceId,
String status) {
+ public boolean updateCRStatus(CustomResourceType crType, String resourceId,
+ Map<String, Object> updateObject) {
if (crType.equals(CustomResourceType.Notebook)) {
- return updateNotebookStatus(resourceId, status);
+ return updateNotebookStatus(resourceId, updateObject);
} else if (crType.equals(CustomResourceType.TFJob) ||
crType.equals(CustomResourceType.PYTORCHJob)) {
- return updateExperimentStatus(resourceId, status);
+ return updateExperimentStatus(resourceId, null);
}
return false;
}
- private boolean updateExperimentStatus(String resourceId, String status) {
+ private boolean updateExperimentStatus(String resourceId, Map<String,
Object> updateObject) {
ExperimentEntity experimentEntity = experimentService.select(resourceId);
if (experimentEntity == null) {
throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(),
String.format("cannot find experiment with id:%s", resourceId));
}
- experimentEntity.setExperimentStatus(status);
+ // experimentEntity.setExperimentStatus(status);
return experimentService.update(experimentEntity);
}
- private boolean updateNotebookStatus(String resourceId, String status) {
+ private boolean updateNotebookStatus(String resourceId, Map<String, Object>
updateObject) {
Notebook notebook = notebookService.select(resourceId);
if (notebook == null) {
throw new SubmarineRuntimeException(Status.NOT_FOUND.getStatusCode(),
String.format("cannot find notebook with id:%s", resourceId));
}
- notebook.setStatus(status);
+
+ if (updateObject.containsKey("status")) {
+ notebook.setStatus(updateObject.get("status").toString());
+ }
+
+ if (updateObject.get("createTime") != null) {
+ notebook.setCreatedTime(updateObject.get("createTime").toString());
+ }
+
+ if (updateObject.get("deletedTime") != null) {
+ notebook.setDeletedTime(updateObject.get("deletedTime").toString());
+ }
+ if (updateObject.get("name") != null) {
+ notebook.setName(updateObject.get("name").toString());;
+ }
+ if (updateObject.get("reason") != null) {
+ notebook.setReason(updateObject.get("reason").toString());
+ }
+ if (updateObject.get("url") != null) {
+ notebook.setUrl(updateObject.get("url").toString());
+ }
return notebookService.update(notebook);
}
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/entity/NotebookEntity.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/entity/NotebookEntity.java
index cb83761..5590d4c 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/entity/NotebookEntity.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/entity/NotebookEntity.java
@@ -19,7 +19,12 @@
package org.apache.submarine.server.notebook.database.entity;
+import java.util.Date;
+
import org.apache.submarine.server.database.entity.BaseEntity;
+import
org.apache.submarine.server.workbench.database.utils.CustomJsonDateDeserializer;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
public class NotebookEntity extends BaseEntity {
/*
@@ -29,6 +34,12 @@ public class NotebookEntity extends BaseEntity {
private String notebookStatus;
+ private String notebookUrl;
+ private String reason;
+ @JsonDeserialize(using = CustomJsonDateDeserializer.class)
+ private Date deletedTime = new Date();
+
+
public NotebookEntity() {
}
@@ -48,6 +59,31 @@ public class NotebookEntity extends BaseEntity {
this.notebookStatus = noteStatus;
}
+
+ public String getNotebookUrl() {
+ return notebookUrl;
+ }
+
+ public void setNotebookUrl(String notebookUrl) {
+ this.notebookUrl = notebookUrl;
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ public void setReason(String reason) {
+ this.reason = reason;
+ }
+
+ public Date getDeletedTime() {
+ return deletedTime;
+ }
+
+ public void setDeletedTime(Date deletedTime) {
+ this.deletedTime = deletedTime;
+ }
+
@Override
public String toString() {
return "NotebookEntity{" +
@@ -58,6 +94,9 @@ public class NotebookEntity extends BaseEntity {
", updateBy='" + updateBy + '\'' +
", updateTime=" + updateTime + '\'' +
", notebookStatus='" + notebookStatus + "\'" +
+ ", notebookUrl= '" + notebookUrl + "\'" +
+ ", reason= '" + reason + "\'" +
+ ", deletedTime= '" + deletedTime + "\'" +
'}';
}
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
index 0b63400..6811309 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/notebook/database/service/NotebookService.java
@@ -21,6 +21,7 @@ package org.apache.submarine.server.notebook.database.service;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+
import java.util.ArrayList;
import org.apache.ibatis.session.SqlSession;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
@@ -30,6 +31,7 @@ import org.apache.submarine.server.api.spec.NotebookSpec;
import org.apache.submarine.server.database.utils.MyBatisUtil;
import org.apache.submarine.server.notebook.database.entity.NotebookEntity;
import org.apache.submarine.server.notebook.database.mappers.NotebookMapper;
+import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -131,6 +133,14 @@ public class NotebookService {
entity.setId(notebook.getNotebookId().toString());
entity.setNotebookSpec(new
GsonBuilder().disableHtmlEscaping().create().toJson(notebook.getSpec()));
entity.setNotebookStatus(notebook.getStatus());
+ entity.setNotebookUrl(notebook.getUrl());
+ entity.setReason(notebook.getReason());
+ if (notebook.getCreatedTime() != null) {
+
entity.setCreateTime(DateTime.parse(notebook.getCreatedTime()).toDate());
+ }
+ if (notebook.getDeletedTime() != null) {
+
entity.setDeletedTime(DateTime.parse(notebook.getDeletedTime()).toDate());
+ }
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new SubmarineRuntimeException("Unable to build entity from
notebook");
@@ -151,6 +161,13 @@ public class NotebookService {
notebook.setSpec(new Gson().fromJson(entity.getNotebookSpec(),
NotebookSpec.class));
notebook.setName(notebook.getSpec().getMeta().getName());
notebook.setStatus(entity.getNotebookStatus());
+ notebook.setCreatedTime(new DateTime(entity.getCreateTime()).toString());
+ notebook.setUrl(entity.getNotebookUrl());
+ notebook.setReason(entity.getReason());
+ if (entity.getDeletedTime() != null) {
+ notebook.setDeletedTime(new
DateTime(entity.getDeletedTime()).toString());
+ }
+
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new SubmarineRuntimeException("Unable to build notebook from
entity");
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/InternalServiceRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/InternalServiceRestApi.java
index 28655d7..d546ee1 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/InternalServiceRestApi.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/InternalServiceRestApi.java
@@ -18,6 +18,8 @@
*/
package org.apache.submarine.server.rest;
+import java.util.Map;
+
import javax.ws.rs.Consumes;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
@@ -63,7 +65,7 @@ public class InternalServiceRestApi {
* @return the detailed info about updated environment
*/
@POST
- @Path("/{customResourceType}/{resourceId}/{status}")
+ @Path("/{customResourceType}/{resourceId}")
@Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON})
@Operation(summary = "Update the environment with job spec",
tags = {"environments"},
@@ -78,9 +80,10 @@ public class InternalServiceRestApi {
public Response updateEnvironment(
@PathParam(RestConstants.CUSTOM_RESOURCE_TYPE) String type,
@PathParam(RestConstants.CUSTOM_RESOURCE_ID) String resourceId,
- @PathParam(RestConstants.CUSTOM_RESOURCE_STATUS) String status) {
+ Map<String, Object> updatedCustomObject) {
try {
- internalServiceManager.updateCRStatus(CustomResourceType.valueOf(type),
resourceId, status);
+ internalServiceManager.updateCRStatus(CustomResourceType.valueOf(type)
+ , resourceId, updatedCustomObject);
return new JsonResponse.Builder<String>(Response.Status.OK)
.success(true).build();
} catch (SubmarineRuntimeException e) {
diff --git
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/NotebookMapper.xml
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/NotebookMapper.xml
index 39bebe1..0d5ceac 100644
---
a/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/NotebookMapper.xml
+++
b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/NotebookMapper.xml
@@ -30,10 +30,13 @@
<resultMap id="NotebookEntityResultMap"
type="org.apache.submarine.server.notebook.database.entity.NotebookEntity"
extends="BaseEntityResultMap">
<result column="notebook_spec" jdbcType="VARCHAR" property="notebookSpec"
/>
<result column="notebook_status" jdbcType="VARCHAR"
property="notebookStatus" />
+ <result column="notebook_url" jdbcType="VARCHAR" property="notebookUrl" />
+ <result column="reason" jdbcType="VARCHAR" property="reason" />
+ <result column="deleted_time" property="deletedTime" />
</resultMap>
<sql id="Base_Column_List">
- id, notebook_spec, create_by, create_time, update_by, update_time,
notebook_status
+ id, notebook_spec, create_by, create_time, update_by, update_time,
notebook_status, notebook_url, reason, deleted_time
</sql>
<select id="selectAll" parameterType="java.lang.String"
resultMap="NotebookEntityResultMap">
@@ -55,10 +58,12 @@
</delete>
<insert id="insert"
parameterType="org.apache.submarine.server.notebook.database.entity.NotebookEntity">
- insert into notebook (id, notebook_spec, create_by, create_time,
update_by, update_time, notebook_status)
+ insert into notebook (id, notebook_spec, create_by, create_time,
update_by, update_time,
+ notebook_status, notebook_url, reason, deleted_time)
values (#{id,jdbcType=VARCHAR}, #{notebookSpec,jdbcType=VARCHAR},
- #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR},
now(),
- #{notebookStatus,jdbcType=VARCHAR})
+ #{createBy,jdbcType=VARCHAR}, #{createTime,jdbcType=TIMESTAMP},
#{updateBy,jdbcType=VARCHAR},
+ now(), #{notebookStatus,jdbcType=VARCHAR},
#{notebookUrl,jdbcType=VARCHAR},
+ #{reason,jdbcType=VARCHAR},null)
</insert>
<update id="update"
parameterType="org.apache.submarine.server.notebook.database.entity.NotebookEntity">
@@ -66,7 +71,11 @@
<set>
<if test="notebookSpec != null and notebookStatus != null">
notebook_spec = #{notebookSpec,jdbcType=VARCHAR},
+ create_time = #{createTime,jdbcType=TIMESTAMP},
notebook_status = #{notebookStatus,jdbcType=VARCHAR},
+ notebook_url = #{notebookUrl,jdbcType=VARCHAR},
+ reason = #{reason,jdbcType=VARCHAR},
+ deleted_time = #{deletedTime,jdbcType=TIMESTAMP},
</if>
update_time = now()
</set>
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/internal/InternalServiceManagerTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/internal/InternalServiceManagerTest.java
index a4575e3..bc96300 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/internal/InternalServiceManagerTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/internal/InternalServiceManagerTest.java
@@ -36,6 +36,9 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.HashMap;
+import java.util.Map;
+
public class InternalServiceManagerTest {
private final Logger LOG =
LoggerFactory.getLogger(InternalServiceManagerTest.class);
private InternalServiceManager internalServiceManager;
@@ -51,6 +54,7 @@ public class InternalServiceManagerTest {
@Test
public void testUpdateNotebook() {
+
Notebook notebook = new Notebook();
notebook.setNotebookId(new NotebookId());
notebook.setName("test");
@@ -59,12 +63,22 @@ public class InternalServiceManagerTest {
notebook.setStatus("running");
notebook.setUid("mock-user");
notebook.setUrl("http://submarine.org";);
+
+ when(notebookService.select(any(String.class))).thenReturn(notebook);
- when(notebookService.select(any(String.class))).thenReturn(notebook);
+ Map<String, Object> updateObject = new HashMap<>();
+ updateObject.put("name", "test");
+ updateObject.put("notebookId", new NotebookId());
+ updateObject.put("reason", "test");
+ updateObject.put("spec", new NotebookSpec());
+ updateObject.put("status", "running");
+ updateObject.put("uid", "mock-user");
+ updateObject.put("url", "http://submarine.org";);
+
when(notebookService.update(any(Notebook.class))).thenReturn(true);
assertEquals(internalServiceManager.updateCRStatus(CustomResourceType.Notebook,
- notebook.getNotebookId().toString(), "complete"), true);
+ updateObject.get("notebookId").toString(), updateObject), true);
}
@Test
@@ -73,11 +87,16 @@ public class InternalServiceManagerTest {
experimentEntity.setId("test");
experimentEntity.setExperimentSpec("");
experimentEntity.setExperimentStatus("running");
-
when(experimentService.select(any(String.class))).thenReturn(experimentEntity);
+
+ Map<String, Object> updateObject = new HashMap<>();
+ updateObject.put("id", "test");
+ updateObject.put("experimentSpec", "");
+ updateObject.put("status", "running");
+
when(experimentService.update(any(ExperimentEntity.class))).thenReturn(true);
assertEquals(internalServiceManager.updateCRStatus(CustomResourceType.TFJob,
- experimentEntity.getId(), "complete"), true);
+ experimentEntity.getId(), updateObject), true);
}
}
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/InternalServiceRestApiTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/InternalServiceRestApiTest.java
index 8fca2b3..c58374d 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/InternalServiceRestApiTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/InternalServiceRestApiTest.java
@@ -24,6 +24,9 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.HashMap;
+import java.util.Map;
+
import javax.ws.rs.core.Response;
import org.apache.submarine.server.api.common.CustomResourceType;
@@ -46,12 +49,12 @@ public class InternalServiceRestApiTest {
@Test
public void testUpdateCRStatus() {
when(internalServiceRestApi.updateEnvironment(any(String.class),
- any(String.class), any(String.class))).thenReturn(new JsonResponse.
+ any(String.class), any(Map.class))).thenReturn(new JsonResponse.
Builder<String>(Response.Status.OK).
success(true).build());
Response response =
internalServiceRestApi.updateEnvironment(CustomResourceType.
- Notebook.getCustomResourceType(), "notebookId", "running");
+ Notebook.getCustomResourceType(), "notebookId", new
HashMap<String, Object>());
assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
}
}
diff --git
a/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/handler/NotebookHandler.java
b/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/handler/NotebookHandler.java
index c27850f..00a75f2 100644
---
a/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/handler/NotebookHandler.java
+++
b/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/handler/NotebookHandler.java
@@ -25,12 +25,15 @@ import
org.apache.submarine.server.api.common.CustomResourceType;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.k8s.agent.util.RestClient;
import org.apache.submarine.server.submitter.k8s.model.NotebookCR;
+import org.apache.submarine.server.submitter.k8s.util.NotebookUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.reflect.TypeToken;
import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.CustomObjectsApi;
import io.kubernetes.client.openapi.models.CoreV1Event;
import io.kubernetes.client.openapi.models.V1PodList;
import io.kubernetes.client.util.Watch;
@@ -42,6 +45,9 @@ public class NotebookHandler extends CustomResourceHandler {
private static final Logger LOG =
LoggerFactory.getLogger(NotebookHandler.class);
private Watchable<CoreV1Event> watcher;
+
+ private CustomObjectsApi customObjectsApi;
+
private String podName;
public NotebookHandler() throws IOException {
super();
@@ -69,6 +75,8 @@ public class NotebookHandler extends CustomResourceHandler {
watcher = Watch.createWatch(client, call, new
TypeToken<Response<CoreV1Event>>(){}.getType());
+ customObjectsApi = new CustomObjectsApi();
+
} catch (ApiException e) {
e.printStackTrace();
}
@@ -77,30 +85,57 @@ public class NotebookHandler extends CustomResourceHandler {
@Override
public void run() {
+ Notebook notebook = null;
while (true) {
for (Response<CoreV1Event> event: watcher) {
- String reason = event.object.getReason();
+ String reason = event.object.getReason();
+ Object object = null;
+ try {
switch (reason) {
case "Created":
case "Scheduled":
- restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, Notebook.Status.STATUS_CREATING.getValue());
+ object =
customObjectsApi.getNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1,
+ namespace, NotebookCR.CRD_NOTEBOOK_PLURAL_V1, crName);
+ notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+ notebook.setStatus(Notebook.Status.STATUS_CREATING.getValue());
+ restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, notebook);
break;
case "Started":
- restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, Notebook.Status.STATUS_RUNNING.getValue());
+ object =
customObjectsApi.getNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1,
+ namespace, NotebookCR.CRD_NOTEBOOK_PLURAL_V1, crName);
+ notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+ notebook.setStatus(Notebook.Status.STATUS_RUNNING.getValue());
+ restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, notebook);
break;
case "Failed":
- restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, Notebook.Status.STATUS_FAILED.getValue());
+ object =
customObjectsApi.getNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1,
+ namespace, NotebookCR.CRD_NOTEBOOK_PLURAL_V1, crName);
+ notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+ notebook.setStatus(Notebook.Status.STATUS_FAILED.getValue());
+ restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, notebook);
break;
case "Pulling":
- restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, Notebook.Status.STATUS_PULLING.getValue());
+ object =
customObjectsApi.getNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1,
+ namespace, NotebookCR.CRD_NOTEBOOK_PLURAL_V1, crName);
+ notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+ notebook.setStatus(Notebook.Status.STATUS_PULLING.getValue());
+ restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, notebook);
break;
case "Killing":
- restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, Notebook.Status.STATUS_TERMINATING.getValue());
+ object =
customObjectsApi.getNamespacedCustomObject(NotebookCR.CRD_NOTEBOOK_GROUP_V1,
NotebookCR.CRD_NOTEBOOK_VERSION_V1,
+ namespace, NotebookCR.CRD_NOTEBOOK_PLURAL_V1, crName);
+ notebook = NotebookUtils.parseObject(object,
NotebookUtils.ParseOpt.PARSE_OPT_GET);
+
notebook.setStatus(Notebook.Status.STATUS_TERMINATING.getValue());
+ restClient.callStatusUpdate(CustomResourceType.Notebook,
this.resourceId, notebook);
+
LOG.info("Receive terminating event, exit progress");
return;
default:
LOG.info(String.format("Unprocessed event type:%s", reason));
}
+ } catch(ApiException e) {
+ LOG.error("error while accessing k8s", e);
+ }
}
}
}
diff --git
a/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/util/RestClient.java
b/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/util/RestClient.java
index 244b867..77114a3 100644
---
a/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/util/RestClient.java
+++
b/submarine-server/server-submitter/submarine-k8s-agent/src/main/java/org/apache/submarine/server/k8s/agent/util/RestClient.java
@@ -21,19 +21,17 @@ package org.apache.submarine.server.k8s.agent.util;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
-import org.apache.submarine.commons.utils.SubmarineConfVars;
-import org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.server.api.common.CustomResourceType;
-import org.apache.submarine.server.k8s.agent.SubmarineAgent;
import org.apache.submarine.server.rest.RestConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RestClient {
private static final Logger LOG = LoggerFactory.getLogger(RestClient.class);
- private final SubmarineConfiguration conf =
SubmarineConfiguration.getInstance();
+
private Client client = ClientBuilder.newClient();
private final String API_SERVER_URL;
public RestClient(String host, Integer port) {
@@ -42,18 +40,16 @@ public class RestClient {
API_SERVER_URL = String.format("http://%s:%d/";, host, port);
}
-
- public void callStatusUpdate(CustomResourceType type, String resourceId,
String status) {
- LOG.info("Targeting url:" + API_SERVER_URL);
- LOG.info("Targeting uri:" + API_SERVER_URL);
+ public void callStatusUpdate(CustomResourceType type, String resourceId,
Object updateObject) {
- String uri = String.format("api/%s/%s/%s/%s/%s", RestConstants.V1,
- RestConstants.INTERNAL, type.toString(), resourceId, status);
+ String uri = String.format("api/%s/%s/%s/%s", RestConstants.V1,
+ RestConstants.INTERNAL, type.toString(), resourceId);
LOG.info("Targeting uri:" + uri);
client.target(API_SERVER_URL)
- .path(uri)
- .request(MediaType.APPLICATION_JSON).post(null, String.class);
+ .path(uri)
+ .request(MediaType.APPLICATION_JSON)
+ .post(Entity.entity(updateObject, MediaType.APPLICATION_JSON),
String.class);
}
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index 18300e7..3a79f57 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -289,7 +289,7 @@ public class NotebookSpecParser {
List<V1Container> containers =
notebookCRSpec.getTemplate().getSpec().getContainers();
V1Container agentContainer = new V1Container();
agentContainer.setName("agent");
- agentContainer.setImage("apache/submarine:sidecar-agent-0.7.0-SNAPSHOT");
+ agentContainer.setImage("apache/submarine:agent-0.7.0-SNAPSHOT");
List<V1EnvVar> envVarList = new ArrayList<>();
V1EnvVar crTypeVar = new V1EnvVar();
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@submarine.apache.org
For additional commands, e-mail: dev-h...@submarine.apache.org
