This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 31f8d33 SUBMARINE-764. Fix environment spec parse errors
31f8d33 is described below
commit 31f8d332e3560df3e80d9d45db7fcb69e0061c36
Author: KUAN-HSUN-LI <[email protected]>
AuthorDate: Sun Apr 4 15:51:21 2021 +0800
SUBMARINE-764. Fix environment spec parse errors
### What is this PR for?
* Support pip dependencies in kernel spec
* Change the old dependencies in kernel spec to condaDependencies
### What type of PR is it?
[Improvement]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-764
### How should this be tested?
run the following command
```
curl -X POST -H "Content-Type: application/json" -d '
{
"name": "my-submarine-env1",
"dockerImage" : "continuumio/anaconda3",
"kernelSpec" : {
"name" : "team_default_python_3.7",
"channels" : ["defaults"],
"condaDependencies" :
["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0",
"alabaster=0.7.12=py37_0",
"anaconda=2020.02=py37_0",
"anaconda-client=1.7.2=py37_0",
"anaconda-navigator=1.9.12=py37_0"],
"pipDependencies" :
["apache-submarine==0.5.0",
"pyarrow==0.17.0"]
}
}
' http://127.0.0.1:32080/api/v1/environment
```
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? Yes
* Does this needs documentation? No
Author: KUAN-HSUN-LI <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #540 from KUAN-HSUN-LI/SUBMARINE-764 and squashes the following
commits:
117e82fa [KUAN-HSUN-LI] SUBMARINE-764. Fix environment spec parse errors
b220b37e [KUAN-HSUN-LI] SUBMARINE-764. Fix environment spec parse errors
75f89956 [KUAN-HSUN-LI] SUBMARINE-764. Fix environment spec parse errors
a9cd5f46 [KUAN-HSUN-LI] fix list environment test
9f656fa9 [KUAN-HSUN-LI] enable pip dependencies
---
dev-support/database/submarine-data.sql | 2 +-
dev-support/pysubmarine/openapi.json | 8 +++++-
.../submarine/server/api/spec/KernelSpec.java | 32 +++++++++++++++-------
.../server/AbstractSubmarineServerTest.java | 2 +-
.../server/rest/EnvironmentRestApiTest.java | 17 ++++++++----
.../server/rest/ExperimentRestApiTest.java | 14 ++++++++--
.../src/test/resources/environment/test_env_1.json | 11 +++++---
.../src/test/resources/environment/test_env_2.json | 7 +++--
.../src/test/resources/environment/test_env_3.json | 5 ++--
.../submitter/k8s/parser/ExperimentSpecParser.java | 4 +--
.../submitter/k8s/parser/NotebookSpecParser.java | 15 ++++++++--
.../submitter/k8s/ExperimentSpecParserTest.java | 6 ++--
.../apache/submarine/rest/ExperimentRestApiIT.java | 2 +-
13 files changed, 86 insertions(+), 39 deletions(-)
diff --git a/dev-support/database/submarine-data.sql
b/dev-support/database/submarine-data.sql
index be6300e..3130896 100644
--- a/dev-support/database/submarine-data.sql
+++ b/dev-support/database/submarine-data.sql
@@ -85,7 +85,7 @@ INSERT INTO `params` (`id`, `key`, `value`, `worker_index`)
VALUES
-- Records of environment
-- ----------------------------
INSERT INTO `environment` VALUES
-('environment_1600862964725_0001', 'notebook-env',
'{"name":"notebook-env","dockerImage":"apache/submarine:jupyter-notebook-0.6.0-SNAPSHOT","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"dependencies":[]}}',
'admin', '2020-09-21 14:00:05', 'admin', '2020-09-21 14:00:14');
+('environment_1600862964725_0001', 'notebook-env',
'{"name":"notebook-env","dockerImage":"apache/submarine:jupyter-notebook-0.6.0-SNAPSHOT","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"condaDependencies":[],"pipDependencies":[]}}',
'admin', '2020-09-21 14:00:05', 'admin', '2020-09-21 14:00:14');
-- ----------------------------
-- Records of experiment_templates
diff --git a/dev-support/pysubmarine/openapi.json
b/dev-support/pysubmarine/openapi.json
index 22694e9..0899dc9 100644
--- a/dev-support/pysubmarine/openapi.json
+++ b/dev-support/pysubmarine/openapi.json
@@ -396,7 +396,13 @@
"type" : "string"
}
},
- "dependencies" : {
+ "condaDependencies" : {
+ "type" : "array",
+ "items" : {
+ "type" : "string"
+ }
+ },
+ "pipDependencies" : {
"type" : "array",
"items" : {
"type" : "string"
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/KernelSpec.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/KernelSpec.java
index 851ff8d..05c14bd 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/KernelSpec.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/spec/KernelSpec.java
@@ -25,21 +25,25 @@ import java.util.List;
* Kernel Spec.
*/
public class KernelSpec {
-
+
/**
* Name of the kernel
*/
private String name;
-
+
/**
* Name of the channels
*/
private List<String> channels;
-
+
+ /**
+ * List of kernel conda dependencies
+ */
+ private List<String> condaDependencies;
/**
- * List of kernel dependencies
+ * List of kernel pip dependencies
*/
- private List<String> dependencies;
+ private List<String> pipDependencies;
public String getName() {
return name;
@@ -57,11 +61,19 @@ public class KernelSpec {
this.channels = channels;
}
- public List<String> getDependencies() {
- return dependencies;
+ public List<String> getCondaDependencies() {
+ return condaDependencies;
}
- public void setDependencies(List<String> dependencies) {
- this.dependencies = dependencies;
- }
+ public void setCondaDependencies(List<String> condaDependencies) {
+ this.condaDependencies = condaDependencies;
+ }
+
+ public List<String> getPipDependencies() {
+ return pipDependencies;
+ }
+
+ public void setPipDependencies(List<String> pipDependencies) {
+ this.pipDependencies = pipDependencies;
+ }
}
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
index 73f41c5..6f25b4d 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
@@ -492,7 +492,7 @@ public abstract class AbstractSubmarineServerTest {
Assert.assertEquals(env.getEnvironmentSpec().getDockerImage(),
"continuumio/miniconda3");
Assert.assertTrue(
- env.getEnvironmentSpec().getKernelSpec().getDependencies().size() ==
1);
+ env.getEnvironmentSpec().getKernelSpec().getCondaDependencies().size()
== 1);
}
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/EnvironmentRestApiTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/EnvironmentRestApiTest.java
index f057fe8..d2aadb8 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/EnvironmentRestApiTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/EnvironmentRestApiTest.java
@@ -47,12 +47,15 @@ public class EnvironmentRestApiTest {
private static String kernelName = "team_default_python_3";
private static String dockerImage = "continuumio/anaconda3";
private static List<String> kernelChannels = Arrays.asList("defaults",
"anaconda");
- private static List<String> kernelDependencies = Arrays.asList(
+ private static List<String> kernelCondaDependencies = Arrays.asList(
"_ipyw_jlab_nb_ext_conf=0.1.0=py37_0",
"alabaster=0.7.12=py37_0",
"anaconda=2020.02=py37_0",
"anaconda-client=1.7.2=py37_0",
"anaconda-navigator=1.9.12=py37_0");
+ private static List<String> kernelPipDependencies = Arrays.asList(
+ "apache-submarine==0.5.0",
+ "pyarrow==0.17.0");
private static GsonBuilder gsonBuilder = new GsonBuilder()
.registerTypeAdapter(EnvironmentId.class, new EnvironmentIdSerializer())
@@ -79,7 +82,8 @@ public class EnvironmentRestApiTest {
KernelSpec kernelSpec = new KernelSpec();
kernelSpec.setName(kernelName);
kernelSpec.setChannels(kernelChannels);
- kernelSpec.setDependencies(kernelDependencies);
+ kernelSpec.setCondaDependencies(kernelCondaDependencies);
+ kernelSpec.setPipDependencies(kernelPipDependencies);
EnvironmentSpec environmentSpec = new EnvironmentSpec();
environmentSpec.setDockerImage(dockerImage);
environmentSpec.setKernelSpec(kernelSpec);
@@ -113,7 +117,10 @@ public class EnvironmentRestApiTest {
assertEquals("foo", environment.getEnvironmentSpec().getName());
assertEquals(kernelName,
environment.getEnvironmentSpec().getKernelSpec().getName());
assertEquals(kernelChannels,
environment.getEnvironmentSpec().getKernelSpec().getChannels());
- assertEquals(kernelDependencies,
environment.getEnvironmentSpec().getKernelSpec().getDependencies());
+ assertEquals(kernelCondaDependencies,
+
environment.getEnvironmentSpec().getKernelSpec().getCondaDependencies());
+ assertEquals(kernelPipDependencies,
+ environment.getEnvironmentSpec().getKernelSpec().getPipDependencies());
assertEquals("continuumio/miniconda",
environment.getEnvironmentSpec().getDockerImage());
}
@@ -141,8 +148,8 @@ public class EnvironmentRestApiTest {
environment.getEnvironmentSpec().getKernelSpec().getName());
assertEquals(kernelChannels,
environment.getEnvironmentSpec().getKernelSpec().getChannels());
- assertEquals(kernelDependencies,
- environment.getEnvironmentSpec().getKernelSpec().getDependencies());
+ assertEquals(kernelCondaDependencies,
+
environment.getEnvironmentSpec().getKernelSpec().getCondaDependencies());
assertEquals("continuumio/miniconda",
environment.getEnvironmentSpec().getDockerImage());
}
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ExperimentRestApiTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ExperimentRestApiTest.java
index 844f53d..5ffff37 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ExperimentRestApiTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ExperimentRestApiTest.java
@@ -79,12 +79,16 @@ public class ExperimentRestApiTest {
private static final String dockerImage = "continuumio/anaconda3";
private static final String kernelSpecName = "team_default_python_3";
private static final List<String> kernelChannels = Arrays.asList("defaults",
"anaconda");
- private static final List<String> kernelDependencies = Arrays.asList(
+ private static final List<String> kernelCondaDependencies = Arrays.asList(
"_ipyw_jlab_nb_ext_conf=0.1.0=py37_0",
"alabaster=0.7.12=py37_0",
"anaconda=2020.02=py37_0",
"anaconda-client=1.7.2=py37_0",
"anaconda-navigator=1.9.12=py37_0");
+ private static final List<String> kernelPipDependencies = Arrays.asList(
+ "apache-submarine==0.5.0",
+ "pyarrow==0.17.0"
+ );
private final ExperimentId experimentId =
ExperimentId.newInstance(SubmarineServer.getServerTimeStamp(),
experimentCounter.incrementAndGet());
@@ -110,7 +114,8 @@ public class ExperimentRestApiTest {
actualExperiment.setExperimentId(experimentId);
kernelSpec.setName(kernelSpecName);
kernelSpec.setChannels(kernelChannels);
- kernelSpec.setDependencies(kernelDependencies);
+ kernelSpec.setCondaDependencies(kernelCondaDependencies);
+ kernelSpec.setPipDependencies(kernelPipDependencies);
meta.setName(metaName);
meta.setFramework(metaFramework);
meta.setNamespace(metaNamespace);
@@ -225,6 +230,9 @@ public class ExperimentRestApiTest {
assertEquals(dockerImage,
experiment.getSpec().getEnvironment().getDockerImage());
assertEquals(kernelChannels,
experiment.getSpec().getEnvironment().getKernelSpec().getChannels());
assertEquals(kernelSpecName,
experiment.getSpec().getEnvironment().getKernelSpec().getName());
- assertEquals(kernelDependencies,
experiment.getSpec().getEnvironment().getKernelSpec().getDependencies());
+ assertEquals(kernelCondaDependencies,
+
experiment.getSpec().getEnvironment().getKernelSpec().getCondaDependencies());
+ assertEquals(kernelPipDependencies,
+
experiment.getSpec().getEnvironment().getKernelSpec().getPipDependencies());
}
}
diff --git
a/submarine-server/server-core/src/test/resources/environment/test_env_1.json
b/submarine-server/server-core/src/test/resources/environment/test_env_1.json
index 7dd341a..cfef8b2 100644
---
a/submarine-server/server-core/src/test/resources/environment/test_env_1.json
+++
b/submarine-server/server-core/src/test/resources/environment/test_env_1.json
@@ -4,11 +4,14 @@
"kernelSpec" : {
"name" : "team_default_python_3.7",
"channels" : ["defaults"],
- "dependencies" :
- ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0",
+ "condaDependencies" :
+ ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0",
"alabaster=0.7.12=py37_0",
"anaconda=2020.02=py37_0",
"anaconda-client=1.7.2=py37_0",
- "anaconda-navigator=1.9.12=py37_0"]
+ "anaconda-navigator=1.9.12=py37_0"],
+ "pipDependencies" :
+ ["apache-submarine==0.5.0",
+ "pyarrow==0.17.0"]
}
-}
\ No newline at end of file
+}
diff --git
a/submarine-server/server-core/src/test/resources/environment/test_env_2.json
b/submarine-server/server-core/src/test/resources/environment/test_env_2.json
index 36c44ff..ea416fe 100644
---
a/submarine-server/server-core/src/test/resources/environment/test_env_2.json
+++
b/submarine-server/server-core/src/test/resources/environment/test_env_2.json
@@ -4,7 +4,8 @@
"kernelSpec" : {
"name" : "team_default_python_3.7",
"channels" : ["defaults"],
- "dependencies" :
- ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"]
+ "condaDependencies" :
+ ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"],
+ "pipDependencies" : []
}
-}
\ No newline at end of file
+}
diff --git
a/submarine-server/server-core/src/test/resources/environment/test_env_3.json
b/submarine-server/server-core/src/test/resources/environment/test_env_3.json
index e7e861d..6576126 100644
---
a/submarine-server/server-core/src/test/resources/environment/test_env_3.json
+++
b/submarine-server/server-core/src/test/resources/environment/test_env_3.json
@@ -4,7 +4,8 @@
"kernelSpec" : {
"name" : "team_default_python_3.7",
"channels" : ["defaults"],
- "dependencies" :
- ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"]
+ "condaDependencies" :
+ ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"],
+ "pipDependencies" : []
}
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
index 0a56b02..0cdb550 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
@@ -262,7 +262,7 @@ public class ExperimentSpecParser {
initContainer.setName(environment.getEnvironmentSpec().getName());
initContainer.setImage(environmentSpec.getDockerImage());
- if (environmentSpec.getKernelSpec().getDependencies().size() > 0) {
+ if (environmentSpec.getKernelSpec().getCondaDependencies().size() > 0)
{
String minVersion = "minVersion=\""
+ conf.getString(
@@ -297,7 +297,7 @@ public class ExperimentSpecParser {
createCommand.append(channel);
}
for (String dependency : environmentSpec.getKernelSpec()
- .getDependencies()) {
+ .getCondaDependencies()) {
createCommand.append(" ");
createCommand.append(dependency);
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
index bdb4b8b..34b94df 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/NotebookSpecParser.java
@@ -107,8 +107,8 @@ public class NotebookSpecParser {
StringBuffer installCommand = new StringBuffer();
installCommand.append(condaVersionValidationCommand);
- // If dependencies isn't empty
- if (kernel.getDependencies().size() > 0) {
+ // If conda dependencies isn't empty
+ if (kernel.getCondaDependencies().size() > 0) {
installCommand.append(" && conda install -y");
for (String channel : kernel.getChannels()) {
installCommand.append(" ");
@@ -116,7 +116,16 @@ public class NotebookSpecParser {
installCommand.append(" ");
installCommand.append(channel);
}
- for (String dependency : kernel.getDependencies()) {
+ for (String dependency : kernel.getCondaDependencies()) {
+ installCommand.append(" ");
+ installCommand.append(dependency);
+ }
+ }
+
+ // If pip dependencies isn't empty
+ if (kernel.getPipDependencies().size() > 0) {
+ installCommand.append(" && pip install");
+ for (String dependency : kernel.getPipDependencies()) {
installCommand.append(" ");
installCommand.append(dependency);
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
index ae4af9f..0ba80e2 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java
@@ -220,7 +220,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
List<String> dependencies = new ArrayList<String>();
String dependency = "_ipyw_jlab_nb_ext_conf=0.1.0=py37_0";
dependencies.add(dependency);
- kernelSpec.setDependencies(dependencies);
+ kernelSpec.setCondaDependencies(dependencies);
spec.setKernelSpec(kernelSpec);
environmentManager.createEnvironment(spec);
@@ -266,7 +266,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
environmentManager.deleteEnvironment(envName);
}
-
+
@Test
public void testValidPyTorchJobSpecWithHTTPGitCodeLocalizer()
throws IOException, URISyntaxException, InvalidSpecException {
@@ -311,7 +311,7 @@ public class ExperimentSpecParserTest extends SpecBuilder {
Assert.assertEquals(AbstractCodeLocalizer.CODE_LOCALIZER_MOUNT_NAME,
V1Volume.getName());
}
-
+
@Test
public void testValidPyTorchJobSpecWithSSHGitCodeLocalizer()
throws IOException, URISyntaxException, InvalidSpecException {
diff --git
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
index 736fd64..44b7c68 100644
---
a/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
+++
b/submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiIT.java
@@ -359,7 +359,7 @@ public class ExperimentRestApiIT extends
AbstractSubmarineServerTest {
String dependencies = "";
for (String dependency : env.getEnvironmentSpec().getKernelSpec()
- .getDependencies()) {
+ .getCondaDependencies()) {
dependencies += " " + dependency;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
