This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 4832fb2 SUBMARINE-774. Support parsing conda config file
4832fb2 is described below
commit 4832fb2f89ec8d25e31f7c15296e1c5e1a3d72a7
Author: KUAN-HSUN-LI <[email protected]>
AuthorDate: Sat Apr 24 02:02:12 2021 +0800
SUBMARINE-774. Support parsing conda config file
### What is this PR for?
1. Improve the UI, created in #534
2. Parse the uploaded conda config file
3. Enable the updated API in #540
4. remove the kernelSpec block
### What type of PR is it?
[Improvement]
### Todos
- [ ] (test the parsing function)
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-785
### How should this be tested?
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? Yes
* Does this needs documentation? No
Author: KUAN-HSUN-LI <[email protected]>
Signed-off-by: Liu Xun <[email protected]>
Closes #550 from KUAN-HSUN-LI/SUBMARINE-774 and squashes the following
commits:
3fef78d [KUAN-HSUN-LI] SUBMARINE-774. Update environment e2e environment
test resourse's License
bd90979 [KUAN-HSUN-LI] SUBMARINE-774. Update environment e2e test
b0984cf [KUAN-HSUN-LI] SUBMARINE-774. Change error msg
eb526ad [KUAN-HSUN-LI] SUBMARINE-774. Support parsing conda config file
---
.../submarine/integration/environmentIT.java | 7 +-
.../test-e2e/src/test/resources/test_config_1.yml | 27 ++++
submarine-workbench/workbench-web/package.json | 1 +
.../environment-form.component.html | 101 ++------------
.../environment-form/environment-form.component.ts | 147 ++++++++-------------
5 files changed, 96 insertions(+), 187 deletions(-)
diff --git
a/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/environmentIT.java
b/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/environmentIT.java
index edd348a..3711279 100644
---
a/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/environmentIT.java
+++
b/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/environmentIT.java
@@ -63,11 +63,8 @@ public class environmentIT extends AbstractSubmarineIT {
pollingWait(By.xpath("//button[@id='btn-newEnvironment']"),
MAX_BROWSER_TIMEOUT_SEC).click();
pollingWait(By.cssSelector("input[ng-reflect-name='environmentName']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("testEnvName");
pollingWait(By.cssSelector("input[ng-reflect-name='dockerImage']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("testDockerImage");
- pollingWait(By.cssSelector("input[ng-reflect-name='name']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("testName");
- pollingWait(By.xpath("//button[@id='addChannel-btn']"),
MAX_BROWSER_TIMEOUT_SEC).click();
- pollingWait(By.xpath("//input[@id='channel0']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("testChannel");
- pollingWait(By.xpath("//button[@id='addDep-btn']"),
MAX_BROWSER_TIMEOUT_SEC).click();
- pollingWait(By.xpath("//input[@id='dependencies0']"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys("testDep");
+ pollingWait(By.xpath("//nz-upload[@id='upload-config']"),
MAX_BROWSER_TIMEOUT_SEC).click();
+ pollingWait(By.cssSelector("input[type=file]"),
MAX_BROWSER_TIMEOUT_SEC).sendKeys(System.getProperty("user.dir") +
"/src/test/resources/test_config_1.yml");
Assert.assertEquals(pollingWait(By.xpath("//button[@id='btn-submit']"),
MAX_BROWSER_TIMEOUT_SEC).isDisplayed(), true);
pollingWait(By.xpath("//button[@id='btn-submit']"),
MAX_BROWSER_TIMEOUT_SEC).click();
Assert.assertEquals(pollingWait(By.xpath("//button[@id='btn-newEnvironment']"),
MAX_BROWSER_TIMEOUT_SEC).isDisplayed(), true);
diff --git a/submarine-test/test-e2e/src/test/resources/test_config_1.yml
b/submarine-test/test-e2e/src/test/resources/test_config_1.yml
new file mode 100644
index 0000000..56cc79a
--- /dev/null
+++ b/submarine-test/test-e2e/src/test/resources/test_config_1.yml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: test
+channels:
+ - defaults
+dependencies:
+ - _ipyw_jlab_nb_ext_conf=0.1.0=py37_0
+ - alabaster=0.7.12=py37_0
+ - anaconda=2020.02=py37_0
+ - anaconda-client=1.7.2=py37_0
+ - anaconda-navigator=1.9.12=py37_0
+ - pip:
+ - apache-submarine==0.5.0
+ - pyarrow==0.17.0
diff --git a/submarine-workbench/workbench-web/package.json
b/submarine-workbench/workbench-web/package.json
index 5548bad..1044930 100644
--- a/submarine-workbench/workbench-web/package.json
+++ b/submarine-workbench/workbench-web/package.json
@@ -30,6 +30,7 @@
"ng-zorro-antd": "8.1.2",
"rxjs": "~6.4.0",
"tslib": "^1.10.0",
+ "yaml": "^1.10.2",
"zone.js": "~0.9.1"
},
"lint-staged": {
diff --git
a/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.html
b/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.html
index b17f5bb..fa72437 100644
---
a/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.html
+++
b/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.html
@@ -59,103 +59,20 @@
</div>
</nz-form-item>
<nz-form-item style="margin-bottom: 0">
- <!-- nzAction is not supported yet -->
- <nz-upload
- nzType="drag"
- [nzMultiple]="false"
- [nzFileList]="configFileList"
- [nzPreview]="isPreview"
- [nzRemove]="closePreview"
- nzAccept=".yml"
- nzAction="api/v1/environment/config"
- (nzChange)="fileUpload($event)"
- >
+ <nz-upload id="upload-config" nzType="drag" [nzMultiple]="false"
[nzBeforeUpload]="beforeUpload">
<p class="ant-upload-drag-icon">
<i nz-icon nzType="inbox"></i>
</p>
<p class="ant-upload-text">Click or drag a conda config YAML file
here</p>
+ <div style="text-align: left; margin: 1rem">
+ <div
+ #preview
+ style="overflow-y: auto"
+ [style.height]="previewCondaConfig !== '' ? '8rem' : '0rem'"
+ [innerText]="previewCondaConfig"
+ ></div>
+ </div>
</nz-upload>
</nz-form-item>
- <div #preview *ngIf="isPreviewVisible"
[innerText]="previewConfigFile"></div>
- <h2>Kernel Spec</h2>
- <nz-form-item>
- <nz-form-label [nzSm]="6" [nzXs]="24" nzRequired
nzFor="name">Name</nz-form-label>
- <div nz-col nzSpan="16">
- <nz-form-control nzErrorTip="Please input name!">
- <input required nz-input style="width: 80%" type="text" name="name"
id="name" formControlName="name" />
- </nz-form-control>
- </div>
- </nz-form-item>
- <div formArrayName="channels">
- <div *ngFor="let channel of channels.controls; let i = index">
- <nz-form-item>
- <nz-form-label [nzSm]="6" [nzXs]="24" nzRequired
nzFor="channel">Channel {{ i + 1 }}</nz-form-label>
- <div nz-col nzSpan="16">
- <nz-form-control nzErrorTip="Please input channel!">
- <input
- required
- nz-input
- style="width: 80%"
- type="text"
- name="channel{{ i }}"
- id="channel{{ i }}"
- [formControlName]="i"
- />
- <i
- nz-icon
- style="margin-left: 5px"
- nzType="close-circle"
- nzTheme="fill"
- (click)="deleteItem(channels, i)"
- ></i>
- </nz-form-control>
- </div>
- </nz-form-item>
- </div>
- </div>
- <div style="margin: 10px">
- <button nz-button style="display: block; margin: auto"
id="addChannel-btn" type="default" (click)="addChannel()">
- New Channel
- </button>
- </div>
- <div formArrayName="dependencies">
- <div *ngFor="let channel of dependencies.controls; let i = index">
- <nz-form-item>
- <nz-form-label [nzSm]="6" [nzXs]="24" nzRequired
nzFor="dependency">Dependency {{ i + 1 }}</nz-form-label>
- <div nz-col nzSpan="16">
- <nz-form-control nzErrorTip="Please input dependency!">
- <input
- required
- nz-input
- style="width: 80%"
- type="text"
- name="dependencies{{ i }}"
- id="dependencies{{ i }}"
- [formControlName]="i"
- />
- <i
- nz-icon
- style="margin-left: 5px"
- nzType="close-circle"
- nzTheme="fill"
- (click)="deleteItem(dependencies, i)"
- ></i>
- </nz-form-control>
- </div>
- </nz-form-item>
- </div>
- </div>
- <div style="margin: 10px">
- <button
- style="margin-top: 10px"
- nz-button
- style="display: block; margin: auto"
- id="addDep-btn"
- type="default"
- (click)="addDependencies()"
- >
- New Dependency
- </button>
- </div>
</form>
</nz-modal>
diff --git
a/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.ts
b/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.ts
index 91a408d..45e44ed 100644
---
a/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.ts
+++
b/submarine-workbench/workbench-web/src/app/pages/workbench/environment/environment-home/environment-form/environment-form.component.ts
@@ -18,10 +18,10 @@
*/
import { Component, EventEmitter, OnInit, Output } from '@angular/core';
-import { FormArray, FormBuilder, Validators } from '@angular/forms';
+import { FormBuilder, Validators } from '@angular/forms';
import { EnvironmentService } from
'@submarine/services/environment-services/environment.service';
import { NzMessageService } from 'ng-zorro-antd';
-import { UploadChangeParam, UploadFile } from 'ng-zorro-antd/upload';
+import { parse } from 'yaml';
@Component({
selector: 'submarine-environment-form',
@@ -33,9 +33,7 @@ export class EnvironmentFormComponent implements OnInit {
isVisible: boolean;
environmentForm;
- configFileList = [];
- isPreviewVisible: boolean;
- previewConfigFile = '';
+ previewCondaConfig = '';
constructor(
private fb: FormBuilder,
@@ -47,16 +45,11 @@ export class EnvironmentFormComponent implements OnInit {
this.environmentForm = this.fb.group({
environmentName: [null, Validators.required],
dockerImage: [null, Validators.required],
- configFile: null,
- name: [null, Validators.required],
- channels: this.fb.array([]),
- dependencies: this.fb.array([]),
});
}
initModal() {
this.isVisible = true;
- this.isPreviewVisible = true;
this.initFormStatus();
}
@@ -72,91 +65,72 @@ export class EnvironmentFormComponent implements OnInit {
return this.environmentForm.get('dockerImage');
}
- get configFile() {
- return this.environmentForm.get('configFile');
- }
-
- set configFile(file: UploadFile) {
- this.environmentForm.controls['configFile'].setValue(file);
- }
-
- get name() {
- return this.environmentForm.get('name');
- }
-
- get channels() {
- return this.environmentForm.get('channels') as FormArray;
- }
-
- get dependencies() {
- return this.environmentForm.get('dependencies') as FormArray;
- }
-
initFormStatus() {
this.isVisible = true;
this.environmentName.reset();
this.dockerImage.reset();
- this.configFile.reset();
- this.name.reset();
- this.channels.clear();
- this.dependencies.clear();
}
checkStatus() {
- return (
- this.environmentName.invalid ||
- this.dockerImage.invalid ||
- this.name.invalid ||
- this.channels.invalid ||
- this.dependencies.invalid
- );
+ return this.environmentName.invalid || this.dockerImage.invalid;
}
closeModal() {
this.isVisible = false;
- this.configFileList = [];
- this.previewConfigFile = '';
- }
-
- addChannel() {
- this.channels.push(this.fb.control(null, Validators.required));
- }
-
- addDependencies() {
- this.dependencies.push(this.fb.control(null, Validators.required));
+ this.previewCondaConfig = '';
}
- deleteItem(arr: FormArray, index: number) {
- arr.removeAt(index);
- }
+ beforeUpload = (file: File): boolean => {
+ let reader = new FileReader();
+ reader.readAsText(file);
+ reader.onload = () => {
+ this.previewCondaConfig = reader.result.toString();
+ this.nzMessageService.success(`${file.name} file read successfully.`);
+ const config = parse(reader.result.toString());
+ };
+ return false;
+ };
- fileUpload(info: UploadChangeParam) {
- info.fileList = info.fileList.slice(-1);
- this.configFileList = info.fileList;
- let file = info.file;
- if (info.type === 'success') {
- this.configFile = file;
- this.isPreviewVisible = true;
- var reader = new FileReader();
- reader.readAsText(info.file.originFileObj);
- reader.onload = () => {
- this.previewConfigFile = reader.result.toString();
- };
- this.nzMessageService.success(`${file.name} file uploaded
successfully.`);
- } else if (status === 'error') {
- this.nzMessageService.error(`${file.name} file upload failed.`);
+ checkCondaConfig(config): Object {
+ if (config === null) {
+ config = {};
+ }
+ if (!config['channels']) {
+ config['channels'] = [];
+ }
+ if (!config['name']) {
+ config['name'] = '';
}
+ config['condaDependencies'] = [];
+ config['pipDependencies'] = [];
+ return config;
+ }
+
+ parseCondaConfig(): Object {
+ let config = this.checkCondaConfig(parse(this.previewCondaConfig));
+ this.previewCondaConfig = '';
+ try {
+ if (config['dependencies'] !== undefined || null) {
+ config['dependencies'].map((e: object | string) => {
+ if (typeof e === 'object') {
+ if (!e['pip']) {
+ this.nzMessageService.error('dependencies include unknown
object');
+ throw Error('dependencies include unknown object');
+ } else {
+ config['pipDependencies'] = e['pip'];
+ }
+ } else if (typeof e === 'string') {
+ config['condaDependencies'].push(e);
+ }
+ });
+ }
+ } catch (error) {
+ this.nzMessageService.error('Unable to parse the conda config file');
+ throw error;
+ }
+ return config;
}
- isPreview = () => {
- this.isPreviewVisible = !this.isPreviewVisible;
- };
-
- closePreview = () => {
- this.configFileList = [];
- this.isPreviewVisible = false;
- };
-
createEnvironment() {
this.isVisible = false;
const newEnvironmentSpec = this.createEnvironmentSpec();
@@ -174,24 +148,17 @@ export class EnvironmentFormComponent implements OnInit {
}
createEnvironmentSpec() {
+ let config = this.parseCondaConfig();
const environmentSpec = {
name: this.environmentForm.get('environmentName').value,
dockerImage: this.environmentForm.get('dockerImage').value,
kernelSpec: {
- name: this.environmentForm.get('name').value,
- channels: [],
- dependencies: [],
+ name: config['name'],
+ channels: config['channels'],
+ condaDependencies: config['condaDependencies'],
+ pipDependencies: config['pipDependencies'],
},
};
-
- for (const channel of this.channels.controls) {
- environmentSpec.kernelSpec.channels.push(channel.value);
- }
-
- for (const dependency of this.dependencies.controls) {
- environmentSpec.kernelSpec.dependencies.push(dependency.value);
- }
-
return environmentSpec;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
