Repository: helix Updated Branches: refs/heads/master 48e56fa76 -> f1c503712
Workflow: job dag for generic workflows Project: http://git-wip-us.apache.org/repos/asf/helix/repo Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/b486d784 Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/b486d784 Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/b486d784 Branch: refs/heads/master Commit: b486d7849a017fe4a5c1df6fcc7ebb18a6c646d1 Parents: 48e56fa Author: Vivo Xu <[email protected]> Authored: Thu Feb 1 18:13:46 2018 -0800 Committer: Vivo Xu <[email protected]> Committed: Wed Aug 8 15:06:37 2018 -0700 ---------------------------------------------------------------------- .../app/workflow/shared/workflow.model.ts | 18 +-- .../workflow-dag/workflow-dag.component.html | 53 +++++++++ .../workflow-dag/workflow-dag.component.scss | 13 +++ .../workflow-dag/workflow-dag.component.spec.ts | 33 ++++++ .../workflow-dag/workflow-dag.component.ts | 63 ++++++++++ .../workflow-detail.component.html | 115 +++++++++---------- .../workflow-detail.component.ts | 7 +- .../client/app/workflow/workflow.module.ts | 10 +- helix-front/package.json | 3 + 9 files changed, 245 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/shared/workflow.model.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/shared/workflow.model.ts b/helix-front/client/app/workflow/shared/workflow.model.ts index 096af32..363c8a7 100644 --- a/helix-front/client/app/workflow/shared/workflow.model.ts +++ b/helix-front/client/app/workflow/shared/workflow.model.ts @@ -9,18 +9,22 @@ export class Job { readonly rawName: string; readonly startTime: string; readonly state: string; + readonly parents: string[]; constructor( rawName: string, workflowName: string, startTime: string, - state: string + state: string, + parents: string[] ) { this.rawName = rawName; // try to reduce the name this.name = _.replace(rawName, workflowName + '_', ''); this.startTime = startTime; this.state = state; + // try to reduce parent names + this.parents = _.map(parents, parent => _.replace(parent, workflowName + '_', '')); } } @@ -28,9 +32,8 @@ export class Workflow { readonly name: string; readonly config: any; readonly jobs: Job[]; - // TODO vxu: will use a better structure for this - readonly parentJobs: any[]; readonly context: any; + readonly json: any; get isJobQueue(): boolean { return this.config && this.config.IsJobQueue.toLowerCase() == 'true'; @@ -41,14 +44,14 @@ export class Workflow { } constructor(obj: any) { + this.json = obj; this.name = obj.id; this.config = obj.WorkflowConfig; this.context = obj.WorkflowContext; - this.jobs = this.parseJobs(obj.Jobs); - this.parentJobs = obj.ParentJobs; + this.jobs = this.parseJobs(obj.Jobs, obj.ParentJobs); } - protected parseJobs(list: string[]): Job[] { + protected parseJobs(list: string[], parents: any): Job[] { let result: Job[] = []; _.forEach(list, jobName => { @@ -56,7 +59,8 @@ export class Workflow { jobName, this.name, _.get(this.context, ['StartTime', jobName]), - _.get(this.context, ['JOB_STATES', jobName]) + _.get(this.context, ['JOB_STATES', jobName]), + parents[jobName] )); }); http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html new file mode 100644 index 0000000..c44d369 --- /dev/null +++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html @@ -0,0 +1,53 @@ +<ngx-graph + #graph + class="chart-container" + [view]="[graph.graphDims.width * 2 / 3, graph.graphDims.height]" + [links]="data.links" + [nodes]="data.nodes" + [curve]="curve" + orientation="TB" + (select)="select($event)" + [autoZoom]="false" + [panningEnabled]="false" + [draggingEnabled]="false" + [minZoomLevel]="1" + [maxZoomLevel]="1"> + + <ng-template #defsTemplate> + <svg:marker id="arrow" viewBox="0 -5 10 10" refX="8" refY="0" markerWidth="4" markerHeight="4" orient="auto"> + <svg:path d="M0,-5L10,0L0,5" class="arrow-head" /> + </svg:marker> + </ng-template> + + <ng-template #nodeTemplate let-node> + <svg:g class="node" + ngx-tooltip + [tooltipPlacement]="'top'" + [tooltipType]="'tooltip'" + [tooltipTitle]="node.description"> + <svg:rect [attr.width]="node.width" [attr.height]="node.height" [attr.fill]="node.options.color" /> + <svg:text alignment-baseline="central" [attr.x]="10" [attr.y]="node.height / 2">{{ node.label }}</svg:text> + <svg:text alignment-baseline="hanging" [attr.x]="node.width - 100" [attr.y]="node.height + 10" [ngClass]="'state-default state-' + node.state">{{ node.state }}</svg:text> + </svg:g> + </ng-template> + + <ng-template #linkTemplate let-link> + <svg:g class="edge"> + <svg:path + class="line" + stroke-width="2" + marker-end="url(#arrow)" > + </svg:path> + <svg:text class="edge-label" text-anchor="middle"> + <textPath + class="text-path" + [attr.href]="'#' + link.id" + [style.dominant-baseline]="link.dominantBaseline" + startOffset="50%"> + {{link.label}} + </textPath> + </svg:text> + </svg:g> + </ng-template> + +</ngx-graph> http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss new file mode 100644 index 0000000..87fd120 --- /dev/null +++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss @@ -0,0 +1,13 @@ +@import '~@angular/material/theming'; + +.state-default { + fill: mat-color(mat-palette($mat-deep-orange)); +} + +.state-COMPLETED { + fill: mat-color(mat-palette($mat-green)); +} + +.state-PENDING { + fill: mat-color(mat-palette($mat-grey)); +} http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts new file mode 100644 index 0000000..40b600c --- /dev/null +++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts @@ -0,0 +1,33 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { NO_ERRORS_SCHEMA } from '@angular/core'; + +import { TestingModule } from '../../../testing/testing.module'; +import { WorkflowDagComponent } from './workflow-dag.component'; + +describe('WorkflowDagComponent', () => { + let component: WorkflowDagComponent; + let fixture: ComponentFixture<WorkflowDagComponent>; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + imports: [ TestingModule ], + declarations: [ WorkflowDagComponent ], + schemas: [ + /* avoid importing modules */ + NO_ERRORS_SCHEMA + ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(WorkflowDagComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + xit('should create', () => { + expect(component).toBeTruthy(); + }); +}); http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts new file mode 100644 index 0000000..e94d913 --- /dev/null +++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts @@ -0,0 +1,63 @@ +import { Component, OnInit, Input, ElementRef, AfterViewInit, ViewChild } from '@angular/core'; + +import * as shape from 'd3-shape'; +import * as _ from 'lodash'; + +import { Workflow, Job } from '../shared/workflow.model'; + +@Component({ + selector: 'hi-workflow-dag', + templateUrl: './workflow-dag.component.html', + styleUrls: ['./workflow-dag.component.scss'] +}) +export class WorkflowDagComponent implements OnInit, AfterViewInit { + + @Input() + workflow: Workflow; + curve: any = shape.curveLinear; + view = [800, 600]; + data = { + nodes: [], + links: [] + }; + jobNameToId = {}; + + @ViewChild('graph') + graph; + + constructor(protected el:ElementRef) { } + + ngOnInit() { + this.loadJobs(); + } + + ngAfterViewInit() { + // console.log(this.el); + // console.log(this.graph); + } + + protected loadJobs() { + // process nodes + _.forEach(this.workflow.jobs, (job: Job) => { + const newId = (this.data.nodes.length + 1).toString(); + + this.data.nodes.push({ + id: newId, + label: job.name, + description: job.rawName, + state: job.state + }); + this.jobNameToId[job.name] = newId; + }); + + // process edges/links + _.forEach(this.workflow.jobs, (job: Job) => { + _.forEach(job.parents, parentName => { + this.data.links.push({ + source: this.jobNameToId[parentName], + target: this.jobNameToId[job.name] + }); + }); + }); + } +} http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html index 0791995..5a940d3 100644 --- a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html +++ b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html @@ -18,69 +18,68 @@ <section fxLayout="column" fxLayoutAlign="center center"> <mat-spinner *ngIf="isLoading"></mat-spinner> <section *ngIf="!isLoading" class="content" fxLayout="column" fxLayoutAlign="center center" fxLayoutGap="10px" fxFlexFill> - <mat-button-toggle-group #group="matButtonToggleGroup" value="queue"> - <mat-button-toggle value="queue"> - Queue View + <mat-button-toggle-group #group="matButtonToggleGroup" [value]="workflow.isJobQueue ? 'list' : 'graph'"> + <mat-button-toggle *ngIf="!workflow.isJobQueue" value="graph"> + Graph View + </mat-button-toggle> + <mat-button-toggle value="list"> + List View </mat-button-toggle> <mat-button-toggle value="json"> JSON View </mat-button-toggle> </mat-button-toggle-group> - <section class="viewer" [ngSwitch]="group.value" fxFlexFill> - <section *ngSwitchCase="'queue'"> - <section *ngIf="workflow.isJobQueue"> - <ngx-datatable - #jobsTable - class="material" - [headerHeight]="headerHeight" - [rowHeight]="rowHeight" - columnMode="force" - [footerHeight]="rowHeight" - [rows]="workflow.jobs" - selectionType="single" - [sorts]="sorts" - (select)="onSelect($event)" - [messages]="messages"> - <ngx-datatable-column - name="Start Time" - [width]="200" - [resizeable]="false" - [draggable]="false" - [canAutoResize]="false"> - <ng-template let-value="value" ngx-datatable-cell-template> - <span *ngIf="value" [matTooltip]="value | date:'medium'"> - {{ parseTime(value) }} - </span> - <span *ngIf="!value">-</span> - </ng-template> - </ngx-datatable-column> - <ngx-datatable-column name="Job Name" prop="name"> - <ng-template let-row="row" let-value="value" ngx-datatable-cell-template> - <span [matTooltip]="row.rawName"> - ...{{ value }} - </span> - </ng-template> - </ngx-datatable-column> - <ngx-datatable-column - name="State" - [width]="120" - [resizeable]="false" - [draggable]="false" - [canAutoResize]="false"> - <ng-template let-value="value" ngx-datatable-cell-template> - <span *ngIf="value" class="state-default state-{{ value }}"> - {{ value }} - </span> - <span *ngIf="!value" class="state-PENDING">PENDING</span> - </ng-template> - </ngx-datatable-column> - </ngx-datatable> - </section> - <section *ngIf="!workflow.isJobQueue"> - {{ workflow | json }} - </section> - </section> - <ngx-json-viewer *ngSwitchCase="'json'" [json]="workflow"></ngx-json-viewer> + <section class="viewer" [ngSwitch]="group.value" fxLayout="column" fxLayoutAlign="center center" fxFill> + <hi-workflow-dag *ngSwitchCase="'graph'" [workflow]="workflow"></hi-workflow-dag> + <ngx-datatable + *ngSwitchCase="'list'" + #jobsTable + class="material" + [headerHeight]="headerHeight" + [rowHeight]="rowHeight" + columnMode="force" + [footerHeight]="rowHeight" + [rows]="workflow.jobs" + selectionType="single" + [sorts]="sorts" + (select)="onSelect($event)" + [messages]="messages" + fxFill> + <ngx-datatable-column + name="Start Time" + [width]="200" + [resizeable]="false" + [draggable]="false" + [canAutoResize]="false"> + <ng-template let-value="value" ngx-datatable-cell-template> + <span *ngIf="value" [matTooltip]="value | date:'medium'"> + {{ parseTime(value) }} + </span> + <span *ngIf="!value">-</span> + </ng-template> + </ngx-datatable-column> + <ngx-datatable-column name="Job Name" prop="name"> + <ng-template let-row="row" let-value="value" ngx-datatable-cell-template> + <span [matTooltip]="row.rawName"> + ...{{ value }} + </span> + </ng-template> + </ngx-datatable-column> + <ngx-datatable-column + name="State" + [width]="120" + [resizeable]="false" + [draggable]="false" + [canAutoResize]="false"> + <ng-template let-value="value" ngx-datatable-cell-template> + <span *ngIf="value" class="state-default state-{{ value }}"> + {{ value }} + </span> + <span *ngIf="!value" class="state-PENDING">PENDING</span> + </ng-template> + </ngx-datatable-column> + </ngx-datatable> + <ngx-json-viewer *ngSwitchCase="'json'" [json]="workflow.json" fxFill></ngx-json-viewer> </section> </section> </section> http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts index 8979d13..f1f04ad 100644 --- a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts +++ b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts @@ -2,9 +2,11 @@ import { Component, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import * as moment from 'moment'; +import * as shape from 'd3-shape'; +import * as _ from 'lodash'; import { Settings } from '../../core/settings'; -import { Workflow } from '../shared/workflow.model'; +import { Workflow, Job } from '../shared/workflow.model'; import { WorkflowService } from '../shared/workflow.service'; @Component({ @@ -25,7 +27,7 @@ export class WorkflowDetailComponent implements OnInit { { prop: 'name', dir: 'asc'} ]; messages = { - emptyMessage: 'The queue is empty.', + emptyMessage: 'The list is empty.', totalMessage: 'total', selectedMessage: 'selected' }; @@ -58,5 +60,4 @@ export class WorkflowDetailComponent implements OnInit { const row = selected[0]; // this.table.rowDetail.toggleExpandRow(row); } - } http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow.module.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/workflow/workflow.module.ts b/helix-front/client/app/workflow/workflow.module.ts index e218928..7ff9868 100644 --- a/helix-front/client/app/workflow/workflow.module.ts +++ b/helix-front/client/app/workflow/workflow.module.ts @@ -2,24 +2,30 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import { NgxDatatableModule } from '@swimlane/ngx-datatable'; +import { NgxChartsModule } from '@swimlane/ngx-charts'; +import { NgxGraphModule } from '@swimlane/ngx-graph'; import { WorkflowListComponent } from './workflow-list/workflow-list.component'; import { WorkflowService } from './shared/workflow.service'; import { SharedModule } from '../shared/shared.module'; import { WorkflowDetailComponent } from './workflow-detail/workflow-detail.component'; +import { WorkflowDagComponent } from './workflow-dag/workflow-dag.component'; @NgModule({ imports: [ CommonModule, SharedModule, - NgxDatatableModule + NgxDatatableModule, + NgxChartsModule, + NgxGraphModule ], providers: [ WorkflowService ], declarations: [ WorkflowListComponent, - WorkflowDetailComponent + WorkflowDetailComponent, + WorkflowDagComponent ] }) export class WorkflowModule { } http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/package.json ---------------------------------------------------------------------- diff --git a/helix-front/package.json b/helix-front/package.json index bd5fdc9..8c8d1b4 100644 --- a/helix-front/package.json +++ b/helix-front/package.json @@ -33,10 +33,13 @@ "@angular/platform-browser": "^5.1.1", "@angular/platform-browser-dynamic": "^5.1.1", "@angular/router": "^5.1.1", + "@swimlane/ngx-charts": "^7.0.1", "@swimlane/ngx-datatable": "^11.1.7", + "@swimlane/ngx-graph": "^4.0.2", "angulartics2": "^2.2.1", "body-parser": "^1.17.2", "core-js": "^2.4.1", + "d3-shape": "^1.2.0", "dotenv": "^4.0.0", "express": "^4.15.3", "express-session": "^1.15.6",
