Optimize resource list fetching
Project: http://git-wip-us.apache.org/repos/asf/helix/repo Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/3c3f289b Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/3c3f289b Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/3c3f289b Branch: refs/heads/master Commit: 3c3f289be18e5ea585170937be8507c6ada22fd5 Parents: 12f24c5 Author: Vivo Xu <[email protected]> Authored: Wed Feb 21 13:55:42 2018 -0800 Committer: Vivo Xu <[email protected]> Committed: Wed Aug 8 15:31:47 2018 -0700 ---------------------------------------------------------------------- .../resource-list/resource-list.component.ts | 25 +++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/helix/blob/3c3f289b/helix-front/client/app/resource/resource-list/resource-list.component.ts ---------------------------------------------------------------------- diff --git a/helix-front/client/app/resource/resource-list/resource-list.component.ts b/helix-front/client/app/resource/resource-list/resource-list.component.ts index fb9db0a..a7be734 100644 --- a/helix-front/client/app/resource/resource-list/resource-list.component.ts +++ b/helix-front/client/app/resource/resource-list/resource-list.component.ts @@ -71,38 +71,29 @@ export class ResourceListComponent implements OnInit { // since resource list contains also workflows and jobs // need to subtract them from original resource list // to obtain all jobs list, need to go through every workflow - // and perform get request for each + // and perform get request for each. + // However, it has huge performance issue when there are thousands of + // workflows. We are using a smart way here: to remove resources whose + // prefix is a workflow name protected fetchResources() { - let jobs = []; - this.isLoading = true; this.resources = null; this.workflowService .getAll(this.clusterName) - .concatMap(workflows => Observable.from(workflows)) - .mergeMap(workflow => { - const name = workflow as string; - jobs.push(name); - return this.workflowService.get(this.clusterName, name); - }) - .map(workflow => workflow.jobs) .subscribe( - list => { - jobs = jobs.concat(list); - }, - error => this.helper.showError(error), - () => { + workflows => { this.service .getAll(this.clusterName) .subscribe( result => { - this.resources = _.differenceWith(result, jobs, (resource: Resource, name) => resource.name === name); + this.resources = _.differenceWith(result, workflows, (resource: Resource, prefix: string) => _.startsWith(resource.name, prefix)); }, error => this.helper.showError(error), () => this.isLoading = false ); - } + }, + error => this.helper.showError(error) ); }
