AMBARI-20519. Runtime filtering is not rendering properly (pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f5d5507 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f5d5507 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f5d5507 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 0f5d55077570821c7759a298d05a13104662b390 Parents: 845e921 Author: pallavkul <[email protected]> Authored: Wed Mar 22 22:31:47 2017 +0530 Committer: pallavkul <[email protected]> Committed: Wed Mar 22 22:31:47 2017 +0530 ---------------------------------------------------------------------- .../ui/app/utils/hive-explainer/processor.js | 98 ++++++++++++++++++++ .../ui/app/utils/hive-explainer/transformer.js | 71 +++++++++++--- 2 files changed, 154 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0f5d5507/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js index f6d3704..298366f 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js @@ -261,10 +261,108 @@ export function getEdgesWithCorrectedUnion(edges) { } +function findAllOutputOperators(vertices, outputOperatorsList, edges, patterns) { + vertices.forEach(cEdge => { + console.log(cEdge); + edges.push(cEdge); + let outputOperator = cEdge["OutputOperators:"]; + if(outputOperator) { + patterns.push({outputOperator:outputOperator.substring(1, outputOperator.length-1), cEdge:[edges[edges.length-4], edges[edges.length-3], edges[edges.length-2], edges[edges.length-1]]}); + outputOperatorsList.push({outputOperator:outputOperator.substring(1, outputOperator.length-1), cEdge:edges}); + } + findAllOutputOperators(cEdge._children, outputOperatorsList, edges, patterns); + }); +} +function findAllOperatorsVertex(cEdge, operatorIdList, edgesOp) { + cEdge.forEach(cChild => { + let operatorId = cChild["OperatorId:"]; + if(operatorId) { + let operatorObj = {}; + operatorObj[operatorId]= cChild["_operator"]; + operatorIdList.push(operatorObj); + edgesOp.push(cChild); + if(edgesOp.length > 4) { + edgesOp.pop(); + } + } + findAllOperatorsVertex(cChild._children, operatorIdList, edgesOp); + }); +} +function findTheOperatorIDChains(vertices, operatorIdList, edgesOp) { + vertices.forEach(cChild => { + let subOperatorList = []; + edgesOp = []; + findAllOperatorsVertex(cChild._children, subOperatorList, edgesOp); + operatorIdList.push({subOperatorList:subOperatorList, edgesOp:edgesOp}); + }); +} + +function findPatternParent(edges, patternArray) { + let newVertex; + edges._children.forEach(cChild => { + cChild._children.forEach(cSubChild => { + if(cSubChild && cSubChild["OperatorId:"] === patternArray[0]["OperatorId:"]){ + if(cChild._children.length>1){ + cChild._children = [cChild._children[0]]; + cChild["OutputOperators:"] = patternArray[1]["OutputOperators:"]; + newVertex = Object.assign(patternArray[2], { + "_operator":"Build Bloom Filter", + "_children":[], + "_groups": [ + ...patternArray[0].groups||[doCloneAndOmit(patternArray[0], ['_groups'])], + ...patternArray[1].groups||[doCloneAndOmit(patternArray[1], ['_groups'])], + ...patternArray[2].groups||[doCloneAndOmit(patternArray[2], ['_groups'])], + ...patternArray[3].groups||[doCloneAndOmit(patternArray[3], ['_groups'])]] + }); + } + } else if(cSubChild && cSubChild["OperatorId:"] === patternArray[3]["OperatorId:"]){ + cChild._children = newVertex ? [newVertex]:[]; + } else { + findPatternParent(cChild, patternArray); + } + }); + + }); +} + +function findTheOperatorIndex(vertices, patternArray) { + vertices.forEach(cChild => { + findPatternParent(cChild, patternArray); + }); +} + +function setTheNewVertex(vertices, patternArray) { + findTheOperatorIndex(vertices, patternArray); +} + +function isPatternExists(outputOperators, operatorIds, vertices){ + let patternArray = [outputOperators.cEdge[2], outputOperators.cEdge[3], operatorIds.edgesOp[0], operatorIds.edgesOp[1]]; + setTheNewVertex(vertices, patternArray); +} + +function findPatterns(outputOperatorsList, operatorIdList, pattern, patterns, vertices) { + patterns.forEach(item => { + operatorIdList.forEach(operatorDetails => { + operatorDetails.edgesOp.forEach(op => { + if(op["OperatorId:"] === item.outputOperator) { + pattern.push(op["OperatorId:"]); + isPatternExists(item, operatorDetails, vertices); + } + }); + }); + }); + +} + // DANGER: impure function // DANGER: breaks if there is a many-one / one-many connection export function getAdjustedVerticesAndEdges(vertices, edges) { + let outputOperatorsList = [], operatorIdList = [], pattern = [], edges_opOperator = [], edges_operatorId = [], patterns = []; + findAllOutputOperators(vertices, outputOperatorsList, edges_opOperator, patterns); + findTheOperatorIDChains(vertices, operatorIdList, edges_operatorId); + findPatterns(outputOperatorsList, operatorIdList, pattern, patterns, vertices); + vertices .filter(cVertex => ['Select Operator', 'HASHTABLEDUMMY', 'File Output Operator'].indexOf(getFirstOperatorOf(cVertex)._operator) >= 0) .map(cVertex => edges.filter(cEdge => cEdge._target === cVertex._vertex)) http://git-wip-us.apache.org/repos/asf/ambari/blob/0f5d5507/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js index ac592ec..57fff1e 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js @@ -405,21 +405,6 @@ function getConnections(vertices, edges) { }); }); - // iterate over edges to build connections - edges.forEach(cEdge => { - // get source uuid from source vertex - const sourceVertex = vertices.find(cVertex => cVertex._vertex === cEdge._source); - const sourceOperator = getLastOperatorOf(sourceVertex); - // get target uuid from target vertex - const targetVertex = vertices.find(cVertex => cVertex._vertex === cEdge._target); - const targetOperator = findVertexAsInputInNode(targetVertex, cEdge._source) || getFirstOperatorOf(targetVertex); - // push connection - connections.push({ - _source: sourceOperator, - _target: targetOperator, - }); - }); - // iterate over vertices to find dynamic partitioning event operator // - build connection from dpp to tablescan of target vertex vertices.forEach(cVertex => { @@ -440,6 +425,38 @@ function getConnections(vertices, edges) { }); }); + // iterate over edges to build connections + edges.forEach(cEdge => { + // get source uuid from source vertex + const sourceVertex = vertices.find(cVertex => cVertex._vertex === cEdge._source); + // get target uuid from target vertex + const targetVertex = vertices.find(cVertex => cVertex._vertex === cEdge._target); + + const sourceOperator = getLastOperatorOf(sourceVertex); + const targetOperator = findVertexAsInputInNode(targetVertex, cEdge._source) || getFirstOperatorOf(targetVertex); + + let sourceOperatorList = [], targetOperatorList = [], srcNode = {}, targNode = {}, sourceVertexDef = undefined, targetVertexDef = undefined; + findAllOperatorsInSourceVertex(sourceVertex, sourceOperatorList, srcNode); + findAllOperatorsInTargetVertex(targetVertex, targetOperatorList, targNode); + + if(sourceOperatorList.length && targetOperatorList.length) { + sourceOperatorList.forEach(function(item){ + if(targetOperatorList.indexOf(item)>-1) { + sourceVertexDef = srcNode[item]; + targetVertexDef = targNode[item]; + console.log(sourceVertexDef, targetVertexDef) + } + }); + } + + connections.push({ + _source: sourceVertexDef?sourceVertexDef:sourceOperator, + _target: targetVertexDef?targetVertexDef:targetOperator, + }); + }); + + + return connections; } @@ -476,6 +493,30 @@ function findVertexAsInputInNode(node, vertexId) { return false; } +function findAllOperatorsInTargetVertex(node, resultsAggregator, targetNode) { + let outputOperator = node["OperatorId:"]; + if(outputOperator) { + resultsAggregator.push(outputOperator); + targetNode[outputOperator] = node; + } + node._children.forEach(cChild => findAllOperatorsInTargetVertex(cChild, resultsAggregator, targetNode)); + if(!node._children) { + return resultsAggregator; + } +} + +function findAllOperatorsInSourceVertex(node, resultsAggregator, srcNode) { + let outputOperator = node["OutputOperators:"]; + if(outputOperator) { + resultsAggregator.push(outputOperator.substring(1, outputOperator.length-1)); + srcNode[outputOperator.substring(1, outputOperator.length-1)] = node; + } + node._children.forEach(cChild => findAllOperatorsInSourceVertex(cChild, resultsAggregator, srcNode)); + if(!node._children) { + return resultsAggregator; + } +} + function getLastOperatorOf(vertex) { let operator = vertex._children[0]; while(operator._children.length > 0) {
