AMBARI-20048. Hive View 2.0: Visual Explain-The operators are being shown out of order. (Abhishek Kumar via pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/812397d3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/812397d3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/812397d3 Branch: refs/heads/branch-feature-AMBARI-20053 Commit: 812397d3a054f2be8bb7c7c172bf59255b95a341 Parents: 07342bc Author: pallavkul <[email protected]> Authored: Mon Feb 20 15:56:41 2017 +0530 Committer: pallavkul <[email protected]> Committed: Mon Feb 20 15:56:41 2017 +0530 ---------------------------------------------------------------------- .../ui/app/components/visual-explain.js | 29 +- .../src/main/resources/ui/app/styles/app.scss | 146 ++--- .../resources/ui/app/utils/hive-explainer.js | 645 ------------------- .../ui/app/utils/hive-explainer/enhancer.js | 37 ++ .../ui/app/utils/hive-explainer/fallback.js | 34 + .../ui/app/utils/hive-explainer/index.js | 31 + .../ui/app/utils/hive-explainer/processor.js | 240 +++++++ .../app/utils/hive-explainer/renderer-force.js | 325 ++++++++++ .../ui/app/utils/hive-explainer/renderer.js | 327 ++++++++++ .../ui/app/utils/hive-explainer/transformer.js | 445 +++++++++++++ .../hive20/src/main/resources/ui/bower.json | 3 +- .../src/main/resources/ui/ember-cli-build.js | 1 + 12 files changed, 1519 insertions(+), 744 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js b/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js index 6805bb8..2800c09 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js +++ b/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js @@ -33,31 +33,14 @@ export default Ember.Component.extend({ isQueryRunning:false, - didInsertElement(){ + didInsertElement() { this._super(...arguments); - const width = '100vw', height = '100vh'; - - d3.select('#explain-container').select('svg').remove(); - const svg = d3.select('#explain-container').append('svg') - .attr('width', width) - .attr('height', height); - - const container = svg.append('g'); - - const zoom = - d3.zoom() - .scaleExtent([1 / 10, 4]) - .on('zoom', () => { - container.attr('transform', d3.event.transform); - }); - - svg - .call(zoom); - const onRequestDetail = data => this.set('explainDetailData', JSON.stringify( data, null, ' ') ); - - explain(JSON.parse(this.get('visualExplainInput')), svg, container, zoom, onRequestDetail); + const explainData = JSON.parse(this.get('visualExplainInput')); + // if(explainData) { + explain(explainData, '#explain-container', onRequestDetail); + // } }, @@ -80,7 +63,7 @@ export default Ember.Component.extend({ closeModal(){ this.set('showDetailsModal', false); this.set('explainDetailData', ''); - false; + return false; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss index a9c91c7..3e89ceb 100644 --- a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss +++ b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss @@ -896,81 +896,6 @@ ul.dropdown-menu { } } - -.step { - font-family: Roboto; - background-color:rgb(255, 255, 255); - padding: 8px 10px; - border: 1px solid rgb(223, 223, 223); - cursor: pointer; - &:hover { - background-color: rgb(223, 240, 247); - } - - body { - font-family: Roboto; - background-color: rgba(0,0,0,0); - } -} -.step-sink { - body { - color: rgb(255, 255, 255); - } - background-color: rgb(42, 179, 119); - padding: 20px; -} -.step-sink .step-body { - font-weight: lighter; - margin-top: 10px; -} - -.step-source { - body { - color: rgb(255, 255, 255); - } - background-color: rgb(85, 100, 105); -} - -.step-job, -.step-vectorization { - border: none; - background: none; - padding: 0; -} - -.step-job body, -.step-vectorization body { - height: 100%; - width: 100%; - display: flex; - align-items: center; - justify-content: center; -} - -.step__pill { - font-size: 12px; - border-radius: 25px; - padding: 5px 10px; - min-width: 60px; - color: rgb(255, 255, 255); - background-color: rgb(85, 100, 105); - text-align: center; -} - - -.step-caption { - font-size: 10px; -} -.step-body { - font-size: 12px; -} - -.edge { - stroke: rgb(83, 100, 106); - stroke-width: 2px; - fill: none; -} - #explain-container { height: 100%; width: 100%; @@ -1028,3 +953,74 @@ ul.dropdown-menu { } + +rect.operator__box { + + fill:rgb(255, 255, 255); + stroke:rgb(223, 223, 223); + stroke-width: 1; + + &:hover { + fill: rgb(223, 240, 247); + } + + &.operator__box--TableScan { + fill: rgb(85, 100, 105); + } + + &.operator__box--Fetch_Operator { + fill: rgb(42, 179, 119); + } +} + +.operator body { + cursor: pointer; + + font-family: Roboto; + padding: 8px 10px; + background-color: rgba(0,0,0,0); +} + +.operator--Fetch_Operator body { + color: rgb(255, 255, 255); + padding: 12px; +} + +.operator--Fetch_Operator .operator-body { + font-weight: lighter; +} + +.operator--TableScan body { + color: rgb(255, 255, 255); +} + +.operator-caption { + font-size: 10px; +} + +.operator-body { + font-size: 12px; +} +.edge--hidden { + display: none; +} +.edge { + stroke: rgb(83, 100, 106); + stroke-width: 2px; + fill: none; +} + +#explain-container { + height: 100%; + width: 100%; + overflow: auto; +} + +.explain--error { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + min-height: 100px; + font-size: 16px; +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js deleted file mode 100644 index 2b59340..0000000 --- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js +++ /dev/null @@ -1,645 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -export default function render(data, svg, container, zoom, onRequestDetail){ - - const steps = createOrder(data).steps; - const plans = data['STAGE PLANS']; - const stageKey = - Object - .keys(plans) - .find(cStageKey => plans[cStageKey].hasOwnProperty('Fetch Operator')); - let rows = 'Unknown'; - if(stageKey && plans[stageKey]['Fetch Operator']['limit:']) { - rows = plans[stageKey]['Fetch Operator']['limit:']; - } - const root = [{ - "type": "sink", - "sink-type": "table", - "sink-label": "Limit", - "rows": rows, - "children": [{ - steps: steps - }] - }]; - const transformed = getTransformed(root); - update(transformed, svg, container, zoom, onRequestDetail); -} - -const RENDER_GROUP = { - join: d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type, d['join-type'])}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>${d['join-type'] === 'merge' ? 'Merge' : 'Map'} Join</div> - <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div> - </div> - </div> - `, - vectorization: d => '<div class="step__pill">U</div>', - job: d => `<div class="step__pill">${d.label.toUpperCase()}</div>`, - broadcast: d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>Broadcast</div> - <!--div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div--> - </div> - </div> - `, - 'partition-sort': d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>Partition / Sort</div> - <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div> - </div> - </div> - `, - sink: d => ` -// TODO - `, - 'group-by': d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>Group By</div> - <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div> - </div> - </div> - `, - select: d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>Select</div> - <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div> - </div> - </div> - `, - source: d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getIcon(d.type, d['source-type'])}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>${d.label}</div> - <div><span style='font-weight: lighter;'>${d.isPartitioned ? 'Partitioned' : 'Unpartitioned'} | Rows:</span> ${abbreviate(d.rows)}</div> - </div> - </div> - ` -}; - -function update(data, svg, container, zoom, onRequestDetail) { - const steps = container.selectAll('g.step') - .data(data) - .enter().append('g') - .attr('class', 'step'); - steps - .append('foreignObject') - .attr('id', d => d.uuid) - .attr('class', 'step step-sink') - .attr('height', 300) - .attr('width', 220) - .append('xhtml:body') - .style('margin', 0) - .html(d => ` - <div> - <div class='step-meta' style='display:flex;'> - <i class='fa ${getIcon(d.type, d['sink-type'])}' aria-hidden='true'></i> - <div class='step-header' style='margin-left: 10px;'> - <div class='step-title'>${d['sink-label']}</div> - <div class='step-caption'>${abbreviate(d.rows)} ${d.row === 1 ? 'row' : 'rows'}</div> - </div> - </div> - <div class='step-body'>${d['sink-description'] || ''}</div> - </div> - `) - .on('click', d => onRequestDetail(d)); - steps - .call(recurse); - const edges = - container.selectAll('p.edge') - .data(getEdges(data)) - .enter().insert('path', ':first-child') - .attr('class', 'edge') - .attr('d', d => getConnectionPath(d, svg, container)); - reset(zoom, svg, container); - - - function recurse(step) { - const children = - step - .selectAll('g.child') - .data(d => d.children || []).enter() - .append('g') - .attr('class', 'child') - .style('transform', (d, index) => `translateY(${index * 100}px)`); - children.each(function(d) { - const child = d3.select(this); - const steps = - child.selectAll('g.step') - .data(d => d.steps || []).enter() - .append('g') - .attr('class', 'step') - .style('transform', (d, index) => `translateX(${250 + index * 150}px)`); - steps - .append('foreignObject') - .attr('id', d => d.uuid) - .attr('class', d => `step step-${d.type}`) - .classed('step-source', d => d.operator === 'TableScan') - .attr('height', 55) - .attr('width', d => d.type === 'source' ? 200 : 140) - .append('xhtml:body') - .style('margin', 0) - .html(d => getRenderer(d.type)(d)) - .on('click', d => onRequestDetail(d)); - steps.filter(d => Array.isArray(d.children)) - .call(recurse); - }); - } -} - -function getRenderer(type) { - const renderer = RENDER_GROUP[type]; - if(renderer) { - return renderer; - } - - if(type === 'stage') { - return (d => ` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>Stage</div> - <!--div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div--> - </div> - </div> - `); - } - - return (d => { - const isSource = d.operator === 'TableScan'; - return (` - <div style='display:flex;'> - <div class='step-meta'> - <i class='fa ${getOperatorIcon(d.operator)}' aria-hidden='true'></i> - </div> - <div class='step-body' style='margin-left: 10px;'> - <div>${isSource ? d['alias:'] : getOperatorLabel(d.operator)}</div> - <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div> - </div> - </div> - `); - }); -} -function getNumberOfRows(statistics) { - const match = statistics.match(/([^\?]*)\Num rows: (\d*)/); - return (match.length === 3 && Number.isNaN(Number(match[2])) === false) ? match[2] : 0; -} -function getOperatorLabel(operator) { - const operatorStr = operator.toString(); - if(operatorStr.endsWith(' Operator')) { - return operatorStr.substring(0, operatorStr.length - ' Operator'.length); - } - if(operatorStr === 'TableScan') { - return 'Scan'; - } - return operatorStr ? operatorStr : 'Unknown'; -} -function getOperatorIcon(operator) { - switch(operator) { - case 'File Output Operator': - return 'fa-file-o'; - case 'Reduce Output Operator': - return 'fa-compress'; - case 'Filter Operator': - return 'fa-filter'; - case 'Dynamic Partitioning Event Operator': - return 'fa-columns' - case 'Map Join Operator': - return 'fa-code-fork' - case 'Limit': - case 'Group By Operator': - case 'Select Operator': - case 'TableScan': - return 'fa-table'; - default: - return ''; - } -} -function getIcon (type, subtype) { - switch(type) { - case 'join': - return 'fa-code-fork' - case 'vectorization': - case 'job': - return; - case 'broadcast': - case 'partition-sort': - return 'fa-compress'; - case 'source': - case 'sink': - case 'group-by': - case 'select': - return 'fa-table'; - } -}; -function abbreviate(value) { - let newValue = value; - if (value >= 1000) { - const suffixes = ["", "k", "m", "b","t"]; - const suffixNum = Math.floor(("" + value).length / 3); - let shortValue = ''; - for (var precision = 2; precision >= 1; precision--) { - shortValue = parseFloat( (suffixNum != 0 ? (value / Math.pow(1000,suffixNum) ) : value).toPrecision(precision)); - const dotLessShortValue = (shortValue + '').replace(/[^a-zA-Z 0-9]+/g,''); - if (dotLessShortValue.length <= 2) { break; } - } - if (shortValue % 1 != 0) { - const shortNum = shortValue.toFixed(1); - } - newValue = shortValue+suffixes[suffixNum]; - } - return newValue; -} -function reset(zoom, svg, container) { - const steps = container.selectAll('g.step'); - const bounds = []; - steps.each(function(d) { - const cStep = d3.select(this); - const box = cStep.node().getBoundingClientRect(); - bounds.push(box); - }); - const PADDING_PERCENT = 0.95; - const fullWidth = svg.node().clientWidth; - const fullHeight = svg.node().clientHeight; - const offsetY = svg.node().getBoundingClientRect().top; - const top = Math.min(...bounds.map(cBound => cBound.top)); - const left = Math.min(...bounds.map(cBound => cBound.left)); - const width = Math.max(...bounds.map(cBound => cBound.right)) - left; - const height = Math.max(...bounds.map(cBound => cBound.bottom)) - top; - const midX = left + width / 2; - const midY = top + height / 2; - if (width == 0 || height == 0) return; // nothing to fit - const scale = PADDING_PERCENT / Math.max(width / fullWidth, height / fullHeight); - const translate = [fullWidth / 2 - scale * midX, fullHeight / 2 - scale * midY]; - const zoomIdentity = - d3.zoomIdentity - .translate(translate[0], translate[1] + offsetY) - .scale(scale); - svg - .transition() - // .delay(750) - .duration(750) - // .call( zoom.transform, d3.zoomIdentity.translate(0, 0).scale(1) ); // not in d3 v4 - .call(zoom.transform, zoomIdentity); -} -function getConnectionPath(edge, svg, container) { - const steps = container.selectAll('foreignObject.step'); - const source = container.select(`#${edge.source}`); - const target = container.select(`#${edge.target}`); - const rSource = source.node().getBoundingClientRect(); - const rTarget = target.node().getBoundingClientRect(); - const pSource = { - x: (rSource.left + rSource.right) / 2, - y: (rSource.top + rSource.bottom) / 2, - }; - const pTarget = { - x: (rTarget.left + rTarget.right) / 2, - y: (rTarget.top + rTarget.bottom) / 2, - }; - const path = [ - pSource - ]; - if(pSource.y !== pTarget.y) { - path.push({ - x: (pSource.x + pTarget.x) / 2, - y: pSource.y - }, { - x: (pSource.x + pTarget.x) / 2, - y: pTarget.y - }) - } - path.push(pTarget); - const offsetY = svg.node().getBoundingClientRect().top; - return path.reduce((accumulator, cPoint, index) => { - if(index === 0) { - return accumulator + `M ${cPoint.x}, ${cPoint.y - offsetY}\n` - } else { - return accumulator + `L ${cPoint.x}, ${cPoint.y - offsetY}\n` - } - }, ''); -} -function uuid() { - return 'step-xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => { - const r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8); - return v.toString(16); - }); -} -function getEdges(steps) { - const edges = []; - for (let prev, index = 0; index < steps.length; index++) { - const cStep = steps[index]; - if(prev) { - edges.push({ - source: prev.uuid, - target: cStep.uuid - }); - } - prev = cStep; - if(Array.isArray(cStep.children)) { - cStep.children.forEach(cChild => { - if(cChild.steps.length === 0) { - return; - } - edges.push({ - source: cStep.uuid, - target: cChild.steps[0].uuid - }); - edges.push(...getEdges(cChild.steps)); - }); - } - } - return edges; -} -function getTransformed(steps) { - return steps.map(cStep => { - let cResStep = cStep; - cResStep = Object.assign({}, cResStep, { - uuid: uuid() - }); - if(Array.isArray(cResStep.children)) { - const children = cResStep.children.map(cChild => Object.assign({}, cChild, { - steps: getTransformed(cChild.steps) - })); - cResStep = Object.assign({}, cResStep, { - children: children - }); - } - return cResStep; - }); -} -function createOrder(data) { - const stageDeps = data['STAGE DEPENDENCIES']; - const stagePlans = data['STAGE PLANS']; - const stageRootKey = Object.keys(stageDeps).find(cStageKey => stageDeps[cStageKey]['ROOT STAGE'] === 'TRUE'); - const root = Object.assign({}, getStageData(stageRootKey, stagePlans), { - _stages: getDependentStageTreeInOrder(stageRootKey, stageDeps, stagePlans) - }); - const expanded = doExpandChild(root); - return doClean(expanded); -} -function getDependentStageTreeInOrder(sourceStageKey, stageDeps, stagePlans) { - const stageKeys = - Object - .keys(stageDeps) - .filter(cStageKey => stageDeps[cStageKey] && stageDeps[cStageKey]['DEPENDENT STAGES'] === sourceStageKey); - const stages = - stageKeys.map(cStageKey => Object.assign({}, getStageData(cStageKey, stagePlans), { - _stages: getDependentStageTreeInOrder(cStageKey, stageDeps, stagePlans) - })); - return stages; -} -function getStageData(stageKey, stagePlans) { - const plan = stagePlans[stageKey]; - const engineKeys = Object.keys(plan); - if(engineKeys.length !== 1) { - return plan; - } - const engineKey = engineKeys[0]; - // returns a job - let step; - switch(engineKey) { - case 'Map Reduce': - step = buildForMR(plan[engineKey]); - break; - case 'Map Reduce Local Work': - step = buildForMRLocal(plan[engineKey]); - break; - case 'Tez': - step = buildForTez(plan[engineKey]); - break; - case 'Fetch Operator': - step = buildForFetch(plan[engineKey]); - break; - default: - step = { - type: 'placeholder', - _engine: 'not_found', - _plan: plan - }; - } - return ({ - steps: [ - step - ] - }); -} -function buildForMR(plan) { - return ({ - type: 'stage', - _engine: 'mr', - _plan: plan - }); -} -function buildForMRLocal(plan) { - return ({ - type: 'stage', - _engine: 'mr-local', - _plan: plan - }); -} -function buildForTez(plan) { - const edges = plan['Edges:']; - const vertices = plan['Vertices:']; - const fEdges = - Object - .keys(edges) - .reduce((accumulator, cTargetKey) => { - if(Array.isArray(edges[cTargetKey])) { - const edgesFromSourceKey = edges[cTargetKey]; - accumulator.push(...edgesFromSourceKey.map(cEdgeFromSourceKey => ({ - source: cEdgeFromSourceKey['parent'], - target: cTargetKey, - type: cEdgeFromSourceKey['type'] - }))); - } else { - const edgeFromSourceKey = edges[cTargetKey]; - accumulator.push({ - source: edgeFromSourceKey['parent'], - target: cTargetKey, - type: edgeFromSourceKey['type'] - }); - } - return accumulator; - }, []); - const rootKey = fEdges.find(cEdge => fEdges.some(iEdge => iEdge.source === cEdge.target) === false).target; - return Object.assign({}, doTezBuildTreeFromEdges(rootKey, fEdges, vertices), { - _engine: 'tez', - _plan: plan - }); -} -function buildForFetch(plan) { - return ({ - type: 'stage', - _engine: 'fetch', - _plan: plan - }); -} -function doTezBuildTreeFromEdges(parentKey, edges, vertices) { - const jobs = - Object - .keys(vertices) - .map(cVertexKey => ({ - type: 'job', - label: cVertexKey, - _data: vertices[cVertexKey], - })) - .reduce((accumulator, cVertex) => Object.assign(accumulator, { - [cVertex.label]: cVertex - }), {}); - edges.forEach(cEdge => { - const job = jobs[cEdge.target]; - if(!Array.isArray(job.children)) { - job.children = []; - } - const steps = []; - if(cEdge.type === 'BROADCAST_EDGE') { - steps.push({ - type: 'broadcast', - _data: jobs[cEdge.target], - }); - } - steps.push(jobs[cEdge.source]); - job.children.push({ - steps - }); - }); - return jobs[parentKey]; -} -function doExpandChild(node) { - return Object.assign({}, node, { - steps: node.steps.reduce((accumulator, cStep) => [...accumulator, ...doExpandStep(cStep, 'step')], []) - }); -} -function doExpandStep(node) { - switch(node.type) { - case 'job': - const key = Object.keys(doOmit(node._data, ['Execution mode:']))[0]; - let root = node._data[key]; - if(!Array.isArray(root)) { - root = [root]; - } - const steps = doGetOperators(root); - const children = Array.isArray(node.children) ? node.children.map(cChild => doExpandChild(cChild)) : []; - return ([ - doOmit(node, ['children']), - ...steps.reverse().slice(0, steps.length - 1), - Object.assign({}, steps[steps.length - 1], { - children - }) - ]); - default: - return [node]; - } -} -function doClean(node) { - let cleaned = - Object - .keys(node) - .filter(cNodeKey => cNodeKey.startsWith('_') === false) - .reduce((accumulator, cNodeKey) => Object.assign(accumulator, { - [cNodeKey]: node[cNodeKey] - }), {}); - if(cleaned.hasOwnProperty('children')) { - cleaned = Object.assign({}, cleaned, { - children: cleaned.children.map(cChild => doClean(cChild)) - }) - } - if(cleaned.hasOwnProperty('steps')) { - cleaned = Object.assign({}, cleaned, { - steps: cleaned.steps.map(cStep => doClean(cStep)) - }) - } - return cleaned; -} -function doGetOperators(node) { - let stepx = node; - if(!Array.isArray(stepx)) { - stepx = [stepx]; - } - const steps = - stepx - .reduce((accumulator, cStep) => { - const key = Object.keys(cStep)[0]; - const obj = cStep[key]; - let children = []; - if(obj.children) { - children = doGetOperators(obj.children); - } - const filtered = - Object - .keys(obj) - .filter(cKey => cKey !== 'children') - .reduce((accumulator, cKey) => { - accumulator[cKey] = obj[cKey]; - return accumulator; - }, {}); - return [ - ...accumulator, - Object.assign({ - _data: cStep, - operator: key - }, filtered), - ...children - ]; - }, []); - return steps; -} -function doGetStep(node) { - const key = Object.keys(node)[0]; - const obj = node[key]; - return { - operator: key, - _data: obj - }; -} -function doOmit(object, keys) { - return Object - .keys(object) - .filter(cKey => keys.indexOf(cKey) === -1) - .reduce((accumulator, cKey) => { - accumulator[cKey] = object[cKey]; - return accumulator; - }, {}); -} - http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/enhancer.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/enhancer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/enhancer.js new file mode 100644 index 0000000..a99b82e --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/enhancer.js @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default function doEnhance(vertices) { + return vertices.map(cVertex => Object.assign({}, cVertex, { + _children: cVertex._children.map(cChild => doEnhanceNode(cChild)) + })); +} + +function doEnhanceNode(node) { + return Object.assign({}, node, { + _uuid: uuid(), + _children: node._children.map(cChild => doEnhanceNode(cChild)) + }); +} + +function uuid() { + return 'operator-xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => { + const r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8); + return v.toString(16); + }); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/fallback.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/fallback.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/fallback.js new file mode 100644 index 0000000..2310b3c --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/fallback.js @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isExplainable(data) { + const stages = data['STAGE PLANS']; + const isValidTezStageAvailable = Object.keys(stages).some(cStageKey => stages[cStageKey].hasOwnProperty('Tez')); + const isValidFetchStageAvailable = Object.keys(stages).find(cStageKey => stages[cStageKey].hasOwnProperty('Fetch Operator')); + return isValidTezStageAvailable && isValidFetchStageAvailable; +} + +export function doRenderError(selector) { + d3.select(selector).select('*').remove(); + + d3.select(selector) + .append('div') + .attr('class', 'explain--error') + .append('div') + .text('No valid Tez plan found.'); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/index.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/index.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/index.js new file mode 100644 index 0000000..3513a23 --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/index.js @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import doTransform from './transformer'; +import doRender from './renderer'; +import {isExplainable, doRenderError} from './fallback'; + + +export default function draw(data, selector, onRequestDetail){ + if(isExplainable(data)) { + const transformed = doTransform(data); + doRender(transformed, selector, onRequestDetail); + } else { + doRenderError(selector); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js new file mode 100644 index 0000000..5dbeb2b --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function getProcessedVertices(vertices, edges) { + const edgedVertices = processEdges(vertices, edges); + return processSource(edgedVertices); +} + +function processEdges(vertices, edges) { + + return vertices + .map(cVertex => { + const isVertexPartOfSimpleEdge = edges.some(cEdge => cEdge.type === 'SIMPLE_EDGE' && cEdge.parent === cVertex._vertex); + const isVertexPartOfBroadcastEdge = edges.some(cEdge => cEdge.type === 'BROADCAST_EDGE' && cEdge.parent === cVertex._vertex); + const isVertexPartOfCustomSimpleEdge = edges.some(cEdge => cEdge.type === 'CUSTOM_SIMPLE_EDGE' && cEdge.parent === cVertex._vertex); + const isVertexPartOfCustomEdge = edges.some(cEdge => cEdge.type === 'CUSTOM_EDGE' && cEdge.parent === cVertex._vertex); + const isVertexPartOfXProdEdge = edges.some(cEdge => cEdge.type === 'XPROD_EDGE' && cEdge.parent === cVertex._vertex); + + let tVertex = cVertex; + + if(isVertexPartOfSimpleEdge) { + tVertex = appendIfTerminusOfOperator(tVertex, { + _operator: 'Partition/Sort Pseudo-Edge' + }); + } + if(isVertexPartOfBroadcastEdge) { + tVertex = appendIfTerminusOfOperator(tVertex, { + _operator: 'Broadcast Pseudo-Edge' + }); + } + if(isVertexPartOfCustomSimpleEdge) { + tVertex = appendIfTerminusOfOperator(tVertex, { + _operator: 'Partition Pseudo-Edge' + }); + } + if(isVertexPartOfCustomEdge) { + tVertex = appendIfTerminusOfOperator(tVertex, { + _operator: 'Co-partition Pseudo-Edge' + }); + } + if(isVertexPartOfXProdEdge) { + tVertex = appendIfTerminusOfOperator(tVertex, { + _operator: 'Cross-product Distribute Pseudo-Edge' + }); + } + + return tVertex; + }); +} + +function appendIfTerminusOfOperator(node, pseudoNode) { + if(Array.isArray(node._children) === false || node._children.length === 0) { + // is terminus + switch(node._operator) { + case 'Reduce Output Operator': + return Object.assign({}, node, pseudoNode); + default: + return node; + } + } + + return Object.assign({}, node, { + _children: node._children.map(cChild => appendIfTerminusOfOperator(cChild, pseudoNode)) + }); +} + +function processSource(vertices) { + return vertices.map(cVertex => Object.assign({}, cVertex, { + _children: cVertex._children.map(cChild => getProcessedSequenceViaStack(cChild)) + })); +} + +// DANGER: impure function +function getProcessedSequenceViaStack(root) { + const stack = []; + + let cNode = root; + stack.push(cNode); + doCompaction(stack); + while(cNode._children.length === 1) { + cNode = cNode._children[0]; + + stack.push(cNode); + doCompaction(stack); + } + + const lNode = stack[stack.length - 1]; + if(lNode._children.length > 1) { + // begin processing new subtree + lNode._children = lNode._children.map(cChild => getProcessedSequenceViaStack(cChild)); + } + + return stack[0]; +} + +function doCompaction(stack) { + let index = stack.length; + + while(index > 0) { + const cNode = stack[index - 0 - 1]; + const cNodeMinus1 = stack[index - 1 - 1]; + const cNodeMinus2 = stack[index - 2 - 1]; + const cNodeMinus3 = stack[index - 3 - 1]; + const cNodeMinus4 = stack[index - 4 - 1]; + + if(cNodeMinus1) { + + if(cNode._operator === 'Select Operator' || cNode._operator === 'HASHTABLEDUMMY' || cNode._operator === 'File Output Operator') { + // remove cNode from stack + stack.pop(); + index--; + // recreate groups + cNodeMinus1._groups = [ + ...(cNodeMinus1._groups || [doCloneAndOmit(cNodeMinus1, ['_groups'])]), + ...(cNode._groups || [doCloneAndOmit(cNode, ['_groups'])]), + ]; + // move children + cNodeMinus1._children = cNode._children; + + continue; + } + if(cNodeMinus1._operator === 'Select Operator' || cNodeMinus1._operator === 'HASHTABLEDUMMY' || cNodeMinus1._operator === 'File Output Operator') { + // remove cNode and cNodeMinus1 from stack + stack.pop(); + index--; + stack.pop(); + index--; + + // recreate groups + cNode._groups = [ + ...(cNodeMinus1._groups || [doCloneAndOmit(cNodeMinus1, ['_groups'])]), + ...(cNode._groups || [doCloneAndOmit(cNode, ['_groups'])]), + ]; + // no need to move chldren + // reinsert cNode + stack.push(cNode); + index++; + + continue; + } + + + if(cNode._operator === 'Map Join Operator' && cNodeMinus1._operator === 'Map Join Operator') { + // remove cNode from stack + stack.pop(); + index--; + // recreate groups + cNodeMinus1._groups = [ + ...(cNodeMinus1._groups || [doCloneAndOmit(cNodeMinus1, ['_groups'])]), + ...(cNode._groups || [doCloneAndOmit(cNode, ['_groups'])]), + ]; + // move chldren + cNodeMinus1._children = cNode._children; + + continue; + } + + if(cNode._operator === 'Filter Operator' && cNodeMinus1._operator === 'TableScan') { + // remove cNode from stack + stack.pop(); + index--; + // recreate groups + cNodeMinus1._groups = [ + ...(cNodeMinus1._groups || [doCloneAndOmit(cNodeMinus1, ['_groups'])]), + ...(cNode._groups || [doCloneAndOmit(cNode, ['_groups'])]), + ]; + // move children + cNodeMinus1._children = cNode._children; + + continue; + } + + if(cNodeMinus2 && cNodeMinus3) { + if(cNode._operator === 'Broadcast Pseudo-Edge' && cNodeMinus1._operator === 'Group By Operator' && cNodeMinus2._operator === 'Reduce Output Operator' && cNodeMinus3._operator === 'Group By Operator') { + // remove cNode from stack + stack.pop(); + index--; + // remove cNodeMinus1 from stack + stack.pop(); + index--; + // remove cNodeMinus2 from stack + stack.pop(); + index--; + // remove cNodeMinus3 from stack + stack.pop(); + index--; + + // recreate groups + cNodeMinus1._groups = [ + ...(cNodeMinus3._groups || [doCloneAndOmit(cNodeMinus3, ['_groups'])]), + ...(cNodeMinus2._groups || [doCloneAndOmit(cNodeMinus2, ['_groups'])]), + ...(cNodeMinus1._groups || [doCloneAndOmit(cNodeMinus1, ['_groups'])]), + ]; + // move children if required, cNodeMinus1 as child of cNodeMinus4 + if(cNodeMinus4) { + cNodeMinus4._children = cNodeMinus2._children; + } + // rename + cNodeMinus1._operator = 'Build Bloom Filter'; + // add renamed node + stack.push(cNodeMinus1); + index++; + // add original broadcast edge node + stack.push(cNode); + index++; + + + continue; + } + } + + } + index--; + + } +} + +function doCloneAndOmit(obj, keys) { + return Object + .keys(obj) + .filter(cObjKey => keys.indexOf(cObjKey) === -1) + .reduce((tObj, cObjKey) => Object.assign({}, tObj, { + [cObjKey]: obj[cObjKey] + }), {}); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer-force.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer-force.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer-force.js new file mode 100644 index 0000000..2dfdc86 --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer-force.js @@ -0,0 +1,325 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default function doRender(data, selector, onRequestDetail) { + + const {connections} = data; + const dataNodes = data.nodes; + const dataLinks = connections.map(cConnection => ({ + source: cConnection._target, + target: cConnection._source + })); + + const width = '960', height = '800'; + + d3.select(selector).select('*').remove(); + const svg = + d3.select(selector) + .append('svg') + .attr('width', width) + .attr('height', height); + + const container = svg.append('g'); + const zoom = + d3.behavior.zoom() + .scaleExtent([1 / 10, 4]) + .on('zoom', () => { + container.attr('transform', `translate(${d3.event.translate}) scale(${d3.event.scale})`); + }); + + svg + .call(zoom); + + var force = cola.d3adaptor(d3) + .avoidOverlaps(true) + .flowLayout('x', 150) + .convergenceThreshold(1e-3) + .size([width, height]) + .jaccardLinkLengths(150); + // .linkDistance(200) + + var links = container.selectAll('path.edge') + .data(dataLinks) + .enter().append('path') + .attr('class', 'edge'); + + var nodes = container.selectAll('g.operator-wrapper') + .data(dataNodes) + .enter().append('g') + .attr('class', 'operator-wrapper'); + // .call(force.drag); + + nodes + .append('rect') + .attr('id', d => d._uuid) + .attr('data-operator', d => d._operator) + .attr('class', d => `operator__box operator__box--${d._operator.toString().replace(/[ ]/g, '_')}`) + .attr('height', d => d._operator === 'Fetch Operator' ? 150 : 55) + .attr('width', 140) + + nodes + .append('foreignObject') + .attr('data-uuid', d => d._uuid) + .attr('data-operator', d => d._operator) + .attr('class', d => `operator operator--${d._operator.toString().replace(/[ ]/g, '_')}`) + .attr('height', d => d._operator === 'Fetch Operator' ? 150 : 55) + .attr('width', 140) + .append('xhtml:body') + .style('margin', 0) + .html(d => getRenderer(d._operator)(d)) + .on('click', d => onRequestDetail(doClean(d))); + + force + .nodes(dataNodes) + .links(dataLinks) + // .constraints([ + // { + // type: 'alignment', + // axis: 'y', + // offsets: dataNodes.map((cNode, index) => ({ + // node: index, + // offset: cNode._offsetY + // })) + // } + // ]) + .on('tick', e => { + // node.each(function (d) { d.innerBounds = d.bounds.inflate(-margin); }) + // .attr("x", function (d) { return d.innerBounds.x; }) + // .attr("y", function (d) { return d.innerBounds.y; }) + // .attr("width", function (d) { + // return d.innerBounds.width(); + // }) + // .attr("height", function (d) { return d.innerBounds.height(); }); + + // link.attr("d", function (d) { + // var route = cola.makeEdgeBetween(d.source.innerBounds, d.target.innerBounds, 5); + // return lineFunction([route.sourceIntersection, route.arrowStart]); + // }); + + // const k = 6 * e.alpha; + + // Push sources up and targets down to form a weak tree. + links + // .each(function(d) { d.source.y -= k, d.target.y += k; }) + .attr('d', d => getConnectionPath({ + x: (d.source.bounds.X + d.source.bounds.x) / 2, + y: (d.source.bounds.Y + d.source.bounds.y) / 2, + }, { + x: (d.target.bounds.X + d.target.bounds.x) / 2, + y: (d.target.bounds.Y + d.target.bounds.y) / 2, + })); + + nodes + .attr("transform", d => `translate(${d.x}, ${d.y})`); + + }) + .on('end', () => { + reset(zoom, svg, container); + }) + .start(); +} + +function getRenderer(type) { + if(type === 'Fetch Operator') { + return (d => { + return (` + <div style='display:flex;align-items: center;'> + <div class='operator-meta'> + <i class='fa ${getOperatorIcon(d._operator)}' aria-hidden='true'></i> + </div> + <div class='operator-body' style='margin-left: 10px;'> + <div>${getOperatorLabel(d)}</div> + ${d['limit:'] ? '<div><span style="font-weight: lighter;">Limit:</span> ' + d['limit:'] + ' </div>' : ''} + </div> + </div> + `); + }); + } + + return (d => { + const stats = d['Statistics:'] ? `<div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div>` : ''; + return (` + <div style='display:flex;'> + <div class='operator-meta'> + <i class='fa ${getOperatorIcon(d._operator)}' aria-hidden='true'></i> + </div> + <div class='operator-body' style='margin-left: 10px;'> + <div>${getOperatorLabel(d)}</div> + ${stats} + </div> + </div> + `); + }); + +} + +function getNumberOfRows(statistics) { + const match = statistics.match(/([^\?]*)\Num rows: (\d*)/); + return (match.length === 3 && Number.isNaN(Number(match[2])) === false) ? match[2] : 0; +} +function getOperatorLabel(d) { + const operator = d._operator; + + if(operator === 'TableScan') { + return d['alias:']; + } + + const operatorStr = operator.toString(); + if(operatorStr.endsWith(' Operator')) { + return operatorStr.substring(0, operatorStr.length - ' Operator'.length); + } + if(operatorStr.endsWith(' Pseudo-Edge')) { + return operatorStr.substring(0, operatorStr.length - ' Pseudo-Edge'.length); + } + return operatorStr ? operatorStr : 'Unknown'; +} +function getOperatorIcon(operator) { + switch(operator) { + case 'File Output Operator': + return 'fa-file-o'; + case 'Partition/Sort Pseudo-Edge': + case 'Broadcast Pseudo-Edge': + case 'Partition Pseudo-Edge': + case 'Co-partition Pseudo-Edge': + case 'Cross-product Distribute Pseudo-Edge': + case 'Reduce Output Operator': + return 'fa-compress'; + case 'Filter Operator': + return 'fa-filter'; + case 'Dynamic Partitioning Event Operator': + return 'fa-columns' + case 'Map Join Operator': + return 'fa-code-fork' + case 'Limit': + case 'Group By Operator': + case 'Select Operator': + case 'TableScan': + case 'Fetch Operator': + return 'fa-table'; + default: + return ''; + } +} +function getIcon (type, subtype) { + switch(type) { + case 'join': + return 'fa-code-fork' + case 'vectorization': + case 'job': + return; + case 'broadcast': + case 'partition-sort': + return 'fa-compress'; + case 'source': + case 'sink': + case 'group-by': + case 'select': + return 'fa-table'; + } +}; +function abbreviate(value) { + let newValue = value; + if (value >= 1000) { + const suffixes = ["", "k", "m", "b","t"]; + const suffixNum = Math.floor(("" + value).length / 3); + let shortValue = ''; + for (var precision = 2; precision >= 1; precision--) { + shortValue = parseFloat( (suffixNum != 0 ? (value / Math.pow(1000,suffixNum) ) : value).toPrecision(precision)); + const dotLessShortValue = (shortValue + '').replace(/[^a-zA-Z 0-9]+/g,''); + if (dotLessShortValue.length <= 2) { break; } + } + if (shortValue % 1 != 0) { + const shortNum = shortValue.toFixed(1); + } + newValue = shortValue+suffixes[suffixNum]; + } + return newValue; +} +function reset(zoom, svg, container) { + const vertices = container.selectAll('g.operator-wrapper'); + const bounds = []; + vertices.each(function(d) { + const cVertex = d3.select(this); + const box = cVertex.node().getBoundingClientRect(); + bounds.push(box); + }); + const PADDING_PERCENT = 0.95; + const svgRect = svg.node().getBoundingClientRect(); + const fullWidth = svgRect.width; + const fullHeight = svgRect.height; + const offsetY = svgRect.top; + const top = Math.min(...bounds.map(cBound => cBound.top)); + const left = Math.min(...bounds.map(cBound => cBound.left)); + const width = Math.max(...bounds.map(cBound => cBound.right)) - left; + const height = Math.max(...bounds.map(cBound => cBound.bottom)) - top; + const midX = left + width / 2; + const midY = top + height / 2; + if (width == 0 || height == 0){ + // nothing to fit + return; + } + const scale = PADDING_PERCENT / Math.max(width / fullWidth, height / fullHeight); + const translate = [fullWidth / 2 - scale * midX, fullHeight / 2 - scale * midY]; + + zoom.scale(scale).translate(translate); + + svg + .transition() + .delay(750) + .call(zoom.event); +} + +function getConnectionPath(pSource, pTarget) { + const path = [ + pTarget + ]; + const junctionXMultiplier = (pTarget.x - pSource.x < 0) ? +1 : -1; + if(pSource.y !== pTarget.y) { + path.push({ + x: pTarget.x + junctionXMultiplier * 90, + y: pTarget.y + }, { + x: pTarget.x + junctionXMultiplier * 90, + y: pSource.y + }); + } + path.push(pSource); + const offsetY = 0; + return path.reduce((accumulator, cPoint, index) => { + if(index === 0) { + return accumulator + `M ${cPoint.x}, ${cPoint.y - offsetY}\n` + } else { + return accumulator + `L ${cPoint.x}, ${cPoint.y - offsetY}\n` + } + }, ''); +} + +function doClean(node) { + if(Array.isArray(node._groups)) { + return node._groups.map(cGroup => doClean(cGroup)); + } else { + return ( + Object.keys(node) + .filter(cNodeKey => cNodeKey !== '_children') + .reduce((accumulator, cNodeKey) => { + accumulator[cNodeKey] = node[cNodeKey]; + return accumulator; + }, {}) + ); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js new file mode 100644 index 0000000..3dedd8f --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js @@ -0,0 +1,327 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default function doRender(data, selector, onRequestDetail) { + + const width = '100vw', height = '100vh'; + + d3.select(selector).select('*').remove(); + const svg = + d3.select(selector) + .append('svg') + .attr('width', width) + .attr('height', height); + + const container = svg.append('g'); + const zoom = + d3.behavior.zoom() + .scaleExtent([1 / 10, 4]) + .on('zoom', () => { + container.attr('transform', `translate(${d3.event.translate}) scale(${d3.event.scale})`); + }); + + svg + .call(zoom); + + const root = + container + .selectAll('g.vertex') + .data([data.tree]) + .enter() + .append('g') + .attr('class', 'vertex') + .attr('data-vertex', d => d._vertex); + + root + .call(recurseC, onRequestDetail); + + root + .call(recurseV, onRequestDetail); + + container.selectAll('path.edge') + .data(data.connections) + .enter() + .insert('path', ':first-child') + .attr('class', 'edge') + .attr('d', d => getConnectionPath(d, svg, container)); + + reset(zoom, svg, container); + +} + +function recurseV(vertices, onRequestDetail) { + vertices.each(function(cVertx) { + const vertex = d3.select(this); + + const vertices = + vertex + .selectAll('g.vertex') + .data(d => d._vertices) + .enter() + .append('g') + .attr('class', 'vertex') + .attr('data-vertex', d => d._vertex) + .style('transform', d => `translate(${d._widthOfSelf * 200}px, ${d._offsetY * 100}px)`); + + vertices + .call(recurseC, onRequestDetail); + + vertices + .call(recurseV, onRequestDetail); + }); +} + +function recurseC(children, onRequestDetail) { + children.each(function(d) { + const child = d3.select(this); + + const children = + child + .selectAll('g.child') + .data(d => d._children || []).enter() + .append('g') + .attr('class', 'child') + .style('transform', (d, index) => `translate(-${200}px, ${index * 100}px)`); + + children + .append('rect') + .attr('id', d => d._uuid) + .attr('data-operator', d => d._operator) + .attr('class', d => `operator__box operator__box--${d._operator.toString().replace(/[ ]/g, '_')}`) + .attr('height', d => d._operator === 'Fetch Operator' ? 150 : 55) + .attr('width', 140) + + children + .append('foreignObject') + .attr('data-uuid', d => d._uuid) + .attr('data-operator', d => d._operator) + .attr('class', d => `operator operator--${d._operator.toString().replace(/[ ]/g, '_')}`) + .attr('height', d => d._operator === 'Fetch Operator' ? 150 : 55) + .attr('width', 140) + .append('xhtml:body') + .style('margin', 0) + .html(d => getRenderer(d._operator)(d)) + .on('click', d => onRequestDetail(doClean(d))); + + children + .call(recurseC, onRequestDetail); + }); +} + +function getRenderer(type) { + if(type === 'Fetch Operator') { + return (d => { + return (` + <div style='display:flex;align-items: center;'> + <div class='operator-meta'> + <i class='fa ${getOperatorIcon(d._operator)}' aria-hidden='true'></i> + </div> + <div class='operator-body' style='margin-left: 10px;'> + <div>${getOperatorLabel(d)}</div> + ${d['limit:'] ? '<div><span style="font-weight: lighter;">Limit:</span> ' + d['limit:'] + ' </div>' : ''} + </div> + </div> + `); + }); + } + + return (d => { + const stats = d['Statistics:'] ? `<div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div>` : ''; + return (` + <div style='display:flex;'> + <div class='operator-meta'> + <i class='fa ${getOperatorIcon(d._operator)}' aria-hidden='true'></i> + </div> + <div class='operator-body' style='margin-left: 10px;'> + <div>${getOperatorLabel(d)}</div> + ${stats} + </div> + </div> + `); + }); + +} + +function getNumberOfRows(statistics) { + const match = statistics.match(/([^\?]*)\Num rows: (\d*)/); + return (match.length === 3 && Number.isNaN(Number(match[2])) === false) ? match[2] : 0; +} +function getOperatorLabel(d) { + const operator = d._operator; + + if(operator === 'TableScan') { + return d['alias:']; + } + + const operatorStr = operator.toString(); + if(operatorStr.endsWith(' Operator')) { + return operatorStr.substring(0, operatorStr.length - ' Operator'.length); + } + if(operatorStr.endsWith(' Pseudo-Edge')) { + return operatorStr.substring(0, operatorStr.length - ' Pseudo-Edge'.length); + } + return operatorStr ? operatorStr : 'Unknown'; +} +function getOperatorIcon(operator) { + switch(operator) { + case 'File Output Operator': + return 'fa-file-o'; + case 'Partition/Sort Pseudo-Edge': + case 'Broadcast Pseudo-Edge': + case 'Partition Pseudo-Edge': + case 'Co-partition Pseudo-Edge': + case 'Cross-product Distribute Pseudo-Edge': + case 'Reduce Output Operator': + return 'fa-compress'; + case 'Filter Operator': + return 'fa-filter'; + case 'Dynamic Partitioning Event Operator': + return 'fa-columns' + case 'Map Join Operator': + return 'fa-code-fork' + case 'Limit': + case 'Group By Operator': + case 'Select Operator': + case 'TableScan': + case 'Fetch Operator': + return 'fa-table'; + default: + return ''; + } +} +function getIcon (type, subtype) { + switch(type) { + case 'join': + return 'fa-code-fork' + case 'vectorization': + case 'job': + return; + case 'broadcast': + case 'partition-sort': + return 'fa-compress'; + case 'source': + case 'sink': + case 'group-by': + case 'select': + return 'fa-table'; + } +}; +function abbreviate(value) { + let newValue = value; + if (value >= 1000) { + const suffixes = ["", "k", "m", "b","t"]; + const suffixNum = Math.floor(("" + value).length / 3); + let shortValue = ''; + for (var precision = 2; precision >= 1; precision--) { + shortValue = parseFloat( (suffixNum != 0 ? (value / Math.pow(1000,suffixNum) ) : value).toPrecision(precision)); + const dotLessShortValue = (shortValue + '').replace(/[^a-zA-Z 0-9]+/g,''); + if (dotLessShortValue.length <= 2) { break; } + } + if (shortValue % 1 != 0) { + const shortNum = shortValue.toFixed(1); + } + newValue = shortValue+suffixes[suffixNum]; + } + return newValue; +} +function reset(zoom, svg, container) { + const vertices = container.selectAll('g.vertex'); + const bounds = []; + vertices.each(function(d) { + const cVertex = d3.select(this); + const box = cVertex.node().getBoundingClientRect(); + bounds.push(box); + }); + const PADDING_PERCENT = 0.95; + const svgRect = svg.node().getBoundingClientRect(); + const fullWidth = svgRect.width; + const fullHeight = svgRect.height; + const offsetY = svgRect.top; + const top = Math.min(...bounds.map(cBound => cBound.top)); + const left = Math.min(...bounds.map(cBound => cBound.left)); + const width = Math.max(...bounds.map(cBound => cBound.right)) - left; + const height = Math.max(...bounds.map(cBound => cBound.bottom)) - top; + const midX = left + width / 2; + const midY = top + height / 2; + if (width == 0 || height == 0){ + // nothing to fit + return; + } + const scale = PADDING_PERCENT / Math.max(width / fullWidth, height / fullHeight); + const translate = [fullWidth / 2 - scale * midX, fullHeight / 2 - scale * midY]; + + zoom.scale(scale).translate([translate[0], 50]); + + svg + .transition() + .delay(750) + .call( zoom.event ); +} + +function getConnectionPath(connector, svg, container) { + const operators = container.selectAll('.operator'); + const source = container.select(`#${connector._source._uuid}`); + const target = container.select(`#${connector._target._uuid}`); + const rSource = source.node().getBoundingClientRect(); + const rTarget = target.node().getBoundingClientRect(); + const pSource = { + x: (rSource.left + rSource.right) / 2, + y: (rSource.top + rSource.bottom) / 2, + }; + const pTarget = { + x: (rTarget.left + rTarget.right) / 2, + y: (rTarget.top + rTarget.bottom) / 2, + }; + const path = [ + pTarget + ]; + const junctionXMultiplier = (pTarget.x - pSource.x < 0) ? +1 : -1; + if(pSource.y !== pTarget.y) { + path.push({ + x: pTarget.x + junctionXMultiplier * 90, + y: pTarget.y + }, { + x: pTarget.x + junctionXMultiplier * 90, + y: pSource.y + }); + } + path.push(pSource); + const offsetY = svg.node().getBoundingClientRect().top; + return path.reduce((accumulator, cPoint, index) => { + if(index === 0) { + return accumulator + `M ${cPoint.x}, ${cPoint.y - offsetY}\n` + } else { + return accumulator + `L ${cPoint.x}, ${cPoint.y - offsetY}\n` + } + }, ''); +} + +function doClean(node) { + if(Array.isArray(node._groups)) { + return node._groups.map(cGroup => doClean(cGroup)); + } else { + return ( + Object.keys(node) + .filter(cNodeKey => cNodeKey !== '_children') + .reduce((accumulator, cNodeKey) => { + accumulator[cNodeKey] = node[cNodeKey]; + return accumulator; + }, {}) + ); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js new file mode 100644 index 0000000..70647a8 --- /dev/null +++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js @@ -0,0 +1,445 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import doEnhance from './enhancer'; +import {getProcessedVertices} from './processor'; + +export default function doTransform(data) { + const plan = getTezPlan(data); + const fetch = getFetchPlan(data); + + const vertices = [ + ...getVertices(plan), + getFetchVertex(fetch) + ]; + + const tezEdges = getEdges(plan, vertices); + const edges = getEdgesWithFetch(tezEdges, vertices); + + const enhancedVertices = doEnhance(vertices); + + const processedVertices = getProcessedVertices(enhancedVertices, edges); + + + const tree = getVertexTree(edges); + const connections = getConnections(processedVertices, edges); + const treeWithOffsetY = getTreeWithOffsetAndHeight(tree, processedVertices, connections); + + + const nodes = getNodes(processedVertices); + + return ({ + vertices: processedVertices, + edges, + tree: treeWithOffsetY, + nodes, + connections, + }); +} + +function getTezPlan(data) { + const stages = data['STAGE PLANS']; + const tezStageKey = Object.keys(stages).find(cStageKey => stages[cStageKey].hasOwnProperty('Tez')); + return stages[tezStageKey]['Tez']; +} + +function getFetchPlan(data) { + const stages = data['STAGE PLANS']; + const fetchStageKey = Object.keys(stages).find(cStageKey => stages[cStageKey].hasOwnProperty('Fetch Operator')); + return stages[fetchStageKey]['Fetch Operator']; +} + +function getFetchVertex(plan) { + return ({ + _vertex: 'Fetch', + _children: [ + Object.assign({}, plan, { + _operator: 'Fetch Operator', + _children: [] + }) + ] + }); +} + +function getVertexTree(edges) { + const rootKey = edges.find(cEdge => edges.every(tcEdge => cEdge._target !== tcEdge._source))._target; + const root = buildTree(rootKey, edges); + + return getPrunedTree(root); +} + +function getPrunedTree(node, used = {}) { + const vertices = node._vertices.filter(cVertex => used[cVertex._vertex] !== true); + vertices.forEach(cVertex => { + used[cVertex._vertex] = true; + }); + return Object.assign({}, node, { + _vertices: vertices.map(cVertex => getPrunedTree(cVertex, used)) + }); +} + +function buildTree(vertexKey, edges) { + const edgesWithVertexAsSource = edges.filter(cEdge => cEdge._target === vertexKey); + + return Object.assign({ + _vertex: vertexKey, + _vertices: edgesWithVertexAsSource.map(cEdge => buildTree(cEdge._source, edges)) + }); +} + +function getEdgesWithFetch(tezEdges, vertices) { + const rootKeys = + tezEdges + .filter(cEdge => tezEdges.every(tcEdge => cEdge._target !== tcEdge._source)) + .map(cRootEdge => cRootEdge._target); + + const uniqueRootKeys = [...new Set(rootKeys)] + + const fetchVertex = vertices.find(cVertex => cVertex._vertex === 'Fetch'); + + return ([ + ...tezEdges, + ...uniqueRootKeys.map(cRootKey => ({ + _source: cRootKey, + _target: fetchVertex._vertex, + parent: cRootKey, + type: '_PSEUDO_STAGE_EDGE', + })) + ]); +} + +function getVertices(plan) { + const VERTEX_TREE_KEYS = ['Reduce Operator Tree:', 'Map Operator Tree:']; + const vertexObj = plan['Vertices:']; + + const vertices = + Object + .keys(vertexObj) + .map(cVertexKey => { + const cVertex = vertexObj[cVertexKey]; + + const cTreeKey = VERTEX_TREE_KEYS.find(cVertexTreeKey => cVertex.hasOwnProperty(cVertexTreeKey)); + let root = [{[cVertexKey]: {}}]; + if(cTreeKey) { + // children available + root = cVertex[cTreeKey]; + } + const children = doHarmonize(root); + + return Object.assign({}, doCloneAndOmit(cVertex, VERTEX_TREE_KEYS), { + _vertex: cVertexKey, + _children: children, + }); + }); + + return vertices; +} + +function doHarmonize(nodes) { + if(Array.isArray(nodes) === false) { + return doHarmonize([ nodes ]); + } + + return nodes.map(cNode => { + const cNodeOperatorKey = Object.keys(cNode)[0]; + const cNodeItem = Object.assign({}, cNode[cNodeOperatorKey], { + _operator: cNodeOperatorKey + }); + + if(!cNodeItem.children) { + return Object.assign({}, cNodeItem, { + _children: [] + }); + } + + if(Array.isArray(cNodeItem.children)) { + return Object.assign({}, doCloneAndOmit(cNodeItem, ['children']), { + _children: doHarmonize(cNodeItem.children) + }); + } + + return Object.assign({}, doCloneAndOmit(cNodeItem, ['children']), { + _children: doHarmonize([ cNodeItem.children ]) + }); + }); +} + +function doGetHeightOfNodes(children) { + if(children.length > 0) { + return children.reduce((height, cChild) => height + doGetHeightOfNodes(cChild._children), 0); + } + return 1; +} + +function getTreeWithOffsetAndHeight(node, vertices, connections) { + const treeWithCumulativeHeight = getTreeWithCumulativeHeight(node, vertices); + const treeWithCumulativeWidth = getTreeWithIndividualWidth(treeWithCumulativeHeight, vertices); + const treeWithOffsetY = Object.assign({}, getTreeWithOffsetYInHiererchy(treeWithCumulativeWidth, connections), { + _offsetY: 0 + }); + + return treeWithOffsetY; +} + +function doGetWidthOfNodes(children = []) { + if(children.length === 0) { + return 0; + } + return 1 + Math.max(0, ...children.map(cChild => doGetWidthOfNodes(cChild._children))); +} + +function getTreeWithCumulativeHeight(node, vertices) { + const vertexKey = node._vertex; + const vertex = vertices.find(cVertex => cVertex._vertex === vertexKey); + + let _height = doGetHeightOfNodes(vertex._children); + let _vertices = []; + if(Array.isArray(node._vertices)){ + _vertices = node._vertices.map(cVertex => getTreeWithCumulativeHeight(cVertex, vertices)); + _height = Math.max(_height, _vertices.reduce((height, cVertex) => height + cVertex._height, 1)); + } + return Object.assign({}, node, vertex, { + _height, + _vertices + }); +} + +function getTreeWithIndividualWidth(node, vertices) { + const vertexKey = node._vertex; + const vertex = vertices.find(cVertex => cVertex._vertex === vertexKey); + + const _widthOfSelf = doGetWidthOfNodes(vertex._children); + + let _vertices = []; + if(Array.isArray(node._vertices) && node._vertices.length > 0){ + _vertices = node._vertices.map(cVertex => getTreeWithIndividualWidth(cVertex, vertices)); + } + return Object.assign({}, node, vertex, { + _widthOfSelf, + _vertices + }); +} + +function getTreeWithOffsetYInHiererchy(node, connections) { + const _vertices = []; + const source = node._vertices[0] && getLastOperatorOf(node._vertices[0]); + const target = getFirstOperatorOf(node); + const isFirstConnectedToLast = connections.some(cConnection => source && target && cConnection._source._uuid === source._uuid && cConnection._target._uuid === target._uuid); + let offsetY = 0; + if(!isFirstConnectedToLast) { + // if parent has a connection but not this && offset y are same, add offset + offsetY = 1; + } + for(let index = 0; index < node._vertices.length; index++) { + const cNode = node._vertices[index]; + const height = cNode._height; + + _vertices.push(Object.assign({}, getTreeWithOffsetYInHiererchy(cNode, connections), { + _offsetY: offsetY + })); + offsetY = offsetY + height; + } + + return Object.assign({}, node, { + _vertices + }); +} + +function getEdges(plan, vertices) { + const edgeObj = plan['Edges:']; + + const edges = + Object + .keys(edgeObj) + .reduce((accumulator, cEdgeKey) => { + const cEdge = edgeObj[cEdgeKey]; + + if(Array.isArray(cEdge)) { + return ([ + ...accumulator, + ...cEdge.map(tcEdge => Object.assign({}, tcEdge, { + _source: tcEdge.parent, + _target: cEdgeKey, + })) + ]); + } else { + return ([ + ...accumulator, + Object.assign({}, cEdge, { + _source: cEdge.parent, + _target: cEdgeKey, + }) + ]); + } + }, []); + + const edgesWithFixedUnions = + edges + .map(cEdge => { + if(cEdge.type === 'CONTAINS') { + return Object.assign({}, cEdge, { + _source: cEdge._target, + _target: cEdge._source, + }); + } else { + return cEdge; + } + }); + + return edgesWithFixedUnions; +} + +function doCloneAndOmit(obj, keys) { + return Object + .keys(obj) + .filter(cObjKey => keys.indexOf(cObjKey) === -1) + .reduce((tObj, cObjKey) => Object.assign({}, tObj, { + [cObjKey]: obj[cObjKey] + }), {}); +} + +function getConnections(vertices, edges) { + const connections = []; + + // iterate inside vertices to build connections between children + vertices.forEach(cVertex => { + cVertex._children.forEach(cChild => { + connections.push(...getIntraNodeConnections(cChild)); + }); + }); + + // iterate over edges to build connections + edges.forEach(cEdge => { + // get source uuid from source vertex + const sourceVertex = vertices.find(cVertex => cVertex._vertex === cEdge._source); + const sourceOperator = getLastOperatorOf(sourceVertex); + // get target uuid from target vertex + const targetVertex = vertices.find(cVertex => cVertex._vertex === cEdge._target); + const targetOperator = findVertexAsInputInNode(targetVertex, cEdge._source) || getFirstOperatorOf(targetVertex); + // push connection + connections.push({ + _source: sourceOperator, + _target: targetOperator, + }); + }); + + // iterate over vertices to find dynamic partitioning event operator + // - build connection from dpp to tablescan of target vertex + vertices.forEach(cVertex => { + // recurse over children to find dpp > source + const sourceOperators = findOperatorsInNode(cVertex, 'Dynamic Partitioning Event Operator', []); + // find first operator of target vertex > target + sourceOperators.forEach(cOperator => { + const targetVertexKey = cOperator['Target Vertex:']; + const targetVertex = vertices.find(cVertex => cVertex._vertex === targetVertexKey); + + const targetOperator = getFirstOperatorOf(targetVertex); + + // push connection + connections.push({ + _source: cOperator, + _target: targetOperator, + }); + }) + }); + + return connections; +} + +function findVertexAsInputInNode(node, vertexId) { + let isInputPresent = false; + + const inputs = node['input vertices:']; + if(inputs) { + isInputPresent = Object.keys(inputs).some(cInputKey => inputs[cInputKey] === vertexId); + } + if(Array.isArray(node._groups)) { + isInputPresent = isInputPresent || node._groups.some(cGroupedOperator => { + const inputs = cGroupedOperator['input vertices:']; + if(inputs) { + return Object.keys(inputs).some(cInputKey => inputs[cInputKey] === vertexId); + } + return false; + }); + } + + if(isInputPresent) { + return node; + } else { + for(let i = 0; i < node._children.length; i++) { + const cChild = node._children[i]; + const operator = findVertexAsInputInNode(cChild, vertexId); + + if(operator) { + return operator; + } + } + } + + return false; +} + +function getLastOperatorOf(vertex) { + let operator = vertex._children[0]; + while(operator._children.length > 0) { + operator = operator._children[0]; + } + return operator; +} + +function getFirstOperatorOf(vertex) { + return vertex._children[0]; +} + +function findOperatorsInNode(node, operatorKey, resultsAggregator) { + if(node._operator === operatorKey) { + return resultsAggregator.push(node); + } + + node._children.forEach(cChild => findOperatorsInNode(cChild, operatorKey, resultsAggregator)); + + return resultsAggregator; +} + +function getIntraNodeConnections(node) { + return node._children.reduce((aggregator, cChild) => { + aggregator.push({ + _source: node, + _target: cChild, + }); + aggregator.push( + ...getIntraNodeConnections(cChild) + ); + return aggregator; + }, []); +} + +function getNodes(vertices) { + return vertices.reduce((accumulator, cVertex) => ([ + ...accumulator, + ...getNodesFromChildren(cVertex._children) + ]), []); +} + +function getNodesFromChildren(children) { + return children.reduce((accumulator, cChild) => ([ + ...accumulator, + cChild, + ...getNodesFromChildren(cChild._children) + ]), []); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/bower.json ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/bower.json b/contrib/views/hive20/src/main/resources/ui/bower.json index 9fa7076..2069c88 100644 --- a/contrib/views/hive20/src/main/resources/ui/bower.json +++ b/contrib/views/hive20/src/main/resources/ui/bower.json @@ -1,7 +1,8 @@ { "name": "ui", "dependencies": { - "d3": "~4.5.0", + "d3": "~3.5.17", + "webcola": "~3.3.2", "ember": "~2.7.0", "ember-cli-shims": "~0.1.1", "ember-qunit-notifications": "0.1.0", http://git-wip-us.apache.org/repos/asf/ambari/blob/812397d3/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js b/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js index d53cdac..c88799a 100644 --- a/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js +++ b/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js @@ -54,6 +54,7 @@ module.exports = function(defaults) { app.import('bower_components/codemirror/addon/hint/sql-hint.js'); app.import('bower_components/codemirror/addon/hint/show-hint.js'); app.import('bower_components/d3/d3.js'); + app.import('bower_components/webcola/WebCola/cola.min.js'); app.import('bower_components/codemirror/lib/codemirror.css'); app.import('bower_components/jquery-ui/jquery-ui.js'); app.import('bower_components/jquery-ui/themes/base/jquery-ui.css');
