This is an automated email from the ASF dual-hosted git repository. karthikz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push: new 84737b0 Add component config in heron UI and Tracker (#2860) 84737b0 is described below commit 84737b088eb224fb04727c820653679aeb297477 Author: Ning Wang <nw...@twitter.com> AuthorDate: Tue Jun 26 17:07:43 2018 -0700 Add component config in heron UI and Tracker (#2860) --- heron/tools/tracker/src/python/tracker.py | 10 ++++ heron/tools/tracker/tests/python/BUILD | 2 +- .../tools/tracker/tests/python/tracker_unittest.py | 19 ++++++++ heron/tools/ui/resources/static/js/config.js | 53 ++++++++++++++++++---- 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py index c03621a..4fb6e87 100644 --- a/heron/tools/tracker/src/python/tracker.py +++ b/heron/tools/tracker/src/python/tracker.py @@ -430,6 +430,7 @@ class Tracker(object): "spouts": {}, "bolts": {}, "config": {}, + "components": {} } if not topology.physical_plan: @@ -447,12 +448,21 @@ class Tracker(object): # Configs if topology.physical_plan.topology.topology_config: physicalPlan["config"] = convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs) + for spout in spouts: spout_name = spout.comp.name physicalPlan["spouts"][spout_name] = [] + if spout_name not in physicalPlan["components"]: + physicalPlan["components"][spout_name] = { + "config": convert_pb_kvs(spout.comp.config.kvs) + } for bolt in bolts: bolt_name = bolt.comp.name physicalPlan["bolts"][bolt_name] = [] + if bolt_name not in physicalPlan["components"]: + physicalPlan["components"][bolt_name] = { + "config": convert_pb_kvs(bolt.comp.config.kvs) + } for stmgr in stmgrs: host = stmgr.host_name diff --git a/heron/tools/tracker/tests/python/BUILD b/heron/tools/tracker/tests/python/BUILD index 2c752e1..26d1bd8 100644 --- a/heron/tools/tracker/tests/python/BUILD +++ b/heron/tools/tracker/tests/python/BUILD @@ -58,7 +58,7 @@ pex_pytest( pex_pytest( name = "tracker_unittest", - srcs = ["tracker_unittest.py"], + srcs = ["tracker_unittest.py", "mock_proto.py"], deps = [ "//heron/tools/tracker/src/python:tracker-py", "//heron/proto:proto-py", diff --git a/heron/tools/tracker/tests/python/tracker_unittest.py b/heron/tools/tracker/tests/python/tracker_unittest.py index 4a4ba67..136c6b9 100644 --- a/heron/tools/tracker/tests/python/tracker_unittest.py +++ b/heron/tools/tracker/tests/python/tracker_unittest.py @@ -8,6 +8,7 @@ import heron.proto.execution_state_pb2 as protoEState from heron.statemgrs.src.python import statemanagerfactory from heron.tools.tracker.src.python.topology import Topology from heron.tools.tracker.src.python.tracker import Tracker +from mock_proto import MockProto class TrackerTest(unittest.TestCase): def setUp(self): @@ -215,3 +216,21 @@ class TrackerTest(unittest.TestCase): self.tracker.removeTopology('top_name4', 'mock_name2') self.assertItemsEqual([self.topology3, self.topology5], self.tracker.topologies) + + def test_extract_physical_plan(self): + # Create topology + pb_pplan = MockProto().create_mock_simple_physical_plan() + topology = Topology('topology_name', 'state_manager') + topology.set_physical_plan(pb_pplan) + # Extract physical plan + pplan = self.tracker.extract_physical_plan(topology) + # Mock topology doesn't have topology config and instances + self.assertEqual(pplan['config'], {}) + self.assertEqual(pplan['bolts'], {'mock_bolt': []}) + self.assertEqual(pplan['spouts'], {'mock_spout': []}) + self.assertEqual(pplan['components']['mock_bolt']['config'], + {'topology.component.parallelism': '1'}) + self.assertEqual(pplan['components']['mock_spout']['config'], + {'topology.component.parallelism': '1'}) + self.assertEqual(pplan['instances'], {}) + self.assertEqual(pplan['stmgrs'], {}) diff --git a/heron/tools/ui/resources/static/js/config.js b/heron/tools/ui/resources/static/js/config.js index a0b19b5..87a0409 100644 --- a/heron/tools/ui/resources/static/js/config.js +++ b/heron/tools/ui/resources/static/js/config.js @@ -2,9 +2,12 @@ var ConfigTable = React.createClass({ getInitialState: function() { - return {'config': {}}; + return { + 'config': {}, + 'component_config': {} + }; }, - getTopologyConfig: function() { + getConfig: function() { urlTokens = [ this.props.baseUrl, 'topologies', this.props.cluster, @@ -17,9 +20,23 @@ var ConfigTable = React.createClass({ url: fetchUrl, dataType: 'json', success: function(response) { - if (response.hasOwnProperty('result') - && response.result.hasOwnProperty('config')) { - this.setState({config: response.result.config}); + if (response.hasOwnProperty('result')) { + // Topology config + var config = {}; + if (response.result.hasOwnProperty('config')) { + config = response.result.config; + } + // Component config + var componentConfig = {}; + if (response.result.hasOwnProperty('components')) { + for (var component in response.result.components) { + componentConfig[component] = response.result.components[component].config; + } + } + this.setState({ + config: config, + component_config: componentConfig + }); } }.bind(this), @@ -80,27 +97,43 @@ var ConfigTable = React.createClass({ }, componentWillMount: function () { - this.getTopologyConfig(); + this.getConfig(); }, componentDidUpdate: function(prevProps, prevState) { if (prevProps.topology != this.props.topology) { - this.getTopologyConfig(); + this.getConfig(); } }, render: function() { var configData = this.state.config; + var componentConfigData = this.state.component_config; var headings = ['Property', 'Value']; var title = 'Configuration'; var rows = []; var self = this; + // Fill topology config data for (property in configData) { var configValue = configData[property]; - if (typeof configValue == 'string') { - rows.push([property, configValue]); - } else { + if (typeof configValue == 'object') { rows.push([property, JSON.parse(configValue.value)]); + } else { + rows.push([property, configValue]); } } + // Fill component config data + for (component in componentConfigData) { + data = componentConfigData[component]; + for (property in data) { + var key = '[' + component + '] ' + property; + var configValue = data[property]; + if (typeof configValue == 'object') { + rows.push([key, JSON.parse(configValue.value)]); + } else { + rows.push([key, configValue]); + } + } + } + var tableContent = ( <tbody>{ rows.map(function(row) {