This is an automated email from the ASF dual-hosted git repository.
karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 84737b0 Add component config in heron UI and Tracker (#2860)
84737b0 is described below
commit 84737b088eb224fb04727c820653679aeb297477
Author: Ning Wang <nw...@twitter.com>
AuthorDate: Tue Jun 26 17:07:43 2018 -0700
Add component config in heron UI and Tracker (#2860)
---
heron/tools/tracker/src/python/tracker.py | 10 ++++
heron/tools/tracker/tests/python/BUILD | 2 +-
.../tools/tracker/tests/python/tracker_unittest.py | 19 ++++++++
heron/tools/ui/resources/static/js/config.js | 53 ++++++++++++++++++----
4 files changed, 73 insertions(+), 11 deletions(-)
diff --git a/heron/tools/tracker/src/python/tracker.py
b/heron/tools/tracker/src/python/tracker.py
index c03621a..4fb6e87 100644
--- a/heron/tools/tracker/src/python/tracker.py
+++ b/heron/tools/tracker/src/python/tracker.py
@@ -430,6 +430,7 @@ class Tracker(object):
"spouts": {},
"bolts": {},
"config": {},
+ "components": {}
}
if not topology.physical_plan:
@@ -447,12 +448,21 @@ class Tracker(object):
# Configs
if topology.physical_plan.topology.topology_config:
physicalPlan["config"] =
convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs)
+
for spout in spouts:
spout_name = spout.comp.name
physicalPlan["spouts"][spout_name] = []
+ if spout_name not in physicalPlan["components"]:
+ physicalPlan["components"][spout_name] = {
+ "config": convert_pb_kvs(spout.comp.config.kvs)
+ }
for bolt in bolts:
bolt_name = bolt.comp.name
physicalPlan["bolts"][bolt_name] = []
+ if bolt_name not in physicalPlan["components"]:
+ physicalPlan["components"][bolt_name] = {
+ "config": convert_pb_kvs(bolt.comp.config.kvs)
+ }
for stmgr in stmgrs:
host = stmgr.host_name
diff --git a/heron/tools/tracker/tests/python/BUILD
b/heron/tools/tracker/tests/python/BUILD
index 2c752e1..26d1bd8 100644
--- a/heron/tools/tracker/tests/python/BUILD
+++ b/heron/tools/tracker/tests/python/BUILD
@@ -58,7 +58,7 @@ pex_pytest(
pex_pytest(
name = "tracker_unittest",
- srcs = ["tracker_unittest.py"],
+ srcs = ["tracker_unittest.py", "mock_proto.py"],
deps = [
"//heron/tools/tracker/src/python:tracker-py",
"//heron/proto:proto-py",
diff --git a/heron/tools/tracker/tests/python/tracker_unittest.py
b/heron/tools/tracker/tests/python/tracker_unittest.py
index 4a4ba67..136c6b9 100644
--- a/heron/tools/tracker/tests/python/tracker_unittest.py
+++ b/heron/tools/tracker/tests/python/tracker_unittest.py
@@ -8,6 +8,7 @@ import heron.proto.execution_state_pb2 as protoEState
from heron.statemgrs.src.python import statemanagerfactory
from heron.tools.tracker.src.python.topology import Topology
from heron.tools.tracker.src.python.tracker import Tracker
+from mock_proto import MockProto
class TrackerTest(unittest.TestCase):
def setUp(self):
@@ -215,3 +216,21 @@ class TrackerTest(unittest.TestCase):
self.tracker.removeTopology('top_name4', 'mock_name2')
self.assertItemsEqual([self.topology3, self.topology5],
self.tracker.topologies)
+
+ def test_extract_physical_plan(self):
+ # Create topology
+ pb_pplan = MockProto().create_mock_simple_physical_plan()
+ topology = Topology('topology_name', 'state_manager')
+ topology.set_physical_plan(pb_pplan)
+ # Extract physical plan
+ pplan = self.tracker.extract_physical_plan(topology)
+ # Mock topology doesn't have topology config and instances
+ self.assertEqual(pplan['config'], {})
+ self.assertEqual(pplan['bolts'], {'mock_bolt': []})
+ self.assertEqual(pplan['spouts'], {'mock_spout': []})
+ self.assertEqual(pplan['components']['mock_bolt']['config'],
+ {'topology.component.parallelism': '1'})
+ self.assertEqual(pplan['components']['mock_spout']['config'],
+ {'topology.component.parallelism': '1'})
+ self.assertEqual(pplan['instances'], {})
+ self.assertEqual(pplan['stmgrs'], {})
diff --git a/heron/tools/ui/resources/static/js/config.js
b/heron/tools/ui/resources/static/js/config.js
index a0b19b5..87a0409 100644
--- a/heron/tools/ui/resources/static/js/config.js
+++ b/heron/tools/ui/resources/static/js/config.js
@@ -2,9 +2,12 @@
var ConfigTable = React.createClass({
getInitialState: function() {
- return {'config': {}};
+ return {
+ 'config': {},
+ 'component_config': {}
+ };
},
- getTopologyConfig: function() {
+ getConfig: function() {
urlTokens = [ this.props.baseUrl,
'topologies',
this.props.cluster,
@@ -17,9 +20,23 @@ var ConfigTable = React.createClass({
url: fetchUrl,
dataType: 'json',
success: function(response) {
- if (response.hasOwnProperty('result')
- && response.result.hasOwnProperty('config')) {
- this.setState({config: response.result.config});
+ if (response.hasOwnProperty('result')) {
+ // Topology config
+ var config = {};
+ if (response.result.hasOwnProperty('config')) {
+ config = response.result.config;
+ }
+ // Component config
+ var componentConfig = {};
+ if (response.result.hasOwnProperty('components')) {
+ for (var component in response.result.components) {
+ componentConfig[component] =
response.result.components[component].config;
+ }
+ }
+ this.setState({
+ config: config,
+ component_config: componentConfig
+ });
}
}.bind(this),
@@ -80,27 +97,43 @@ var ConfigTable = React.createClass({
},
componentWillMount: function () {
- this.getTopologyConfig();
+ this.getConfig();
},
componentDidUpdate: function(prevProps, prevState) {
if (prevProps.topology != this.props.topology) {
- this.getTopologyConfig();
+ this.getConfig();
}
},
render: function() {
var configData = this.state.config;
+ var componentConfigData = this.state.component_config;
var headings = ['Property', 'Value'];
var title = 'Configuration';
var rows = [];
var self = this;
+ // Fill topology config data
for (property in configData) {
var configValue = configData[property];
- if (typeof configValue == 'string') {
- rows.push([property, configValue]);
- } else {
+ if (typeof configValue == 'object') {
rows.push([property, JSON.parse(configValue.value)]);
+ } else {
+ rows.push([property, configValue]);
}
}
+ // Fill component config data
+ for (component in componentConfigData) {
+ data = componentConfigData[component];
+ for (property in data) {
+ var key = '[' + component + '] ' + property;
+ var configValue = data[property];
+ if (typeof configValue == 'object') {
+ rows.push([key, JSON.parse(configValue.value)]);
+ } else {
+ rows.push([key, configValue]);
+ }
+ }
+ }
+
var tableContent = (
<tbody>{
rows.map(function(row) {
