Datapath flows can be arranged into a "tree"-like structure based on recirculation ids and input ports.
A recirculation group is composed of flows sharing the same "recirc_id" and "in_port" match. Within that group, flows are arranged in blocks of flows that have the same action list. Finally, if an action associated with one of this "blocks" contains a "recirc" action, the recirculation group is shown underneath. When filtering, instead of blindly dropping non-matching flows, drop all the "subtrees" that don't have any matching flow. Examples: $ ovs-flowviz -i dpflows.txt --style dark datapath tree | less -R $ ovs-flowviz -i dpflows.txt --filter "output.port=eth0" datapath tree This patch adds the logic to build this structure in a format-agnostic object called FlowTree and adds support for formatting it in the console. Console format supports: - head-maps formatting of statistics - hash-based pallete of recirculation ids: each recirculation id is assigned a unique color to easily follow the sequence of related actions. Acked-by: Eelco Chaudron <[email protected]> Signed-off-by: Adrian Moreno <[email protected]> --- python/automake.mk | 1 + python/ovs/flow/kv.py | 9 + python/ovs/flowviz/console.py | 41 ++- python/ovs/flowviz/format.py | 60 +++- python/ovs/flowviz/odp/cli.py | 20 ++ python/ovs/flowviz/odp/tree.py | 512 +++++++++++++++++++++++++++++++++ 6 files changed, 625 insertions(+), 18 deletions(-) create mode 100644 python/ovs/flowviz/odp/tree.py diff --git a/python/automake.mk b/python/automake.mk index 532e2c872..f372a72a2 100644 --- a/python/automake.mk +++ b/python/automake.mk @@ -71,6 +71,7 @@ ovs_flowviz = \ python/ovs/flowviz/main.py \ python/ovs/flowviz/odp/__init__.py \ python/ovs/flowviz/odp/cli.py \ + python/ovs/flowviz/odp/tree.py \ python/ovs/flowviz/ofp/__init__.py \ python/ovs/flowviz/ofp/cli.py \ python/ovs/flowviz/ofp/html.py \ diff --git a/python/ovs/flow/kv.py b/python/ovs/flow/kv.py index f7d7be0cf..3afbf9fce 100644 --- a/python/ovs/flow/kv.py +++ b/python/ovs/flow/kv.py @@ -67,6 +67,15 @@ class KeyValue(object): def __repr__(self): return "{}('{}')".format(self.__class__.__name__, self) + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.key == other.key and self.value == other.value + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + class KVDecoders(object): """KVDecoders class is used by KVParser to select how to decode the value diff --git a/python/ovs/flowviz/console.py b/python/ovs/flowviz/console.py index c8a78ec11..ab91512fe 100644 --- a/python/ovs/flowviz/console.py +++ b/python/ovs/flowviz/console.py @@ -13,6 +13,8 @@ # limitations under the License. import colorsys +import itertools +import zlib from rich.console import Console from rich.color import Color @@ -79,6 +81,14 @@ class ConsoleBuffer(FlowBuffer): """ return self._append(kv.meta.vstring, style) + def append_value_omitted(self, kv): + """Append an omitted value. + Args: + kv (KeyValue): the KeyValue instance to append + """ + dots = "." * len(kv.meta.vstring) + return self._append(dots, None) + def append_extra(self, extra, style): """Append extra string. Args: @@ -107,20 +117,21 @@ class ConsoleFormatter(FlowFormatter): def style_from_opts(self, opts): return self._style_from_opts(opts, "console", Style) - def print_flow(self, flow, highlighted=None): + def print_flow(self, flow, highlighted=None, omitted=None): """Prints a flow to the console. Args: flow (ovs_dbg.OFPFlow): the flow to print style (dict): Optional; style dictionary to use highlighted (list): Optional; list of KeyValues to highlight + omitted (list): Optional; list of KeyValues to omit """ buf = ConsoleBuffer(Text()) - self.format_flow(buf, flow, highlighted) - self.console.print(buf.text) + self.format_flow(buf, flow, highlighted, omitted) + self.console.print(buf.text, soft_wrap=True) - def format_flow(self, buf, flow, highlighted=None): + def format_flow(self, buf, flow, highlighted=None, omitted=None): """Formats the flow into the provided buffer as a rich.Text. Args: @@ -128,9 +139,10 @@ class ConsoleFormatter(FlowFormatter): flow (ovs_dbg.OFPFlow): the flow to format style (FlowStyle): Optional; style object to use highlighted (list): Optional; list of KeyValues to highlight + omitted (list): Optional; list of KeyValues to omit """ return super(ConsoleFormatter, self).format_flow( - buf, flow, self.style, highlighted + buf, flow, self.style, highlighted, omitted ) @@ -157,6 +169,25 @@ def heat_pallete(min_value, max_value): return heat +def hash_pallete(hue, saturation, value): + """Generates a color pallete with the cartesian product + of the hsv values provided and returns a callable that assigns a color for + each value hash + """ + HSV_tuples = itertools.product(hue, saturation, value) + RGB_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), HSV_tuples) + styles = [ + Style(color=Color.from_rgb(r * 255, g * 255, b * 255)) + for r, g, b in RGB_tuples + ] + + def get_style(string): + hash_val = zlib.crc32(bytes(str(string), "utf-8")) + return styles[hash_val % len(styles)] + + return get_style + + def default_highlight(): """Generates a default style for highlights.""" return Style(underline=True) diff --git a/python/ovs/flowviz/format.py b/python/ovs/flowviz/format.py index 70af2fa26..67711a92f 100644 --- a/python/ovs/flowviz/format.py +++ b/python/ovs/flowviz/format.py @@ -225,7 +225,8 @@ class FlowFormatter: return FlowStyle({k: style_constructor(**v) for k, v in style.items()}) - def format_flow(self, buf, flow, style_obj=None, highlighted=None): + def format_flow(self, buf, flow, style_obj=None, highlighted=None, + omitted=None): """Formats the flow into the provided buffer. Args: @@ -233,25 +234,41 @@ class FlowFormatter: flow (ovs_dbg.OFPFlow): the flow to format style_obj (FlowStyle): Optional; style to use highlighted (list): Optional; list of KeyValues to highlight + omitted (list): Optional; dict of keys to omit indexed by section + name. """ last_printed_pos = 0 + first = True - if style_obj: + if style_obj or omitted: style_obj = style_obj or FlowStyle() for section in sorted(flow.sections, key=lambda x: x.pos): - buf.append_extra( - flow.orig[last_printed_pos : section.pos], - style=style_obj.get("default"), - ) + section_omitted = (omitted or {}).get(section.name) + if isinstance(section_omitted, str) and \ + section_omitted == "all": + last_printed_pos += section.pos + len(section.string) + continue + + # Do not print leading extra strings (e.g: spaces and commas) + # if it's the first section that gets printed. + if not first: + buf.append_extra( + flow.orig[last_printed_pos : section.pos], + style=style_obj.get("default"), + ) + self.format_kv_list( - buf, section.data, section.string, style_obj, highlighted + buf, section.data, section.string, style_obj, highlighted, + section_omitted ) last_printed_pos = section.pos + len(section.string) + first = False else: # Don't pay the cost of formatting each section one by one. buf.append_extra(flow.orig.strip(), None) - def format_kv_list(self, buf, kv_list, full_str, style_obj, highlighted): + def format_kv_list(self, buf, kv_list, full_str, style_obj, highlighted, + omitted=None): """Format a KeyValue List. Args: @@ -260,10 +277,14 @@ class FlowFormatter: full_str (str): the full string containing all k-v style_obj (FlowStyle): a FlowStyle object to use highlighted (list): Optional; list of KeyValues to highlight + highlighted (list): Optional; list of KeyValues to highlight + omitted (list): Optional; list of keys to omit """ for i, kv in enumerate(kv_list): + key_omitted = kv.key in omitted if omitted else False written = self.format_kv( - buf, kv, style_obj=style_obj, highlighted=highlighted + buf, kv, style_obj=style_obj, highlighted=highlighted, + omitted=key_omitted ) end = ( @@ -277,7 +298,7 @@ class FlowFormatter: style=style_obj.get("default"), ) - def format_kv(self, buf, kv, style_obj, highlighted=None): + def format_kv(self, buf, kv, style_obj, highlighted=None, omitted=False): """Format a KeyValue A formatted keyvalue has the following parts: @@ -288,6 +309,7 @@ class FlowFormatter: kv (KeyValue): The KeyValue to print style_obj (FlowStyle): The style object to use highlighted (list): Optional; list of KeyValues to highlight + omitted(boolean): Whether the value shall be omitted. Returns the number of printed characters. """ @@ -308,9 +330,14 @@ class FlowFormatter: buf.append_delim(kv, style_obj.get_delim_style(is_highlighted)) ret += len(kv.meta.delim) - value_style = style_obj.get_value_style(kv, is_highlighted) - buf.append_value(kv, value_style) # format value - ret += len(kv.meta.vstring) + if omitted: + buf.append_value_omitted(kv) + ret += len(kv.meta.vstring) + + else: + value_style = style_obj.get_value_style(kv, is_highlighted) + buf.append_value(kv, value_style) # format value + ret += len(kv.meta.vstring) if kv.meta.end_delim: buf.append_end_delim(kv, style_obj.get_delim_style(is_highlighted)) @@ -362,6 +389,13 @@ class FlowBuffer: """ raise NotImplementedError + def append_value_omitted(self, kv): + """Append an omitted value. + Args: + kv (KeyValue): the KeyValue instance to append + """ + raise NotImplementedError + def append_extra(self, extra, style): """Append extra string. Args: diff --git a/python/ovs/flowviz/odp/cli.py b/python/ovs/flowviz/odp/cli.py index 2b82d02fe..36f5b3db2 100644 --- a/python/ovs/flowviz/odp/cli.py +++ b/python/ovs/flowviz/odp/cli.py @@ -15,6 +15,7 @@ import click from ovs.flowviz.main import maincli +from ovs.flowviz.odp.tree import ConsoleTreeProcessor from ovs.flowviz.process import ( ConsoleProcessor, JSONDatapathProcessor, @@ -54,3 +55,22 @@ def console(opts, heat_map): ) proc.process() proc.print() + + [email protected]() [email protected]( + "-h", + "--heat-map", + is_flag=True, + default=False, + show_default=True, + help="Create heat-map with packet and byte counters", +) [email protected]_obj +def tree(opts, heat_map): + """Print the flows in a tree based on the 'recirc_id'.""" + processor = ConsoleTreeProcessor( + opts, heat_map=["packets", "bytes"] if heat_map else [] + ) + processor.process() + processor.print() diff --git a/python/ovs/flowviz/odp/tree.py b/python/ovs/flowviz/odp/tree.py new file mode 100644 index 000000000..d6526504b --- /dev/null +++ b/python/ovs/flowviz/odp/tree.py @@ -0,0 +1,512 @@ +# Copyright (c) 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from rich.style import Style +from rich.console import Group +from rich.panel import Panel +from rich.text import Text +from rich.tree import Tree + +from ovs.compat.sortedcontainers import SortedList +from ovs.flowviz.console import ( + ConsoleFormatter, + ConsoleBuffer, + hash_pallete, + heat_pallete, + file_header, +) +from ovs.flowviz.process import ( + FileProcessor, +) + + +class TreeFlow(object): + """A flow within a Tree.""" + + def __init__(self, flow, filter=None): + self._flow = flow + self._visible = True + if filter: + self._matches = filter.evaluate(flow) + else: + self._matches = True + + @property + def flow(self): + return self._flow + + @property + def visible(self): + return self._visible + + @visible.setter + def visible(self, new_visible): + self._visible = new_visible + + @property + def matches(self): + return self._matches + + +class FlowBlock(object): + """A block of flows in a Tree. Flows are arranged together in a block + if they have the same action. + """ + + def __init__(self, tflow): + """Create a FlowBlock based on a flow. + Args: + flow: TreeFlow + """ + self._flows = SortedList([], self.__key) + self._next_recirc_nodes = SortedList([], key=lambda x: -x.pkts) + self._actions = tflow.flow.actions_kv + self._sum_pkts = tflow.flow.info.get("packets") or 0 + self._visible = False + + self._flows.add(tflow) + + self._equal_match = [ + (i, kv) + for i, kv in enumerate(tflow.flow.match_kv) + if kv.key not in ["in_port", "recirc_id"] + ] + + in_port = tflow.flow.match.get("in_port") + self._next_recirc_inport = [ + (recirc, in_port) for recirc in self._get_next_recirc(tflow.flow) + ] + + @property + def flows(self): + return self._flows + + @property + def pkts(self): + return self._sum_pkts + + @property + def visible(self): + return self._visible + + @property + def equal_match(self): + return self._equal_match + + @property + def next_recirc_nodes(self): + return self._next_recirc_nodes + + def add_if_belongs(self, tflow): + """Add TreeFlow to block if it belongs here.""" + if not self._belongs(tflow): + return False + + to_del = [] + for i, (orig_i, kv) in enumerate(self.equal_match): + if orig_i >= len(tflow.flow.match_kv): + kv_i = None + else: + kv_i = tflow.flow.match_kv[orig_i] + + if kv_i != kv: + to_del.append(i) + + for i in sorted(to_del, reverse=True): + del self.equal_match[i] + + self._sum_pkts += tflow.flow.info.get("packets") or 0 + self._flows.add(tflow) + return True + + def build(self, recirc_nodes): + """Populates next_recirc_nodes given a dictionary of RecircNode objects + indexed by recirc_id and in_port. + """ + for recirc, in_port in self._next_recirc_inport: + try: + self._next_recirc_nodes.add(recirc_nodes[recirc][in_port]) + except KeyError: + print( + f"mising [recirc_id {hex(recirc)} inport {in_port}]. " + "Flow tree will be incomplete.", + file=sys.stderr, + ) + + def compute_visible(self): + """Determines if the block should be visible. + A FlowBlock is visible if any of its flows is. + + If any of the nested RecircNodes is visible, all flows should be + visible. If not, only the ones that match should. + """ + nested_recirc_visible = False + + for recirc in self._next_recirc_nodes: + recirc.compute_visible() + if recirc.visible: + nested_recirc_visible = True + + for tflow in self._flows: + tflow.visible = True if nested_recirc_visible else tflow.matches + if tflow.visible: + self._visible = True + + def _belongs(self, tflow): + if len(tflow.flow.actions_kv) != len(self._actions): + return False + return all( + [a == b for a, b in zip(tflow.flow.actions_kv, self._actions)] + ) + + def __key(self, f): + return -(f.flow.info.get("packets") or 0) + + def _get_next_recirc(self, flow): + """Get the next recirc_ids from a Flow. + + The recirc_id is obtained from actions such as recirc, but also + complex actions such as check_pkt_len and sample + Args: + flow (ODPFlow): flow to get the recirc_id from. + Returns: + set of next recirculation ids. + """ + + # Helper function to find a recirc in a dictionary of actions. + def find_in_list(actions_list): + recircs = [] + for item in actions_list: + (action, value) = next(iter(item.items())) + if action == "recirc": + recircs.append(value) + elif action == "check_pkt_len": + recircs.extend(find_in_list(value.get("gt"))) + recircs.extend(find_in_list(value.get("le"))) + elif action == "clone": + recircs.extend(find_in_list(value)) + elif action == "sample": + recircs.extend(find_in_list(value.get("actions"))) + return recircs + + recircs = [] + recircs.extend(find_in_list(flow.actions)) + + return set(recircs) + + +class RecircNode(object): + def __init__(self, recirc, in_port, heat_map=[]): + self._recirc = recirc + self._in_port = in_port + self._visible = False + self._sum_pkts = 0 + self._heat_map_fields = heat_map + self._min = dict.fromkeys(self._heat_map_fields, -1) + self._max = dict.fromkeys(self._heat_map_fields, 0) + + self._blocks = [] + self._sorted_blocks = SortedList([], key=lambda x: -x.pkts) + + @property + def recirc(self): + return self._recirc + + @property + def in_port(self): + return self._in_port + + @property + def visible(self): + return self._visible + + @property + def pkts(self): + """Returns the blocks sorted by pkts. + Should not be called before running build().""" + return self._sum_pkts + + @property + def min(self): + return self._min + + @property + def max(self): + return self._max + + def visible_blocks(self): + """Returns visible blocks sorted by pkts. + Should not be called before running build().""" + return filter(lambda x: x.visible, self._sorted_blocks) + + def add_flow(self, tflow): + assert tflow.flow.match.get("recirc_id") == self.recirc + assert tflow.flow.match.get("in_port") == self.in_port + + self._sum_pkts += tflow.flow.info.get("packets") or 0 + + # Accumulate minimum and maximum values for later use in heat-map. + for field in self._heat_map_fields: + val = tflow.flow.info.get(field) + if self._min[field] == -1 or val < self._min[field]: + self._min[field] = val + if val > self._max[field]: + self._max[field] = val + + for b in self._blocks: + if b.add_if_belongs(tflow): + return + + self._blocks.append(FlowBlock(tflow)) + + def build(self, recirc_nodes): + """Builds the recirculation links of nested blocks. + + Args: + recirc_nodes: Dictionary of RecircNode objects indexed by + recirc_id and in_port. + """ + for block in self._blocks: + block.build(recirc_nodes) + self._sorted_blocks.add(block) + + def compute_visible(self): + """Determine if the RecircNode should be visible. + A RecircNode is visible if any of its blocks is. + """ + for block in self._blocks: + block.compute_visible() + if block.visible: + self._visible = True + + +class FlowTree: + """A Flow tree is a a class that processes datapath flows into a tree based + on recirculation ids. + + Args: + flows (list[ODPFlow]): Optional, initial list of flows + heat_map_fields (list[str]): Optional, info fields to calculate + maximum and minimum values. + """ + + def __init__(self, flows=None, heat_map_fields=[]): + self._recirc_nodes = {} + self._all_recirc_nodes = [] + self._heat_map_fields = heat_map_fields + if flows: + for flow in flows: + self.add(flow) + + @property + def recirc_nodes(self): + """Recirculation nodes in a double-dictionary. + First-level key: recirc_id. Second-level key: in_port. + """ + return self._recirc_nodes + + @property + def all_recirc_nodes(self): + """All Recirculation nodes in a list.""" + return self._all_recirc_nodes + + def add(self, flow, filter=None): + """Add a flow""" + rid = flow.match.get("recirc_id") or 0 + in_port = flow.match.get("in_port") or 0 + + if not self._recirc_nodes.get(rid): + self._recirc_nodes[rid] = {} + + if not self._recirc_nodes.get(rid).get(in_port): + node = RecircNode(rid, in_port, heat_map=self._heat_map_fields) + self._recirc_nodes[rid][in_port] = node + self._all_recirc_nodes.append(node) + + self._recirc_nodes[rid][in_port].add_flow(TreeFlow(flow, filter)) + + def build(self): + """Build the flow tree.""" + for node in self._all_recirc_nodes: + node.build(self._recirc_nodes) + + # Once recirculation links have been built. Determine what should stay + # visible recursively starting by recirc_id = 0. + for _, node in self._recirc_nodes.get(0).items(): + node.compute_visible() + + def min_max(self): + """Return a dictionary, indexed by the heat_map_fields, of minimum + and maximum values. + """ + min_vals = {field: [] for field in self._heat_map_fields} + max_vals = {field: [] for field in self._heat_map_fields} + + if not self._heat_map_fields: + return None + + for node in self._all_recirc_nodes: + if not node.visible: + continue + for field in self._heat_map_fields: + min_vals[field].append(node.min[field]) + max_vals[field].append(node.max[field]) + + return { + field: ( + min(min_vals[field]) if min_vals[field] else 0, + max(max_vals[field]) if max_vals[field] else 0, + ) + for field in self._heat_map_fields + } + + +class ConsoleTreeProcessor(FileProcessor): + def __init__(self, opts, heat_map=[]): + super().__init__(opts, "odp") + self.trees = {} + self.ofconsole = ConsoleFormatter(self.opts) + self.style = self.ofconsole.style + self.heat_map = heat_map + self.tree = None + self.curr_file = "" + + if self.style: + # Generate a color pallete for recirc ids. + self.recirc_style_gen = hash_pallete( + hue=[x / 50 for x in range(0, 50)], + saturation=[0.7], + value=[0.8], + ) + + self.style.set_default_value_style(Style(color="grey66")) + self.style.set_key_style("output", Style(color="green")) + self.style.set_value_style("output", Style(color="green")) + self.style.set_value_style("recirc", self.recirc_style_gen) + self.style.set_value_style("recirc_id", self.recirc_style_gen) + + def start_file(self, name, filename): + self.tree = FlowTree(heat_map_fields=self.heat_map) + self.curr_file = name + + def start_thread(self, name): + if not self.tree: + self.tree = FlowTree(heat_map_fields=self.heat_map) + + def stop_thread(self, name): + full_name = self.curr_file + f" ({name})" + if self.tree: + self.trees[full_name] = self.tree + self.tree = None + + def process_flow(self, flow, name): + self.tree.add(flow, self.opts.get("filter")) + + def process(self): + super().process(False) + + def stop_file(self, name, filename): + if self.tree: + self.trees[name] = self.tree + self.tree = None + + def print(self): + for name, tree in self.trees.items(): + self.ofconsole.console.print("\n") + self.ofconsole.console.print(file_header(name)) + + tree.build() + if self.style: + min_max = tree.min_max() + for field in self.heat_map: + min_val, max_val = min_max[field] + self.style.set_value_style( + field, heat_pallete(min_val, max_val) + ) + + self.print_tree(tree) + + def print_tree(self, tree): + root = Tree("Datapath Flows (logical)") + # Start by shoing recirc_id = 0 + for in_port in sorted(tree.recirc_nodes[0].keys()): + node = tree.recirc_nodes[0][in_port] + if node.visible: + self.print_recirc_node(root, node) + + self.ofconsole.console.print(root) + + def print_recirc_node(self, parent, node): + if self.ofconsole.style: + recirc_style = self.recirc_style_gen(hex(node.recirc)) + else: + recirc_style = None + + node_text = Text( + "[recirc_id({}) in_port({})]".format( + hex(node.recirc), node.in_port + ), + style=recirc_style, + ) + console_node = parent.add( + Panel.fit(node_text), guide_style=recirc_style + ) + + for block in node.visible_blocks(): + self.print_block(block, console_node) + + def print_block(self, block, parent): + # Print the flow matches and the statistics. + flow_text = [] + omit_first = { + "actions": "all", + } + omit_rest = { + "actions": "all", + "match": [kv.key for _, kv in block.equal_match], + } + for i, flow in enumerate(filter(lambda x: x.visible, block.flows)): + omit = omit_rest if i > 0 else omit_first + buf = ConsoleBuffer(Text()) + self.ofconsole.format_flow(buf, flow.flow, omitted=omit) + flow_text.append(buf.text) + + # Print the action associated with the block. + omit = { + "match": "all", + "info": "all", + "ufid": "all", + "dp_extra_info": "all", + } + act_buf = ConsoleBuffer(Text()) + act_buf.append_extra("actions: ", Style(bold=(self.style is not None))) + + self.ofconsole.format_flow(act_buf, block.flows[0].flow, omitted=omit) + + flows_node = parent.add( + Panel(Group(*flow_text)), guide_style=Style(color="default") + ) + action_node = flows_node.add( + Panel.fit( + act_buf.text, border_style="green" if self.style else "default" + ), + guide_style=Style(color="default"), + ) + + # Nested to the action, print the next recirc nodes. + for node in block.next_recirc_nodes: + if node.visible: + self.print_recirc_node(action_node, node) -- 2.46.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
