yongwww commented on code in PR #17053: URL: https://github.com/apache/tvm/pull/17053#discussion_r1631733451
########## python/tvm/relax/frontend/nnef/nnef_frontend.py: ########## @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF: Neural Network Exchange Format frontend for TVM relay""" +import os +import typing +import nnef +import numpy as np + +import tvm +from tvm import relax +from tvm.ir import IRModule +from tvm.relax import expr as tvm_expr + +from .nnef_ops import _get_converter_map + + +def get_type(elem_type: str): + """ + Gives numpy style type for nnef primitive types, uses x32 versions. + + :param elem_type: string, (scalar, integer, logical, string) + :return: returns numpy dtype equivalent (float32, int32, bool, string) + """ + if elem_type == "scalar": + return "float32" + if elem_type == "integer": + return "int32" + if elem_type == "logical": + return "bool" + if elem_type == "string": + return "string" + raise TypeError(f'Type "{elem_type}" is not implemented') + + +# Converter class +class NNEFConverter: + """ + Helper class for class level attributes, for conversion of NNEF model. + Public method to use is from_nnef. + + Parameters + ---------- + + keep_params_in_input : bool, optional + If this parameter is true, the nnef variables will be converted to + constants, and be embedded into the relay model, allowing optimizations + at compile time. + If False the params will have to be added as inputs, + the model can't load them automatically + + """ + + def __init__(self, keep_params_in_input=False): + self._nodes = {} + self._consts = {} + self._inputs = {} + self._num_inputs = 0 + self._params = {} + self._num_params = 0 + self._keep_params_in_input = keep_params_in_input + self._bb = relax.BlockBuilder() + + def from_nnef(self, graph: nnef.Graph) -> tvm.IRModule: + """ + Convert an NNEF model into an equivalent TVM Relay IRModule. + + Parameters + ---------- + graph : nnef.Graph + An NNEF Graph object that was imported with nnef.load_graph. + Shapes should be inferred by nnef.infer_shapes on graph beforehand. + + Returns + ------- + mod : tvm.IRModule + The relay module for compilation + + params : dict of str to tvm.nd.NDArray + The parameter dictionary to be used + + """ + with self._bb.function("main"): + with self._bb.dataflow(): + self._parse_inputs(graph) + self._construct_nodes(graph) + + outputs = [self._nodes[n] for n in graph.outputs] + outputs = outputs[0] if len(outputs) == 1 else tvm_expr.Tuple(outputs) + + output_var = self._bb.emit_output(outputs) + + func_attrs = {"num_input": self._num_inputs} + + input_list = [value for value in self._inputs.values() if isinstance(value, relax.Var)] + + if self._keep_params_in_input and self._params: + param_var_list, param_value_list = map(list, zip(*self._params.values())) + input_list.append(param_var_list) + func_attrs["params"] = param_value_list + + self._bb.emit_func_output(output_var, input_list) + + relax_mod = self._bb.get() + relax_mod["main"] = relax_mod["main"].with_attrs(func_attrs) + return relax_mod + + def _parse_inputs(self, graph): + """Save inputs into class from inputs attrib of graph""" + for inp in graph.inputs: + self._num_inputs += 1 + tensor = graph.tensors[inp] + self._nodes[inp] = self._new_var(inp, shape=tensor.shape, dtype=get_type(tensor.dtype)) + self._inputs[inp] = self._nodes[inp] + + def _construct_nodes(self, graph): + """Construct TVM relay calls from every operation of the nnef graph""" + for op in graph.operations: + if op.name == "external": + # externals are handled as input, not needed, + # but nnef treats them as operations as well + continue + + if op.name == "variable": + self._set_variable(graph.tensors[op.outputs["output"]]) + + elif op.name == "constant": + self._set_const(op) + + else: + # every other operator can be grouped more easily, + # as it does not need self for conversion + self._set_operator(op) + + def _set_operator(self, node): + self._set_literal_inputs(node) + inputs = [] + for ink, inv in node.inputs.items(): + if isinstance(inv, list): + for i, linv in enumerate(inv): + if linv in self._nodes.keys(): + inputs.append(self._nodes[linv]) + else: # handle literal inputs + name = f"{node.name}_{ink}_{i}" + assert name in self._nodes, f"{name} has not been properly handled" + inputs.append(self._nodes[name]) + + else: + if inv in self._nodes.keys(): + inputs.append(self._nodes[inv]) + else: # handle literal inputs + name = f"{node.name}_{ink}" + assert name in self._nodes, f"{name} has not been properly handled" + inputs.append(self._nodes[name]) + + converted = self._get_relay_op_call(node.name, inputs, node.attribs) + converted = self._bb.normalize(converted) + + if not isinstance(converted.struct_info, relax.TupleStructInfo): + outputs_num = 1 + else: + outputs_num = len(converted.struct_info.fields) + + if outputs_num == 1: + # check if the singular ret val is a list of only one element + ret_val = list(node.outputs.values())[0] + if isinstance(ret_val, list): + self._nodes[ret_val[0]] = converted + else: + self._nodes[ret_val] = converted + else: + for i, out in zip(range(outputs_num), node.outputs["values"]): + self._nodes[out] = converted[i] + + def _set_const(self, node): + """Create a tvm.relay.Constant from a nnef constant tensor""" + name = node.outputs["output"] + data = node.attribs["value"] + shape = node.attribs["shape"] + if len(data) == 1: + data = np.full(shape, data, dtype=get_type(node.dtype)) + else: + data = np.array(data, dtype=get_type(node.dtype)) + self._consts[name] = tvm_expr.const(data) + self._nodes[name] = self._consts[name] + + def _set_variable(self, tensor): + """Create a tvm.relay.Var (or Constant) from a nnef variable tensor""" + tens_data = tensor.data + if not self._keep_params_in_input: + self._consts[tensor.name] = tvm_expr.const(tens_data) + self._nodes[tensor.name] = self._consts[tensor.name] + else: + var = self._new_var(tensor.name, shape=tensor.shape, dtype=get_type(tensor.dtype)) + self._nodes[tensor.name] = var + self._params[tensor.name] = (var, tvm.nd.array(tens_data)) + + def _set_literal_inputs(self, node): + """Checks if node has literal inputs and saves them into a tvm.relay.Constant. + naming as {node.name}_{input field name}""" + for field_name, value in node.inputs.items(): + if isinstance(value, list): + for v in value: + if v not in self._nodes.keys(): + self._nodes[f"{node.name}_{v}"] = tvm_expr.const(v) + + else: + if value not in self._nodes.keys(): + self._nodes[f"{node.name}_{field_name}"] = tvm_expr.const(value) + + def _get_relay_op_call(self, name, inputs, attrs): + """Returns the tvm.Call equivalent to the nnef operator""" + conv_map = _get_converter_map() + if name in conv_map: + + call = conv_map[name](self._bb, *inputs, **attrs) + else: + # This error is reached if NNEF is expanded with additional ops + raise NotImplementedError( + f"Operator {name} is not implemented, as {name} has been added after 1.0.5." + ) + return call + + def _infer_type(self, val): + if isinstance(val, bool): + return "bool", True + if isinstance(val, float): + return "float32", True + if isinstance(val, int): + return "int32", True + if isinstance(val, str): + # the string vals can be names of nodes in some of the cases + if isinstance(val, nnef.Identifier): + if val in self._nodes.keys(): + node = self._nodes[val] + if isinstance(node, tvm_expr.Var): + return node.type_annotation.dtype, False + if isinstance(node, tvm_expr.Constant): + return node.data.dtype, False + if isinstance(node, tvm_expr.Call): + return node.checked_type.dtype, False + raise Exception( + f"{val} has not been loaded into the model " + "but it should have been, as a var or call." + ) + return "string", True + + raise TypeError(f'Value "{val}" is not a recognized type') + + def _new_var(self, name, shape, dtype="float32"): + return relax.Var( + name_hint=name, + struct_info=relax.TensorStructInfo(shape=shape, dtype=dtype), + ) + + +def from_nnef( + model: typing.Union[str, os.PathLike, nnef.Graph], + keep_params_in_input: bool = False, +) -> IRModule: + """ + Convert an NNEF model into an equivalent TVM Relay IRModule. Review Comment: relax ########## python/tvm/relax/frontend/nnef/nnef_frontend.py: ########## @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF: Neural Network Exchange Format frontend for TVM relay""" Review Comment: typo: Relay here should be Relax ########## python/tvm/relax/frontend/nnef/nnef_frontend.py: ########## @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF: Neural Network Exchange Format frontend for TVM relay""" +import os +import typing +import nnef +import numpy as np + +import tvm +from tvm import relax +from tvm.ir import IRModule +from tvm.relax import expr as tvm_expr + +from .nnef_ops import _get_converter_map + + +def get_type(elem_type: str): + """ + Gives numpy style type for nnef primitive types, uses x32 versions. + + :param elem_type: string, (scalar, integer, logical, string) + :return: returns numpy dtype equivalent (float32, int32, bool, string) + """ + if elem_type == "scalar": + return "float32" + if elem_type == "integer": + return "int32" + if elem_type == "logical": + return "bool" + if elem_type == "string": + return "string" + raise TypeError(f'Type "{elem_type}" is not implemented') + + +# Converter class +class NNEFConverter: + """ + Helper class for class level attributes, for conversion of NNEF model. + Public method to use is from_nnef. + + Parameters + ---------- + + keep_params_in_input : bool, optional + If this parameter is true, the nnef variables will be converted to + constants, and be embedded into the relay model, allowing optimizations + at compile time. + If False the params will have to be added as inputs, + the model can't load them automatically + + """ + + def __init__(self, keep_params_in_input=False): + self._nodes = {} + self._consts = {} + self._inputs = {} + self._num_inputs = 0 + self._params = {} + self._num_params = 0 + self._keep_params_in_input = keep_params_in_input + self._bb = relax.BlockBuilder() + + def from_nnef(self, graph: nnef.Graph) -> tvm.IRModule: + """ + Convert an NNEF model into an equivalent TVM Relay IRModule. Review Comment: typo: Relay to Relax, several similar typo in the frontend ########## python/tvm/relax/frontend/nnef/nnef_ops.py: ########## @@ -0,0 +1,1957 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF frontend converter helper funcs and ops""" +import math + +import itertools +from functools import reduce + +import numpy as np + +import tvm +from tvm import relax +from tvm.relax import expr as tvm_expr +from tvm.relax import op as tvm_op +from tvm import topi + + +# Base methods + + +def dimension_picker(prefix, kernel_shape, suffix=""): + """ + Returns the correct name for nth dimensional operator. Uses the "kernel_shape" attribute.\n + E.g.call: dimension_picker(op_name)(attr) + + :param prefix: the name of the operator (e.g. conv) + :param kernel_shape: shape of the tensor to fit the operation + :param suffix: optional suffix for ops + :return: "prefix`n`d" where n is the correct dimension for the kernel + """ + + rank = len(kernel_shape[2:]) + if rank == 1: + return prefix + "1d" + suffix + if rank == 2: + return prefix + "2d" + suffix + if rank == 3: + return prefix + "3d" + suffix + op_name = prefix + "1d/2d/3d" + msg = f"Only 1D, 2D, and 3D kernels are supported for operator {op_name}." + raise tvm.error.OpAttributeInvalid(msg) + + +def _size_conv(size, rank): + # window of size (DH)W is only possible when it is checked outside, + # which is needed for alternative solution + if rank == 3: + if len(size) == 1: + return size + if len(size) == 3: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2] + if rank == 4: + if len(size) == 2: + return size + if len(size) == 4: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2:] + if rank == 5: + if len(size) == 3: + return size + if len(size) == 5: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2:] + + raise ValueError(f"Unexpected window size, got {len(size)}") + + +def _stride_conv(stride, rank): + if rank == 3: + # {conv style} :: [s] -> [s] + if len(stride) == 1: + return stride + # {pool style} :: [N, C, s] -> asrt N,C == 1; [s] + if len(stride) == 3: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + if rank == 4: + # {conv style} :: [sh, sw] -> [sh, sw] + if len(stride) == 2: + return stride + # {pool style} :: [N, C, sh, sw] -> asrt N,C == 1; [sh, sw] + if len(stride) == 4: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + if rank == 5: + # {conv style} :: [sd, sh, sw] -> [sd, sh, sw] + if len(stride) == 3: + return stride + # {pool style} :: [N, C, sd, sh, sw] -> asrt N,C == 1; [sd, sh, sw] + if len(stride) == 5: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + raise ValueError(f"Unexpected stride in {rank - 2}D, got {len(stride)}: {stride}") + + +def _padding_conv(padding, rank, keepdims=False): + if isinstance(padding[0], (tuple, list)): + # 1D + if rank == 3: + # {conv style} :: [(l,r)] -> (l,r) + if len(padding) == 1: + return padding[0] + if len(padding) == 3: + # {pool style} :: [(batch),(channel),(l,r)] -> asrt N,C == 0, (l,r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + return padding[2] + # {sliding window style} :: [(batch),(channel),(l,r)] -> [(batch),(channel),(l,r)] + else: + return padding + + # 2D + + if rank == 4: + # {conv style} :: [(u,d),(l,r)] -> (u, l, d, r) + if len(padding) == 2: + # change UDLR to ULDR padding, LC is faster here + return [x[i] for i in [0, 1] for x in padding] + + if len(padding) == 4: + # {pool style} :: [(batch size),(channel),(u,d),(l,r)] -> + # -> asrt N,C == 0, (u, l, d, r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + # itertools is faster than LC (slicing) + return list(itertools.chain.from_iterable(zip(padding[2], padding[3]))) + # {sliding window style} :: [(batch),(channel),(u,d),(l,r)] -> + # -> [(batch),(channel),(u,d),(l,r)] + else: + return padding + + # 3D + + if rank == 5: + # {conv style} :: [(f,b),(u,d),(l,r)] -> (f, u, l, b, d, r) + if len(padding) == 3: + # LC is faster + return [x[i] for i in [0, 1] for x in padding] + + if len(padding) == 5: + # {pool style} :: [(batch size),(channel),(f,b)(u,p),(l,r)] -> + # -> asrt N,C == 0, (f, u, l, b, d, r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + # itertools faster barely + return list( + itertools.chain.from_iterable(zip(padding[2], padding[3], padding[4])) + ) + # {s-w style} :: [(batch),(channel),(f,b),(u,d),(l,r)] -> + # -> [(batch),(channel),(f,b),(u,d),(l,r)] + else: + return padding + + raise ValueError( + f"Incorrect padding style for {rank - 2}D operand. Only length of {rank - 2}, {rank} " + f"supported, got {len(padding)}: {padding}" + ) + + raise ValueError("nnef should not have singular padding") + + +def _calculate_nnef_padding(active_shape, strides, kernel_shape, dilation): + """Ordering of nnef autopad and tvm autopad are sometimes different, + this method calculates nnef like padding from dimensions + + Parameters + ---------- + active_shape + the data dimensions + strides + the strides over the active dimensions + kernel_shape + the shape of the window, must have the same rank as active shape + dilation + the dilations over the active dimensions + """ + output = [(ui + (s - 1)) // s for ui, s in zip(active_shape, strides)] + dilated = [(f - 1) * d + 1 for f, d in zip(kernel_shape, dilation)] + total = [ + max(0, (di - 1) * s + df - ui) + for di, s, df, ui in zip(output, strides, dilated, active_shape) + ] + padding = [(pad // 2, (pad + 1) // 2) for pad in total] + return padding + + +def _calculate_nnef_padding_deconv(data_sh, strides, kernel_active_sh, dilation, output_shape): + out_sh = output_shape[2:] if output_shape else [ui * s for ui, s in zip(data_sh, strides)] + dilated = [(f - 1) * d + 1 for f, d in zip(kernel_active_sh[2:], dilation)] + total = [ + max(0, (di - 1) * s + df - ui) for di, s, df, ui in zip(data_sh, strides, dilated, out_sh) + ] + return total, out_sh + + +def __unexpected_attrs(op, kwargs): + raise NotImplementedError( + f"{op} received unexpected attributes(s), possibly mismatched versions. " + "Attributes(s) ignored: " + ", ".join(f"{k} := {v}" for k, v in kwargs.items()) + ) + + +# Conversion map, operator functions + + +def _get_converter_map(): + return { # Unary + "copy": copy_converter, # arithmetic + "neg": neg_converter, + "rcp": rcp_converter, + "exp": exp_converter, + "log": log_converter, + "sin": sin_converter, + "cos": cos_converter, + "tan": tan_converter, + "sinh": sinh_converter, + "cosh": cosh_converter, + "tanh": tanh_converter, + "asin": asin_converter, + "acos": acos_converter, + "atan": atan_converter, + "asinh": asinh_converter, + "acosh": acosh_converter, + "atanh": atanh_converter, + "abs": abs_converter, + "sign": sign_converter, + "not": not_converter, # logical + "floor": floor_converter, # rounding + "ceil": ceil_converter, + "round": round_converter, + # Binary + "add": add_converter, # arithmetic + "sub": sub_converter, + "mul": mul_converter, + "div": div_converter, + "pow": pow_converter, + "lt": lt_converter, # comparison + "gt": gt_converter, + "le": le_converter, + "ge": ge_converter, + "eq": eq_converter, + "ne": ne_converter, + "and": and_converter, # logical + "or": or_converter, + # select + "select": select_converter, + # simplifier + "sqr": sqr_converter, + "sqrt": sqrt_converter, + "rsqr": rsqr_converter, + "rsqrt": rsqrt_converter, + "log2": log2_converter, + "min": min_converter, + "max": max_converter, + "clamp": clamp_converter, + # sliding-window + "conv": conv_converter, + "deconv": deconv_converter, + "box": box_converter, + "debox": debox_converter, + "argmax_pool": ndop, + "sample": ndop, + "desample": ndop, + "nearest_downsample": nearest_downsample_converter, + "area_downsample": area_downsample_converter, + "nearest_upsample": nearest_upsample_converter, + "multilinear_upsample": multilinear_upsample_converter, + # reduce + "sum_reduce": sum_reduce_converter, + "max_reduce": max_reduce_converter, + "min_reduce": min_reduce_converter, + "argmax_reduce": argmax_reduce_converter, + "argmin_reduce": argmin_reduce_converter, + "all_reduce": all_reduce_converter, + "any_reduce": any_reduce_converter, + "mean_reduce": mean_reduce_converter, + # tensor shape + "reshape": reshape_converter, + "squeeze": squeeze_converter, + "unsqueeze": unsqueeze_converter, + "transpose": transpose_converter, + "split": split_converter, + "concat": concat_converter, + "stack": stack_converter, + "unstack": unstack_converter, + "slice": slice_converter, + "pad": pad_converter, + "tile": tile_converter, + # region-of-interest - not needed - not supported + "avg_roi_pool": ndop, + "max_roi_pool": ndop, + "roi_resample": ndop, + "avg_roi_align": ndop, + "max_roi_align": ndop, + # matrix multiplication + "matmul": matmul_converter, + # variables + "update": ndop, # --- not used + # Compound + "sigmoid": sigmoid_converter, # activation + "relu": relu_converter, + "prelu": prelu_converter, + "leaky_relu": leaky_relu_converter, + "elu": elu_converter, + "selu": selu_converter, + "gelu": gelu_converter, + "silu": silu_converter, + "softmax": softmax_converter, + "softplus": softplus_converter, + "linear": linear_converter, # linear + "separable_conv": separable_conv_converter, + "separable_deconv": separable_deconv_converter, + "max_pool_with_index": ndop, # pooling + "max_pool": max_pool_converter, + "avg_pool": avg_pool_converter, + "rms_pool": rms_pool_converter, + "local_response_normalization": local_response_normalization_converter, # normalization + "local_mean_normalization": local_mean_normalization_converter, + "local_variance_normalization": local_variance_normalization_converter, + "local_contrast_normalization": local_contrast_normalization_converter, + "l1_normalization": l1_normalization_converter, + "l2_normalization": l2_normalization_converter, + "batch_normalization": batch_normalization_converter, + "min_max_linear_quantize": ndop, # quantization + "zero_point_linear_quantize": ndop, + "linear_quantize": ndop, + "logarithmic_quantize": ndop, + # MISC + "copy_n": ndop, + "add_n": ndop, + "moments": ndop, + } + + +# pylint: disable=unused-argument + +# not implemented ops +def ndop(*args, **kwargs): + # print(args, kwargs) Review Comment: remove the print here ########## python/tvm/relax/frontend/nnef/nnef_frontend.py: ########## @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF: Neural Network Exchange Format frontend for TVM relay""" +import os +import typing +import nnef +import numpy as np + +import tvm +from tvm import relax +from tvm.ir import IRModule +from tvm.relax import expr as tvm_expr + +from .nnef_ops import _get_converter_map + + +def get_type(elem_type: str): + """ + Gives numpy style type for nnef primitive types, uses x32 versions. + + :param elem_type: string, (scalar, integer, logical, string) + :return: returns numpy dtype equivalent (float32, int32, bool, string) + """ + if elem_type == "scalar": + return "float32" + if elem_type == "integer": + return "int32" + if elem_type == "logical": + return "bool" + if elem_type == "string": + return "string" + raise TypeError(f'Type "{elem_type}" is not implemented') + + +# Converter class +class NNEFConverter: + """ + Helper class for class level attributes, for conversion of NNEF model. + Public method to use is from_nnef. + + Parameters + ---------- + + keep_params_in_input : bool, optional + If this parameter is true, the nnef variables will be converted to + constants, and be embedded into the relay model, allowing optimizations + at compile time. + If False the params will have to be added as inputs, + the model can't load them automatically + + """ + + def __init__(self, keep_params_in_input=False): + self._nodes = {} + self._consts = {} + self._inputs = {} + self._num_inputs = 0 + self._params = {} + self._num_params = 0 + self._keep_params_in_input = keep_params_in_input + self._bb = relax.BlockBuilder() + + def from_nnef(self, graph: nnef.Graph) -> tvm.IRModule: + """ + Convert an NNEF model into an equivalent TVM Relay IRModule. + + Parameters + ---------- + graph : nnef.Graph + An NNEF Graph object that was imported with nnef.load_graph. + Shapes should be inferred by nnef.infer_shapes on graph beforehand. + + Returns + ------- + mod : tvm.IRModule + The relay module for compilation + + params : dict of str to tvm.nd.NDArray + The parameter dictionary to be used + + """ + with self._bb.function("main"): + with self._bb.dataflow(): + self._parse_inputs(graph) + self._construct_nodes(graph) + + outputs = [self._nodes[n] for n in graph.outputs] + outputs = outputs[0] if len(outputs) == 1 else tvm_expr.Tuple(outputs) + + output_var = self._bb.emit_output(outputs) + + func_attrs = {"num_input": self._num_inputs} + + input_list = [value for value in self._inputs.values() if isinstance(value, relax.Var)] + + if self._keep_params_in_input and self._params: + param_var_list, param_value_list = map(list, zip(*self._params.values())) + input_list.append(param_var_list) + func_attrs["params"] = param_value_list + + self._bb.emit_func_output(output_var, input_list) + + relax_mod = self._bb.get() + relax_mod["main"] = relax_mod["main"].with_attrs(func_attrs) + return relax_mod + + def _parse_inputs(self, graph): + """Save inputs into class from inputs attrib of graph""" + for inp in graph.inputs: + self._num_inputs += 1 + tensor = graph.tensors[inp] + self._nodes[inp] = self._new_var(inp, shape=tensor.shape, dtype=get_type(tensor.dtype)) + self._inputs[inp] = self._nodes[inp] + + def _construct_nodes(self, graph): + """Construct TVM relay calls from every operation of the nnef graph""" + for op in graph.operations: + if op.name == "external": + # externals are handled as input, not needed, + # but nnef treats them as operations as well + continue + + if op.name == "variable": + self._set_variable(graph.tensors[op.outputs["output"]]) + + elif op.name == "constant": + self._set_const(op) + + else: + # every other operator can be grouped more easily, + # as it does not need self for conversion + self._set_operator(op) + + def _set_operator(self, node): + self._set_literal_inputs(node) + inputs = [] + for ink, inv in node.inputs.items(): + if isinstance(inv, list): + for i, linv in enumerate(inv): + if linv in self._nodes.keys(): + inputs.append(self._nodes[linv]) + else: # handle literal inputs + name = f"{node.name}_{ink}_{i}" + assert name in self._nodes, f"{name} has not been properly handled" + inputs.append(self._nodes[name]) + + else: + if inv in self._nodes.keys(): + inputs.append(self._nodes[inv]) + else: # handle literal inputs + name = f"{node.name}_{ink}" + assert name in self._nodes, f"{name} has not been properly handled" + inputs.append(self._nodes[name]) + + converted = self._get_relay_op_call(node.name, inputs, node.attribs) + converted = self._bb.normalize(converted) + + if not isinstance(converted.struct_info, relax.TupleStructInfo): + outputs_num = 1 + else: + outputs_num = len(converted.struct_info.fields) + + if outputs_num == 1: + # check if the singular ret val is a list of only one element + ret_val = list(node.outputs.values())[0] + if isinstance(ret_val, list): + self._nodes[ret_val[0]] = converted + else: + self._nodes[ret_val] = converted + else: + for i, out in zip(range(outputs_num), node.outputs["values"]): + self._nodes[out] = converted[i] + + def _set_const(self, node): + """Create a tvm.relay.Constant from a nnef constant tensor""" + name = node.outputs["output"] + data = node.attribs["value"] + shape = node.attribs["shape"] + if len(data) == 1: + data = np.full(shape, data, dtype=get_type(node.dtype)) + else: + data = np.array(data, dtype=get_type(node.dtype)) + self._consts[name] = tvm_expr.const(data) + self._nodes[name] = self._consts[name] + + def _set_variable(self, tensor): + """Create a tvm.relay.Var (or Constant) from a nnef variable tensor""" + tens_data = tensor.data + if not self._keep_params_in_input: + self._consts[tensor.name] = tvm_expr.const(tens_data) + self._nodes[tensor.name] = self._consts[tensor.name] + else: + var = self._new_var(tensor.name, shape=tensor.shape, dtype=get_type(tensor.dtype)) + self._nodes[tensor.name] = var + self._params[tensor.name] = (var, tvm.nd.array(tens_data)) + + def _set_literal_inputs(self, node): + """Checks if node has literal inputs and saves them into a tvm.relay.Constant. + naming as {node.name}_{input field name}""" + for field_name, value in node.inputs.items(): + if isinstance(value, list): + for v in value: + if v not in self._nodes.keys(): + self._nodes[f"{node.name}_{v}"] = tvm_expr.const(v) + + else: + if value not in self._nodes.keys(): + self._nodes[f"{node.name}_{field_name}"] = tvm_expr.const(value) + + def _get_relay_op_call(self, name, inputs, attrs): Review Comment: relax ########## tests/python/relax/test_frontend_nnef.py: ########## @@ -0,0 +1,3142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np + +import _nnef +import nnef + +import tvm +import tvm.testing +from tvm import relax +import tvm.relax.frontend.nnef + +from tvm.script import ir as I +from tvm.script import relax as R +from tvm.script import tir as T +import tvm.topi as topi + +from ..nightly.frontend.nnef import cases_string + + +def get_case_graph(name): + if "-" in name: + name = name.replace("-", "_") + return nnef.parse_string(getattr(cases_string, name)) + + +def verify_model_struct(model_name, binding, expected): + graph = get_case_graph(model_name) + for operation in graph.operations: + if operation.name == "variable": + tensor_name = operation.outputs["output"] + + shape = operation.attribs["shape"] + + assert ( + operation.dtype == "scalar" + ), f"variable of type {operation.dtype} is not supported, please update verify_model" + + data = np.ones(shape).astype("float32") + + tensor = graph.tensors[tensor_name] + graph.tensors[tensor_name] = _nnef.Tensor( + tensor.name, tensor.dtype, shape, data, tensor.quantization + ) + + binding = {k: tvm.nd.array(v) for k, v in binding.items()} + expected = relax.transform.BindParams("main", binding)(expected) + + mod = relax.frontend.nnef.from_nnef(graph) + tvm.ir.assert_structural_equal(mod, expected) + + +def get_unary_mod(method, dt="float32", o_dtype=None): + global dtype + dtype = dt + if not o_dtype: + o_dtype = dtype + + def _appl_shape(sh, osh=None): + global shape, o_shape + if not osh: + osh = sh + shape, o_shape = sh, osh + + @tvm.script.ir.ir_module + class expected: + @R.function + def main(in1: R.Tensor(shape, dtype)) -> R.Tensor(o_shape, o_dtype): + R.func_attr({"num_input": 1}) + with R.dataflow(): + lv: R.Tensor(o_shape, dtype=o_dtype) = method(in1) + R.output(lv) + return lv + + return expected + + return _appl_shape + + +def get_binary_mod(method, dt="float32", o_dtype=None): + global dtype + dtype = dt + if not o_dtype: + o_dtype = dtype + + def _appl_shape(sh1, sh2=None, osh=None): + global shape1, shape2, o_shape + if not sh2: + sh2 = sh1 + if not osh: + osh = sh1 + shape1, shape2, o_shape = sh1, sh2, osh + + @tvm.script.ir.ir_module + class expected: + @R.function + def main( + lhs: R.Tensor(shape1, dtype), rhs: R.Tensor(shape2, dtype) + ) -> R.Tensor(o_shape, o_dtype): + R.func_attr({"num_input": 2}) + with R.dataflow(): + lv: R.Tensor(o_shape, dtype=o_dtype) = method(lhs, rhs) + R.output(lv) + return lv + + return expected + + return _appl_shape + + +# graph tests +def test_copy(): + @I.ir_module + class expected_2d: + @R.function + def main(input: R.Tensor((4, 16), dtype="float32")) -> R.Tensor((4, 16), dtype="float32"): + R.func_attr({"num_input": 1}) + with R.dataflow(): + lv = R.emit_te(topi.identity, input) + gv: R.Tensor((4, 16), dtype="float32") = lv + R.output(gv) + return gv + + verify_model_struct("copy_2d", {}, expected_2d) Review Comment: i feel we can pass the nnef op, dimension, dtype as argument instead of a str like `copy_2d`. It could be like `verify_model_struct(op=nnef.copy, dim="2", dtype="float32", {}, expected_2d)`, for some cases, we can also use relax blockbuilder to generate the expected IR as well ########## python/tvm/relax/frontend/nnef/nnef_ops.py: ########## @@ -0,0 +1,1957 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NNEF frontend converter helper funcs and ops""" +import math + +import itertools +from functools import reduce + +import numpy as np + +import tvm +from tvm import relax +from tvm.relax import expr as tvm_expr +from tvm.relax import op as tvm_op +from tvm import topi + + +# Base methods + + +def dimension_picker(prefix, kernel_shape, suffix=""): + """ + Returns the correct name for nth dimensional operator. Uses the "kernel_shape" attribute.\n + E.g.call: dimension_picker(op_name)(attr) + + :param prefix: the name of the operator (e.g. conv) + :param kernel_shape: shape of the tensor to fit the operation + :param suffix: optional suffix for ops + :return: "prefix`n`d" where n is the correct dimension for the kernel + """ + + rank = len(kernel_shape[2:]) + if rank == 1: + return prefix + "1d" + suffix + if rank == 2: + return prefix + "2d" + suffix + if rank == 3: + return prefix + "3d" + suffix + op_name = prefix + "1d/2d/3d" + msg = f"Only 1D, 2D, and 3D kernels are supported for operator {op_name}." + raise tvm.error.OpAttributeInvalid(msg) + + +def _size_conv(size, rank): + # window of size (DH)W is only possible when it is checked outside, + # which is needed for alternative solution + if rank == 3: + if len(size) == 1: + return size + if len(size) == 3: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2] + if rank == 4: + if len(size) == 2: + return size + if len(size) == 4: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2:] + if rank == 5: + if len(size) == 3: + return size + if len(size) == 5: + assert ( + size[0] == 1 and size[1] == 1 + ), "Incorrect window dimensions, first two dimensions must be 1" + return size[2:] + + raise ValueError(f"Unexpected window size, got {len(size)}") + + +def _stride_conv(stride, rank): + if rank == 3: + # {conv style} :: [s] -> [s] + if len(stride) == 1: + return stride + # {pool style} :: [N, C, s] -> asrt N,C == 1; [s] + if len(stride) == 3: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + if rank == 4: + # {conv style} :: [sh, sw] -> [sh, sw] + if len(stride) == 2: + return stride + # {pool style} :: [N, C, sh, sw] -> asrt N,C == 1; [sh, sw] + if len(stride) == 4: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + if rank == 5: + # {conv style} :: [sd, sh, sw] -> [sd, sh, sw] + if len(stride) == 3: + return stride + # {pool style} :: [N, C, sd, sh, sw] -> asrt N,C == 1; [sd, sh, sw] + if len(stride) == 5: + assert ( + stride[0] == 1 and stride[1] == 1 + ), "Not supported stride dimensions, first two dimensions must be 1" + return stride[2:] + raise ValueError(f"Unexpected stride in {rank - 2}D, got {len(stride)}: {stride}") + + +def _padding_conv(padding, rank, keepdims=False): + if isinstance(padding[0], (tuple, list)): + # 1D + if rank == 3: + # {conv style} :: [(l,r)] -> (l,r) + if len(padding) == 1: + return padding[0] + if len(padding) == 3: + # {pool style} :: [(batch),(channel),(l,r)] -> asrt N,C == 0, (l,r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + return padding[2] + # {sliding window style} :: [(batch),(channel),(l,r)] -> [(batch),(channel),(l,r)] + else: + return padding + + # 2D + + if rank == 4: + # {conv style} :: [(u,d),(l,r)] -> (u, l, d, r) + if len(padding) == 2: + # change UDLR to ULDR padding, LC is faster here + return [x[i] for i in [0, 1] for x in padding] + + if len(padding) == 4: + # {pool style} :: [(batch size),(channel),(u,d),(l,r)] -> + # -> asrt N,C == 0, (u, l, d, r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + # itertools is faster than LC (slicing) + return list(itertools.chain.from_iterable(zip(padding[2], padding[3]))) + # {sliding window style} :: [(batch),(channel),(u,d),(l,r)] -> + # -> [(batch),(channel),(u,d),(l,r)] + else: + return padding + + # 3D + + if rank == 5: + # {conv style} :: [(f,b),(u,d),(l,r)] -> (f, u, l, b, d, r) + if len(padding) == 3: + # LC is faster + return [x[i] for i in [0, 1] for x in padding] + + if len(padding) == 5: + # {pool style} :: [(batch size),(channel),(f,b)(u,p),(l,r)] -> + # -> asrt N,C == 0, (f, u, l, b, d, r) + if not keepdims: + assert padding[0] == (0, 0) and padding[1] == (0, 0), ( + "Incorrect padding. " "Padding on C,I dimensions not supported" + ) + # itertools faster barely + return list( + itertools.chain.from_iterable(zip(padding[2], padding[3], padding[4])) + ) + # {s-w style} :: [(batch),(channel),(f,b),(u,d),(l,r)] -> + # -> [(batch),(channel),(f,b),(u,d),(l,r)] + else: + return padding + + raise ValueError( + f"Incorrect padding style for {rank - 2}D operand. Only length of {rank - 2}, {rank} " + f"supported, got {len(padding)}: {padding}" + ) + + raise ValueError("nnef should not have singular padding") + + +def _calculate_nnef_padding(active_shape, strides, kernel_shape, dilation): + """Ordering of nnef autopad and tvm autopad are sometimes different, + this method calculates nnef like padding from dimensions + + Parameters + ---------- + active_shape + the data dimensions + strides + the strides over the active dimensions + kernel_shape + the shape of the window, must have the same rank as active shape + dilation + the dilations over the active dimensions + """ + output = [(ui + (s - 1)) // s for ui, s in zip(active_shape, strides)] + dilated = [(f - 1) * d + 1 for f, d in zip(kernel_shape, dilation)] + total = [ + max(0, (di - 1) * s + df - ui) + for di, s, df, ui in zip(output, strides, dilated, active_shape) + ] + padding = [(pad // 2, (pad + 1) // 2) for pad in total] + return padding + + +def _calculate_nnef_padding_deconv(data_sh, strides, kernel_active_sh, dilation, output_shape): + out_sh = output_shape[2:] if output_shape else [ui * s for ui, s in zip(data_sh, strides)] + dilated = [(f - 1) * d + 1 for f, d in zip(kernel_active_sh[2:], dilation)] + total = [ + max(0, (di - 1) * s + df - ui) for di, s, df, ui in zip(data_sh, strides, dilated, out_sh) + ] + return total, out_sh + + +def __unexpected_attrs(op, kwargs): + raise NotImplementedError( + f"{op} received unexpected attributes(s), possibly mismatched versions. " + "Attributes(s) ignored: " + ", ".join(f"{k} := {v}" for k, v in kwargs.items()) + ) + + +# Conversion map, operator functions + + +def _get_converter_map(): + return { # Unary + "copy": copy_converter, # arithmetic + "neg": neg_converter, + "rcp": rcp_converter, + "exp": exp_converter, + "log": log_converter, + "sin": sin_converter, + "cos": cos_converter, + "tan": tan_converter, + "sinh": sinh_converter, + "cosh": cosh_converter, + "tanh": tanh_converter, + "asin": asin_converter, + "acos": acos_converter, + "atan": atan_converter, + "asinh": asinh_converter, + "acosh": acosh_converter, + "atanh": atanh_converter, + "abs": abs_converter, + "sign": sign_converter, + "not": not_converter, # logical + "floor": floor_converter, # rounding + "ceil": ceil_converter, + "round": round_converter, + # Binary + "add": add_converter, # arithmetic + "sub": sub_converter, + "mul": mul_converter, + "div": div_converter, + "pow": pow_converter, + "lt": lt_converter, # comparison + "gt": gt_converter, + "le": le_converter, + "ge": ge_converter, + "eq": eq_converter, + "ne": ne_converter, + "and": and_converter, # logical + "or": or_converter, + # select + "select": select_converter, + # simplifier + "sqr": sqr_converter, + "sqrt": sqrt_converter, + "rsqr": rsqr_converter, + "rsqrt": rsqrt_converter, + "log2": log2_converter, + "min": min_converter, + "max": max_converter, + "clamp": clamp_converter, + # sliding-window + "conv": conv_converter, + "deconv": deconv_converter, + "box": box_converter, + "debox": debox_converter, + "argmax_pool": ndop, + "sample": ndop, + "desample": ndop, + "nearest_downsample": nearest_downsample_converter, + "area_downsample": area_downsample_converter, + "nearest_upsample": nearest_upsample_converter, + "multilinear_upsample": multilinear_upsample_converter, + # reduce + "sum_reduce": sum_reduce_converter, + "max_reduce": max_reduce_converter, + "min_reduce": min_reduce_converter, + "argmax_reduce": argmax_reduce_converter, + "argmin_reduce": argmin_reduce_converter, + "all_reduce": all_reduce_converter, + "any_reduce": any_reduce_converter, + "mean_reduce": mean_reduce_converter, + # tensor shape + "reshape": reshape_converter, + "squeeze": squeeze_converter, + "unsqueeze": unsqueeze_converter, + "transpose": transpose_converter, + "split": split_converter, + "concat": concat_converter, + "stack": stack_converter, + "unstack": unstack_converter, + "slice": slice_converter, + "pad": pad_converter, + "tile": tile_converter, + # region-of-interest - not needed - not supported + "avg_roi_pool": ndop, + "max_roi_pool": ndop, + "roi_resample": ndop, + "avg_roi_align": ndop, + "max_roi_align": ndop, + # matrix multiplication + "matmul": matmul_converter, + # variables + "update": ndop, # --- not used + # Compound + "sigmoid": sigmoid_converter, # activation + "relu": relu_converter, + "prelu": prelu_converter, + "leaky_relu": leaky_relu_converter, + "elu": elu_converter, + "selu": selu_converter, + "gelu": gelu_converter, + "silu": silu_converter, + "softmax": softmax_converter, + "softplus": softplus_converter, + "linear": linear_converter, # linear + "separable_conv": separable_conv_converter, + "separable_deconv": separable_deconv_converter, + "max_pool_with_index": ndop, # pooling + "max_pool": max_pool_converter, + "avg_pool": avg_pool_converter, + "rms_pool": rms_pool_converter, + "local_response_normalization": local_response_normalization_converter, # normalization + "local_mean_normalization": local_mean_normalization_converter, + "local_variance_normalization": local_variance_normalization_converter, + "local_contrast_normalization": local_contrast_normalization_converter, + "l1_normalization": l1_normalization_converter, + "l2_normalization": l2_normalization_converter, + "batch_normalization": batch_normalization_converter, + "min_max_linear_quantize": ndop, # quantization + "zero_point_linear_quantize": ndop, + "linear_quantize": ndop, + "logarithmic_quantize": ndop, + # MISC + "copy_n": ndop, + "add_n": ndop, + "moments": ndop, + } + + +# pylint: disable=unused-argument + +# not implemented ops +def ndop(*args, **kwargs): + # print(args, kwargs) + raise Exception("Not supported operator was called, please check for compatibility") + + +# # Unary ops Review Comment: seems no need to have this comment -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
