manupa-arm commented on a change in pull request #8795: URL: https://github.com/apache/tvm/pull/8795#discussion_r695438323
########## File path: python/tvm/relay/backend/contrib/ethosu/vela_api.py ########## @@ -0,0 +1,314 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +conversions between TVM and Vela. Therefore, all interactions with the +Vela API are supposed to go through this adapter, with the hope that +any changes to Vela API, TVM only needs to change this file. +The following conversion APIs are added : + *Obtaining the best block config + *Compressing weights + *Packing biases +""" +import logging +import math +import numpy as np +from ethosu.vela import api as vapi + +from tvm.relay.backend.contrib.ethosu import util + +# pylint: disable=invalid-name +logger = logging.getLogger("Ethos-U") + +VELA_TO_NP_DTYPES = { + vapi.NpuDataType.UINT8: np.uint8, + vapi.NpuDataType.UINT16: np.uint16, + vapi.NpuDataType.INT8: np.int8, + vapi.NpuDataType.INT16: np.int16, + vapi.NpuDataType.INT32: np.int32, +} + +SCALE_BIAS_LENGTH = 10 + + +def get_optimal_block_config(npu_op, accel_type): + """ + "The NPU's unit of work is known as a block. It will fetch block(s) from Input + Feature Map (IFM) and a compute block for Output Feature Map (OFM). + Therefore, we need to pick an optimal block configuration considering bandwidth + to bring IFM blocks and the number of OFM block computes need to happen + to cover the OFM as indicated by the npu op. + + Parameters + ---------- + npu_op : ethosu.vela.api.NpuOperation + The NPU operation and its params + accel_type : ethosu.vela.api.NpuAccelerator + The NPU accelerator variant + Returns + ------- + ethosu.vela.api.NpuShape3d : + The optimal block config for the operator + """ + all_valid_block_configs = vapi.npu_find_block_configs(npu_op, accel_type) + return _get_optimal_block_config(all_valid_block_configs) + + +def _get_optimal_block_config(all_valid_block_configs): + """An internal function to get block config with largest depth + and then highest volume/area""" + assert isinstance(all_valid_block_configs, list) + for block_cfg in all_valid_block_configs: + assert isinstance(block_cfg, vapi.NpuShape3D) + + # Getting the largest volume block for benchmarksing + all_valid_block_configs.sort( + key=lambda _cfg: _cfg.depth * _cfg.height * _cfg.width, reverse=True + ) + largest_volume_block_config = all_valid_block_configs[0] + largest_volume = ( + largest_volume_block_config.depth + * largest_volume_block_config.height + * largest_volume_block_config.width + ) + + all_valid_block_configs.sort(key=lambda _cfg: _cfg.depth, reverse=True) + max_d = all_valid_block_configs[0].depth + max_depth_block_configs = [_cfg for _cfg in all_valid_block_configs if _cfg.depth == max_d] + max_depth_block_configs.sort(key=lambda _cfg: _cfg.height * _cfg.width, reverse=True) + max_area = max_depth_block_configs[0].height * max_depth_block_configs[0].width + max_area_depth_block_configs = [ + _cfg for _cfg in max_depth_block_configs if _cfg.height * _cfg.width == max_area + ] + # This to get a deterministic anwser everytime + max_area_depth_block_configs.sort(key=lambda _cfg: _cfg.height, reverse=True) + assert len(max_area_depth_block_configs) > 0 + current_volume = ( + max_area_depth_block_configs[0].depth + * max_area_depth_block_configs[0].height + * max_area_depth_block_configs[0].width + ) + logger.info("Using block config=%s", max_area_depth_block_configs[0]) + logger.info( + "Quality of the block config w.r.t. max volume block config=%s", + 100.0 * (current_volume / largest_volume), + ) + return max_area_depth_block_configs[0] + + +def compress_weights( + weights, + weights_zp, + weights_layout, + ifm_bitdepth, + block_depth, + dilation, + accel_type, + is_depthwise=False, +): + """Obtain compressed weights from vela + + Parameters + ---------- + weights : numpy.ndarray + The raw weights + weights_zp : int + The zero point of the weights + weights_layout : str + A string literal indicating the layout + Supported values : HWIO, HWOI, OHWI + ifm_bitdepth : int + The bit depth of the ifm the weights are used with + block_depth : int + The depth of the optimal block config for the operator + dilation : tuple + A tuple of 2 elements indicating dilation in h and w + accel_type : ethosu.vela.api.NpuAccelerator + The NPU accelerator variant + is_depthwise : bool, Optional + This indicates whether the weights are compressed for depthwise convolution + + Returns + ------- + compressed_weights : bytearray + Compressed weights + """ + layout_transform_indices = {"HWIO": (3, 0, 1, 2), "HWOI": (2, 0, 1, 3), "OHWI": (0, 1, 2, 3)} + assert weights_layout in layout_transform_indices.keys() + assert isinstance(weights_zp, np.int64) + weights = weights.astype(np.int64) - weights_zp + # Vela needs the weights in OHWI layout + weights_ohwi = np.transpose(weights, layout_transform_indices[weights_layout]) + shape_ohwi = [ + weights.shape[layout_transform_indices[weights_layout][0]], + weights.shape[layout_transform_indices[weights_layout][1]], + weights.shape[layout_transform_indices[weights_layout][2]], + weights.shape[layout_transform_indices[weights_layout][3]], + ] + block_traversal = calculate_block_traversal_mode(is_depthwise, shape_ohwi, ifm_bitdepth) + compressed_weights = vapi.npu_encode_weights( + accelerator=accel_type, + weights_volume=weights_ohwi, + dilation_xy=dilation, + ifm_bitdepth=ifm_bitdepth, + ofm_block_depth=block_depth, + is_depthwise=is_depthwise, + block_traversal=block_traversal, + ) + return compressed_weights + + +def calculate_block_traversal_mode(is_depthwise, weights_shape_ohwi, ifm_bitdepth): + """Calculate a block traversal mode given whether the op is depthwise convolution, + shape of weights and bit-depth of the ifm. + """ + + if is_depthwise: + return vapi.NpuBlockTraversal.DEPTH_FIRST + # Determine which block traversal strategy has better DPU utilization + kernel_size = weights_shape_ohwi[1] * weights_shape_ohwi[2] + depth_utilization = weights_shape_ohwi[3] / util.round_up( + weights_shape_ohwi[3], 32 if ifm_bitdepth == 8 else 16 + ) + part_kernel_utilization = (weights_shape_ohwi[3] / util.round_up(weights_shape_ohwi[3], 8)) * ( + kernel_size / util.round_up(kernel_size, 4 if ifm_bitdepth == 8 else 2) + ) + if part_kernel_utilization >= depth_utilization or weights_shape_ohwi[3] <= 8: + # Part-kernel first is always better for ifm depths <= 8 + return vapi.NpuBlockTraversal.PART_KERNEL_FIRST + return vapi.NpuBlockTraversal.DEPTH_FIRST + + +def pack_biases( + biases, + ifm_scale, + ifm_dtype, + weight_scales, + ofm_scale, + is_activation_tanh_or_sigmoid=False, +): + """ + Obtain packed bias bytearray as the hardware requires from Review comment: explained further -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
