lhutton1 commented on a change in pull request #6532: URL: https://github.com/apache/incubator-tvm/pull/6532#discussion_r494901379
########## File path: tests/python/contrib/test_arm_compute_lib/test_add.py ########## @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Arm Compute Library integration reshape tests.""" + +import numpy as np + +import tvm +import tvm.testing +from tvm import relay + +from test_arm_compute_lib.infrastructure import ( + skip_runtime_test, + skip_codegen_test, + build_and_run, + verify, + verify_codegen, +) +from test_arm_compute_lib.infrastructure import Device + + +def _get_model(shape, dtype, var_names, op, op_params): + a = relay.var(next(var_names), shape=shape, dtype=dtype) + b = relay.var(next(var_names), shape=shape, dtype=dtype) + return op(a, b, **op_params) + + +def _get_expected_codegen(shape, dtype): + node = { + "op": "kernel", + "name": "add", + "inputs": [[0, 0, 0], [1, 0, 0]], + "attrs": { + "num_inputs": "2", + "num_outputs": "1", + "shape": [[list(shape)]], + "dtype": [[dtype]], + }, + } + + input_a = {"op": "input", "name": "", "attrs": {"shape": [[list(shape)]], "dtype": [[dtype]]}} + input_b = {"op": "input", "name": "", "attrs": {"shape": [[list(shape)]], "dtype": [[dtype]]}} + return [input_a, input_b, node] + + +def test_add(): + Device.load("test_config.json") + + if skip_runtime_test(): + return + + device = Device() + np.random.seed(0) + + for dtype, low, high, atol, rtol, op, op_params in [ + ("float32", -127, 128, 1e-7, 1e-7, relay.add, {}), + # different qnn params + ( + "uint8", + 0, + 255, + 0.0, + 1.0, + relay.qnn.op.add, + { + "lhs_scale": relay.const(0.0156863, "float32"), + "lhs_zero_point": relay.const(127, "int32"), + "rhs_scale": relay.const(0.0117647, "float32"), + "rhs_zero_point": relay.const(85, "int32"), + "output_scale": relay.const(0.0235294, "float32"), + "output_zero_point": relay.const(128, "int32"), + }, + ), + # same qnn params + ( + "uint8", + 0, + 255, + 0.0, + 1.0, + relay.qnn.op.add, + { + "lhs_scale": relay.const(0.0126863, "float32"), + "lhs_zero_point": relay.const(127, "int32"), + "rhs_scale": relay.const(0.0126863, "float32"), + "rhs_zero_point": relay.const(127, "int32"), + "output_scale": relay.const(0.0126863, "float32"), + "output_zero_point": relay.const(127, "int32"), + }, + ), + ]: + shape = (2, 2) + for inputs in [ + { + "a": tvm.nd.array(np.random.uniform(low, high, shape).astype(dtype)), + "b": tvm.nd.array(np.random.uniform(low, high, shape).astype(dtype)), + } + ]: + outputs = [] + func = _get_model(shape, dtype, iter(inputs), op, op_params) + for acl in [True, False]: + outputs.append(build_and_run(func, inputs, 1, None, device, enable_acl=acl)[0]) + + config = { + "shape": shape, + "dtype": dtype, + "inputs": inputs, + "operation": op, + "op_params": op_params, + } + verify(outputs, atol=atol, rtol=rtol, config=config) Review comment: The intention was to ensure that most of the values output are not saturated i.e. most values are not 255 or 0. I can see why this would cause an issue with test cases of 2x2 as a single value that is 255 or 0 would trigger the asserts. I think the best approach here would be to increase the size of the test case? ########## File path: tests/python/contrib/test_arm_compute_lib/test_add.py ########## @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Arm Compute Library integration reshape tests.""" + +import numpy as np + +import tvm +import tvm.testing +from tvm import relay + +from test_arm_compute_lib.infrastructure import ( + skip_runtime_test, + skip_codegen_test, + build_and_run, + verify, + verify_codegen, +) +from test_arm_compute_lib.infrastructure import Device + + +def _get_model(shape, dtype, var_names, op, op_params): + a = relay.var(next(var_names), shape=shape, dtype=dtype) + b = relay.var(next(var_names), shape=shape, dtype=dtype) + return op(a, b, **op_params) + + +def _get_expected_codegen(shape, dtype): + node = { + "op": "kernel", + "name": "add", + "inputs": [[0, 0, 0], [1, 0, 0]], + "attrs": { + "num_inputs": "2", + "num_outputs": "1", + "shape": [[list(shape)]], + "dtype": [[dtype]], + }, + } + + input_a = {"op": "input", "name": "", "attrs": {"shape": [[list(shape)]], "dtype": [[dtype]]}} + input_b = {"op": "input", "name": "", "attrs": {"shape": [[list(shape)]], "dtype": [[dtype]]}} + return [input_a, input_b, node] + + +def test_add(): + Device.load("test_config.json") + + if skip_runtime_test(): + return + + device = Device() + np.random.seed(0) + + for dtype, low, high, atol, rtol, op, op_params in [ + ("float32", -127, 128, 1e-7, 1e-7, relay.add, {}), + # different qnn params + ( + "uint8", + 0, + 255, + 0.0, + 1.0, + relay.qnn.op.add, + { + "lhs_scale": relay.const(0.0156863, "float32"), + "lhs_zero_point": relay.const(127, "int32"), + "rhs_scale": relay.const(0.0117647, "float32"), + "rhs_zero_point": relay.const(85, "int32"), + "output_scale": relay.const(0.0235294, "float32"), + "output_zero_point": relay.const(128, "int32"), + }, + ), + # same qnn params + ( + "uint8", + 0, + 255, + 0.0, + 1.0, + relay.qnn.op.add, + { + "lhs_scale": relay.const(0.0126863, "float32"), + "lhs_zero_point": relay.const(127, "int32"), + "rhs_scale": relay.const(0.0126863, "float32"), + "rhs_zero_point": relay.const(127, "int32"), + "output_scale": relay.const(0.0126863, "float32"), + "output_zero_point": relay.const(127, "int32"), + }, + ), + ]: + shape = (2, 2) + for inputs in [ + { + "a": tvm.nd.array(np.random.uniform(low, high, shape).astype(dtype)), + "b": tvm.nd.array(np.random.uniform(low, high, shape).astype(dtype)), + } + ]: + outputs = [] + func = _get_model(shape, dtype, iter(inputs), op, op_params) + for acl in [True, False]: + outputs.append(build_and_run(func, inputs, 1, None, device, enable_acl=acl)[0]) + + config = { + "shape": shape, + "dtype": dtype, + "inputs": inputs, + "operation": op, + "op_params": op_params, + } + verify(outputs, atol=atol, rtol=rtol, config=config) Review comment: The intention was to ensure that most of the values output are not saturated i.e. most values are not 255 or 0. I can see why this would cause an issue with test cases of 2x2 as a single value that is 255 or 0 would trigger the asserts. I think the best approach here would be to increase the size of the test case, or to reduce 0.25 down to a value less than 25%? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
