hxzd5568 opened a new issue, #16211: URL: https://github.com/apache/tvm/issues/16211
We find unacceptable numerical errors exist between optimized and un-optimized models. Two imprecise passes cause the errors. And the errors are silent, meaning only particular inputs trigger the errors. These severe yet slient errors accumulate along the computational graph, harming the security of the models. ### Expected behavior The results of optimized models remains consistent with that of the un-optimized models. ### Actual behavior Significant discrepancies are between un-optimized and optimized models' results. The relative error caused by optimization is more than 0.9 when the model has several operators and its type is float32. (tvm's built-in tolerance is 10^-4)  ### Environment Normal ### Steps to reproduce ```python # showing three small cases import tvm from tvm import relay,runtime import os import numpy as np import queue import shutil import os.path import random from tvm import transform, relay, parser, cpu, TVMError, IRModule from tvm.contrib.graph_executor import GraphModule from argparse import Namespace, ArgumentParser from typing import Iterable, List, cast, Optional, Dict TensorDict = Dict[str, np.ndarray] target = tvm.target.Target("llvm", host="llvm") layout = None dev = tvm.cpu(0) import time Required_pass1 = ['EliminateCommonSubexpr','CombineParallelDense','CombineParallelBatchMatmul','CombineParallelConv2D'] def MSE(y_true, y_pred,): #precision along with tf.keras.metrics.MeanRelativeError d = np.abs(y_true.astype(np.float64) - y_pred) relative_error = np.average( d \ / (np.abs(y_true).astype(np.float64) + 1e-8) ) return relative_error def SE(y_true, y_pred,): #precision along with tf.keras.metrics.MeanRelativeError d = np.abs(y_true.astype(np.float64) - y_pred) relative_error = np.max( d \ / (np.abs(y_true).astype(np.float64) + 1e-8))# * np.abs(y_true) / np.mean(np.abs(y_true)) return relative_error def run_gmod( gmod: GraphModule, inputs: Dict[str, np.ndarray]=None) -> List[np.ndarray]: if inputs is not None: gmod.run(**inputs) else: gmod.run() return [gmod.get_output(i).numpy() for i in range(gmod.get_num_outputs())] def build_workload(mod, params=None, Disabled_pass=['SimplifyExpr']): with transform.PassContext(opt_level=1, required_pass=Required_pass1,disabled_pass=Disabled_pass): lib1 = relay.build(mod, target) with transform.PassContext(opt_level=5):#disabled_pass=Disabled_pass lib5 = relay.build(mod, target) return lib1, lib5 def replay(mod,params): factorymod1, factorymod5 = build_workload(\ mod,params= params) gmod1 = GraphModule(factorymod1["default"](dev)) gmod5 = GraphModule(factorymod5["default"](dev)) outs1 = run_gmod(gmod1,params) outs5 = run_gmod(gmod5,params) tdiff = 0. for (ro,o) in zip(outs1,outs5): diff = MSE(ro,o) tdiff = max(tdiff,diff) print('mean relative error = ' ,tdiff) tdiff2 = 0. for (ro,o) in zip(outs1,outs5): diff = SE(ro,o) tdiff2 = max(tdiff2,diff) print(' max relative error = ' ,tdiff2) def test_mod1(): def mod1(): shape = (4,3) x = relay.var("x", shape=shape, dtype="float32") y = relay.var("y", shape=shape, dtype="float32") m = relay.sqrt(relay.abs(y)) n = relay.divide(x,m) l = relay.round(relay.nn.relu(relay.tan(relay.sum(n,axis=[1])))) return tvm.IRModule.from_expr(l) params = {'x': np.array([[-3.0407448 , 5. , 1.4677091 ], [ 5. , -0.08194685, 3.0596933 ], [ 5. , 5. , 3.7800522 ], [ 5. , 3.1617928 , 5. ]], dtype=np.float32), 'y': np.array([[-0.11967325 , -0.018634353 , 0.1582024 ], [-0.09131396 , -0.0047433637, -0.020964164 ], [-0.08089028 , -0.01746996 , -0.008808094 ], [ 0.1787599 , 0.1756186 , 0.041228298 ]], dtype=np.float32)} mod = mod1() replay(mod,params) test_mod1() # mean relative error = 0.14285714265306124 # max relative error = 0.571428570612245 def test_mod2(): def mod2(): n = 16 c1_val = np.ones(shape=n).astype("float32")/1.0 c2_val = np.ones(shape=n).astype("float32")/100.0 c3_val = np.ones(shape=n).astype("float32")/10000.0 x = relay.var("x", shape=(n,), dtype="float32") c1 = relay.const(c1_val) c2 = relay.const(c2_val) c3 = relay.const(c3_val) return tvm.IRModule.from_expr(c2 + (c1 + x) + c3,) params = {'x': np.array([-1.0100999 , -1.0346043 , -1.9652936 , 5. , 5. , 5. , 5. , 5. , 0.3813362 , 5. , -0.052576065, 5. , 3.8130388 , 5. , -5. , 5. ], dtype=np.float32)} mod = mod2() replay(mod,params) test_mod2() def test_mod3(): def mod3(): data = relay.var("data", shape=(1, 3, 3, 10), dtype="float32") in_bias= relay.var("in_bias", shape=(16,1 ,1), dtype="float32") weight= relay.var("weight", shape=(16, 3, 3, 3), dtype="float32") f = relay.const(3.0) m = relay.nn.conv2d(data, weight, padding=[1, 1, 1, 1], channels=16, kernel_size=[3, 3]) n = relay.add(m, in_bias) l = relay.nn.relu(n) k = relay.multiply(l, f) return tvm.IRModule.from_expr(k) mod = mod3() params = {'data': np.array([[[[ 2.3390198 , -4.3133545 , -4.168396 , 4.1300964 , -1.585846 , 4.627075 , -1.7875671 , 4.581299 , -0.3274536 , 4.811096 ], [-4.8747253 , -1.4906311 , 2.1806335 , 1.2471008 , 1.650238 , 1.1271667 , 4.062042 , -0.6352234 , -4.7898865 , 0.9611511 ], [ 3.6834717 , -2.626648 , 3.9530945 , -2.3561096 , -3.7078857 , -3.3638 , 4.926605 , -3.468933 , -0.47225952 , -4.3060303 ]], [[ 0.9333801 , -0.86120605 , 2.9893494 , 0.5119324 , -0.31982422 , -1.7929077 , -3.085022 , 0.080566406, 4.5147705 , 2.816162 ], [ 0.78552246 , 3.9726257 , -2.258606 , -4.65744 , -0.9147644 , 2.1720886 , 2.206726 , 4.0460205 , 3.1819153 , 1.4639282 ], [ 3.8554382 , 3.4243774 , -3.8671875 , 3.833313 , -1.5910339 , -1.302185 , -0.5026245 , -0.6347656 , 3.161621 , 0.45211792 ]], [[-1.6278076 , -1.9839478 , 2.6101685 , 4.23645 , -3.1515503 , -3.9056396 , 2.9049683 , 2.2320557 , -3.10318 , -2.545929 ], [-2.974701 , -3.9118958 , 2.2982788 , -0.61187744 , -1.4146423 , 2.8793335 , 4.4039917 , -0.27786255 , -2.4920654 , 4.8028564 ], [ 3.308258 , 4.4880676 , -2.1774292 , 2.526703 , -3.170166 , 2.5920105 , 3.2528687 , 3.97995 , 2.3698425 , 4.1340637 ]]]], dtype=np.float32), 'weight': np.array([[[[ 1.84359588e-02, -5.67617603e-02, 9.23220292e-02], [-4.69010361e-02, 1.30543604e-01, 1.42569663e-02], [ 1.10723212e-01, 3.06858160e-02, 4.02834564e-02]], [[-6.04484677e-02, -1.16323540e-02, -7.41606206e-02], [ 5.89779206e-02, 1.91885035e-03, 1.82958841e-02], [ 5.99368215e-02, -8.83646682e-03, 4.10590395e-02]], [[ 4.28394899e-02, -3.14531587e-02, 1.21172234e-01], [ 1.31590543e-02, -8.29366129e-03, -7.28027299e-02], [-1.23870615e-02, -4.20262702e-02, -1.88857093e-01]]], [[[-1.03945844e-02, 9.29425210e-02, -8.81018341e-02], [-2.49063715e-01, -2.91489027e-02, -2.77917013e-02], [ 5.65432804e-03, 3.25867012e-02, -1.96195301e-02]], [[ 5.90098090e-04, 3.00193541e-02, 1.23762675e-02], [ 6.20452948e-02, 1.14255715e-02, 1.12046309e-01], [-1.55365765e-01, 1.00451596e-01, -3.63353193e-02]], [[ 6.40910268e-02, 5.50712086e-02, -1.47086740e-01], [ 3.21470462e-02, -1.05340764e-01, -2.43274271e-02], [ 1.63880765e-01, 3.19500528e-02, 2.85845120e-02]]], [[[-7.54672959e-02, -1.84240818e-01, 1.66560218e-01], [ 7.61865601e-02, -3.92302051e-02, 2.13027801e-02], [-1.18684247e-01, -1.19206876e-01, 1.58000495e-02]], [[-2.73413863e-02, 3.07147522e-02, -3.98259722e-02], [-5.09087034e-02, -1.37307912e-01, -2.26264130e-02], [-4.66945954e-02, 8.06659088e-03, 1.43448249e-01]], [[-1.97702169e-01, -1.28330439e-01, 9.40558389e-02], [-9.67540219e-02, 1.21365443e-01, 4.25816700e-03], [-1.32838383e-01, 3.16822715e-02, 7.35550001e-02]]], [[[-6.24646656e-02, -2.47376531e-01, 1.87695637e-01], [-7.13282032e-03, 2.61037312e-02, -2.38758460e-01], [ 1.17336325e-01, 1.22818805e-01, 5.43967858e-02]], [[-4.20460431e-03, -1.23153338e-02, -8.22688490e-02], [-1.68626159e-02, -1.84650291e-02, -3.81258987e-02], [ 6.09956495e-02, -1.84162110e-01, 1.01075836e-01]], [[ 9.78261158e-02, 1.84520796e-01, -1.23352215e-01], [-7.67529085e-02, 7.17516094e-02, -1.24184690e-01], [-1.81342319e-01, -1.13486223e-01, -1.04571888e-02]]], [[[ 3.39533240e-02, 2.22197428e-01, 1.33450195e-01], [-4.55260910e-02, -1.81334484e-02, 2.67483033e-02], [ 9.47716013e-02, -6.55783489e-02, 1.96428239e-01]], [[-2.03044191e-01, 7.57685825e-02, 1.07348405e-01], [-3.82460803e-02, -1.24667190e-01, 3.98001522e-02], [-1.48324087e-01, -3.48817557e-02, -7.91698471e-02]], [[-1.69110790e-01, 1.02850795e-01, -4.68062460e-02], [ 2.52880841e-01, -3.60808475e-03, -8.77002068e-03], [-1.44789457e-01, -1.41288405e-02, 8.95667821e-02]]], [[[ 7.32647926e-02, 1.09085865e-01, 2.38027200e-02], [-3.67730558e-02, 3.61794345e-02, -1.14263669e-01], [ 3.16222087e-02, -1.88899547e-01, 1.25211999e-01]], [[-4.96775247e-02, 1.27293184e-01, -1.92385808e-01], [ 1.00658663e-01, 1.67965457e-01, 4.09109183e-02], [ 5.65359816e-02, -9.89584178e-02, -3.17060528e-03]], [[-1.14270553e-01, -8.16770084e-03, 1.02076098e-01], [ 1.69018716e-01, 7.87431374e-02, -2.32533123e-02], [-3.83749530e-02, -6.88052028e-02, -4.91857007e-02]]], [[[ 2.21293047e-01, -2.12399922e-02, 1.62791774e-01], [-1.49898762e-02, -1.78696007e-01, -1.94149807e-01], [-1.12235673e-01, 1.36905193e-01, -2.63103824e-02]], [[-1.74314335e-01, -1.63090099e-02, 8.88844803e-02], [-3.73804383e-02, 1.61483679e-02, 6.13411143e-02], [ 2.53802150e-01, 1.21361576e-01, -5.73121831e-02]], [[-6.32467344e-02, 4.52737063e-02, 9.87602174e-02], [ 7.08832592e-02, -5.51897362e-02, 2.08691750e-02], [ 1.14512995e-01, 9.42543224e-02, 3.78478840e-02]]], [[[-1.36125147e-01, -2.46732458e-02, -2.89720222e-02], [ 1.21674858e-01, 2.63642728e-01, -5.44232950e-02], [ 1.35930451e-02, -4.22105975e-02, 2.43935958e-01]], [[ 6.04593121e-02, -7.35267177e-02, -2.41658371e-02], [ 1.28745139e-01, -3.39329019e-02, -9.99284629e-03], [-2.47891042e-02, -5.08213192e-02, -6.92131743e-02]], [[-7.34953955e-02, -1.50193602e-01, -1.14641331e-01], [ 7.63458461e-02, 8.32802802e-02, -8.10493380e-02], [-2.30264664e-02, 9.38710347e-02, 5.29599339e-02]]], [[[-1.01752892e-01, 3.78644131e-02, -1.05646417e-01], [-1.90132737e-01, 4.82698232e-02, 1.74660552e-02], [-1.97341219e-02, -7.59350806e-02, 4.21354733e-02]], [[ 5.20936176e-02, -1.12383403e-01, 2.10713223e-01], [ 1.37035966e-01, 1.02595329e-01, -1.42810881e-01], [ 4.12588939e-02, 4.41232920e-02, 2.76997276e-02]], [[-4.14286256e-02, 7.52515495e-02, -1.79455191e-01], [-7.91246220e-02, 8.62184614e-02, 3.77373546e-02], [ 3.27565558e-02, -3.27662304e-02, -2.36460604e-02]]], [[[ 1.40866674e-02, 2.93224957e-02, 2.20708586e-02], [ 7.25577250e-02, 1.35610819e-01, -1.40848858e-02], [ 3.38155210e-01, 1.15791587e-02, 2.42292553e-01]], [[-6.46224171e-02, -3.00012045e-02, -3.07604298e-03], [-9.67979431e-02, -4.97113541e-02, 3.61424945e-02], [-5.24158170e-03, 5.15089333e-02, -7.03435019e-02]], [[-8.28879252e-02, 8.13878477e-02, 2.27911413e-01], [-9.23482552e-02, 7.64181614e-02, -5.98769821e-02], [-4.01492231e-02, 4.20970134e-02, -8.66010413e-02]]], [[[-4.29190546e-02, -3.57790962e-02, -5.63404001e-02], [ 4.04255055e-02, 3.42209241e-03, -4.27752621e-02], [ 8.86510964e-03, 5.89131303e-02, 1.54032260e-01]], [[ 4.25744317e-02, -3.01371086e-02, -1.23406254e-01], [ 9.40691978e-02, 3.69642600e-02, 1.70801394e-02], [-9.53103006e-02, 1.36304617e-01, -1.01172701e-01]], [[ 1.35000601e-01, -5.69341742e-02, -1.56284571e-01], [-1.64234579e-01, -9.41175446e-02, -8.31616744e-02], [ 3.36662941e-02, -1.03074778e-02, 7.58383572e-02]]], [[[ 2.95754969e-02, -1.48214787e-01, -1.81754842e-01], [ 1.17960386e-01, 1.88315772e-02, -1.98493838e-01], [ 6.22544475e-02, -1.58774257e-01, -9.24718380e-02]], [[-1.07496576e-02, -9.41871628e-02, -4.32278179e-02], [-1.68052524e-01, 3.74041535e-02, -3.19579393e-02], [ 2.32069433e-01, -4.76445891e-02, -5.61368242e-02]], [[ 4.54570726e-02, 1.07900694e-01, 1.63872778e-01], [ 7.96293095e-02, 1.21984616e-01, -1.16307959e-01], [ 6.61069825e-02, 3.60182784e-02, 1.50917560e-01]]], [[[ 1.10181384e-01, 2.11550489e-01, 1.13034308e-01], [ 8.22045375e-03, 7.39668235e-02, 1.42670065e-01], [ 8.36678296e-02, 8.83488208e-02, 7.93456584e-02]], [[-7.12593831e-03, -6.91364184e-02, 4.90392074e-02], [ 1.46005005e-01, 8.01168382e-03, -2.30196491e-02], [ 7.68633038e-02, -3.72724570e-02, 3.52027714e-02]], [[-8.87521580e-02, -3.91909182e-02, 4.67454866e-02], [-1.49627283e-01, 6.64105117e-02, 1.28207460e-01], [-5.04156798e-02, -1.46469146e-01, 4.09836462e-03]]], [[[-8.73182490e-02, -1.89433470e-02, 1.91175863e-01], [ 1.80323645e-01, -5.56149632e-02, -1.27435118e-01], [ 7.88302645e-02, 1.17271870e-01, 1.75540030e-01]], [[-1.12294868e-01, 3.20856348e-02, 1.48295602e-02], [ 1.32150665e-01, 1.18797995e-01, 2.55297571e-02], [-1.31113976e-01, 1.80141106e-02, 8.62932727e-02]], [[-1.98585801e-02, -2.25724086e-01, 4.68168445e-02], [-1.11531204e-04, 1.35372922e-01, 6.66687712e-02], [-1.15940377e-01, 1.08834676e-01, -3.09612781e-01]]], [[[ 1.81119405e-02, 3.16587724e-02, 8.70872363e-02], [ 7.67229311e-03, 1.42912254e-01, -4.06906055e-03], [ 8.77697859e-03, 3.55612077e-02, 1.82276309e-01]], [[-1.52334207e-04, 5.16612008e-02, -2.03733996e-01], [ 2.39904020e-02, 8.05658698e-02, 5.61261401e-02], [-1.62812844e-01, -3.55103216e-03, -2.10080013e-01]], [[ 8.14223289e-02, 7.14186952e-02, -1.17138296e-01], [ 1.84469387e-01, -9.65789612e-03, -5.88245988e-02], [ 5.03358208e-02, -1.89822316e-02, 9.73736495e-02]]], [[[-1.22605346e-01, -5.99360019e-02, -9.02506858e-02], [ 1.44807352e-02, -3.94839048e-02, -3.47705632e-02], [ 6.88567832e-02, -1.49033563e-02, 1.97620183e-01]], [[-1.27270564e-01, 1.18250437e-01, 7.46178553e-02], [-5.93046285e-03, -1.34781212e-01, -1.09252250e-02], [-1.37259021e-01, 6.32974505e-02, 6.53135553e-02]], [[-7.47158751e-02, -6.36640117e-02, -2.11437374e-01], [-4.92087454e-02, 1.71410684e-02, -7.83042759e-02], [ 1.28504217e-01, -1.08570503e-02, 3.75919379e-02]]]], dtype=np.float32), 'in_bias': np.array([[[ 0.0134733375]], [[-0.106072426 ]], [[ 0.031830218 ]], [[ 0.08156953 ]], [[-0.016601322 ]], [[-0.0849547 ]], [[ 0.049820814 ]], [[-0.2598204 ]], [[ 0.05602941 ]], [[ 0.1593517 ]], [[-0.018954927 ]], [[ 0.03032378 ]], [[-0.058005586 ]], [[ 0.0047690133]], [[ 0.029218856 ]], [[ 0.020432528 ]]], dtype=np.float32)} replay(mod,params) test_mod3() ``` ### Triage [imprecision or nonequivalence of relay:transformer] There are 3 types of buggy patterns which lead to numerical errors. 1. sqrt-divide --> rsqrt -mul (converted by SimplifyExpr) 2. conv-relu-multiply --> multiply-conv-relu (converted by FoldScaleAxis) 3. const expression folding c1+(c2+x)+c3---> x+ c4, s.t., c4= c1+c2+c3 (converted by SimplifyExpr) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
