This is an automated email from the ASF dual-hosted git repository.
mshr-h pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 28ff3867c8 [FIX][Relax][ONNX] Correct normalization semantics (#20510)
28ff3867c8 is described below
commit 28ff3867c8c6a9008fc786aa1a60eca6e32764ac
Author: Yuhx141 <[email protected]>
AuthorDate: Tue Nov 17 13:53:46 2026 +0000
[FIX][Relax][ONNX] Correct normalization semantics (#20510)
This corrects three normalization semantics in the ONNX frontend and
Relax training decomposition:
- Convert ONNX BatchNormalization momentum to Relax's update-coefficient
convention, keeping existing Relax/TOPI semantics unchanged.
- Return real float32 LayerNormalization Mean and InvStdDev outputs,
preserve ONNX Scale/B broadcasting, and cast the normalized tensor back
before affine arithmetic. Compatible FP16/FP32 Y-only nodes retain the
fused operator, including nodes with trailing empty output names. Main
graphs and If branches trim trailing empty outputs consistently while
preserving named statistics positions.
- Normalize a negative BatchNorm axis before constructing training
reduction axes. FP16 LayerNorm training decomposition now uses FP32
statistics and casts back before scaling, matching the fused operator.
Closes #20496
Closes #20503
Closes #20504
CPU validation:
- ONNX normalization and If regressions: 77 passed.
- Complete decomposition pass tests: 7 passed, including full expected
IR and fused/decomposed FP16 numerical checks.
- New regressions reproduce 7 failures against the previous
implementation; all 17 new cases pass after the fix.
- Rebuilt the modified C++ pass against the matching existing CPU build.
Pre-commit checks pass for all four changed files.
GPU/CUTLASS/TensorRT execution was not tested; the regressions verify
that the fused Relax operator is retained for eligible imports.
---------
Signed-off-by: Yuhx141 <[email protected]>
---
python/tvm/relax/frontend/onnx/onnx_frontend.py | 86 +++++++----
src/relax/transform/decompose_ops.cc | 19 ++-
tests/python/relax/test_frontend_onnx.py | 166 +++++++++++++++++++++
tests/python/relax/test_transform_decompose_ops.py | 83 +++++++++++
4 files changed, 322 insertions(+), 32 deletions(-)
diff --git a/python/tvm/relax/frontend/onnx/onnx_frontend.py
b/python/tvm/relax/frontend/onnx/onnx_frontend.py
index 95c26aa55f..52715b9fb5 100644
--- a/python/tvm/relax/frontend/onnx/onnx_frontend.py
+++ b/python/tvm/relax/frontend/onnx/onnx_frontend.py
@@ -58,7 +58,6 @@ import tvm_ffi
import tvm_tirx
from tvm import relax, topi
-from tvm.topi.utils import get_const_tuple
from tvm_tirx import tirx
from tvm_tirx.ir import Expr as TvmExpr
from tvm_tirx.ir import IRModule
@@ -4379,7 +4378,7 @@ class BatchNormalization(OnnxOpConverter):
moving_var=cast_for_compute(var, var_dtype),
axis=1,
epsilon=epsilon,
- momentum=momentum,
+ momentum=1 - momentum,
training=bool(training_mode),
)
@@ -4712,25 +4711,48 @@ class LayerNormalization(OnnxOpConverter):
bias = inputs[2]
axis = attr.get("axis", -1)
epsilon = attr.get("epsilon", 1e-05)
+ stash_type = attr.get("stash_type", 1)
- gamma_shape = get_const_tuple(scale.ty.shape)
-
- if bias is None:
- bias = relax.const(_np.zeros(gamma_shape), dtype=scale.ty.dtype)
- else:
- beta_shape = get_const_tuple(bias.ty.shape)
- if gamma_shape != beta_shape:
- raise ValueError("gamma and beta shapes do not match")
-
- axis = list(axis) if isinstance(axis, list | tuple) else [axis]
- if len(axis) < len(gamma_shape):
- axis.extend(range(axis[-1] + 1, axis[-1] + 1 + len(gamma_shape) -
len(axis)))
+ if stash_type != 1:
+ raise ValueError("LayerNormalization currently only supports
stash_type=1 (FLOAT).")
+ ndim = _get_known_tensor_rank(data)
+ if ndim is None:
+ raise ValueError("LayerNormalization requires a statically known
input rank.")
+ axis = _normalize_constant_axes([axis], ndim, "LayerNormalization")[0]
+ axes = list(range(axis, ndim))
+ num_outputs = attr["tvm_custom"]["num_outputs"]
+ dtype = str(data.ty.dtype)
- output = relax.op.nn.layer_norm(data, scale, bias, axis, epsilon)
- # Onnx layernorm has 3 outputs but only the first is used.
- # We construct two empty constants for this.
- placeholder = relax.const(0, dtype="float32")
- return relax.Tuple([output, placeholder, placeholder])
+ # Relax's fused operator requires affine parameters to match the
normalized shape.
+ if (
+ num_outputs == 1
+ and dtype in ("float16", "float32")
+ and isinstance(data.ty.shape, relax.ShapeExpr)
+ ):
+ normalized_shape = relax.ShapeExpr(data.ty.shape.values[axis:])
+ if tvm_ffi.structural_equal(scale.ty.shape, normalized_shape) and (
+ bias is None or tvm_ffi.structural_equal(bias.ty.shape,
normalized_shape)
+ ):
+ if bias is None:
+ bias = relax.op.zeros_like(scale)
+ return relax.op.nn.layer_norm(data, scale, bias, axes, epsilon)
+
+ stats_data = data if dtype == "float32" else
bb.emit(relax.op.astype(data, "float32"))
+ mean = bb.emit(relax.op.mean(stats_data, axis=axes, keepdims=True))
+ centered = bb.emit(relax.op.subtract(stats_data, mean))
+ variance = relax.op.mean(relax.op.multiply(centered, centered),
axis=axes, keepdims=True)
+ inv_std_dev = bb.emit(
+ relax.op.rsqrt(relax.op.add(variance, relax.const(epsilon,
dtype="float32")))
+ )
+ normalized = relax.op.multiply(centered, inv_std_dev)
+ if dtype != "float32":
+ normalized = relax.op.astype(normalized, dtype)
+ output = relax.op.multiply(normalized, scale)
+ if bias is not None:
+ output = relax.op.add(output, bias)
+ if num_outputs == 1:
+ return output
+ return relax.Tuple([output, mean, inv_std_dev])
class RMSNormalization(OnnxOpConverter):
@@ -6982,6 +7004,13 @@ class ONNXGraphImporter:
msg += ", ".join(unsupported_ops)
raise NotImplementedError(msg)
+ @staticmethod
+ def _get_node_outputs(node):
+ outputs = list(node.output)
+ while outputs and not outputs[-1]:
+ outputs.pop()
+ return outputs
+
def _construct_nodes(self, graph: onnx.onnx_ml_pb2.GraphProto):
"""Nodes are stored as directed acyclic graph."""
for node in graph.node:
@@ -6995,7 +7024,7 @@ class ONNXGraphImporter:
else:
inputs.append(None)
i_name = self._parse_value_proto(node)
- outputs = node.output
+ outputs = self._get_node_outputs(node)
attr["tvm_custom"] = {}
attr["tvm_custom"]["name"] = i_name
attr["tvm_custom"]["num_outputs"] = len(outputs)
@@ -7011,7 +7040,8 @@ class ONNXGraphImporter:
self._nodes[outputs[0]] = if_result
else:
for i, k in enumerate(outputs):
- self._nodes[k] =
self.bb.emit(relax.TupleGetItem(if_result, i))
+ if k:
+ self._nodes[k] =
self.bb.emit(relax.TupleGetItem(if_result, i))
continue
# Perform special handling for shape expressions. If an input is a
@@ -7078,7 +7108,8 @@ class ONNXGraphImporter:
self._nodes[outputs[0]] = op
else:
for i, k in enumerate(outputs):
- self._nodes[k] = op[i]
+ if k:
+ self._nodes[k] = op[i]
def _parse_value_proto(self, value_proto: onnx.onnx_ml_pb2.GraphProto):
"""Parse ValueProto or raw str."""
@@ -7182,7 +7213,8 @@ class ONNXGraphImporter:
attr["tvm_custom"] = {}
attr["tvm_custom"]["name"] = node.name
- attr["tvm_custom"]["num_outputs"] = len(node.output)
+ outputs = self._get_node_outputs(node)
+ attr["tvm_custom"]["num_outputs"] = len(outputs)
# Handle nested If recursively.
if op_name == "If":
@@ -7192,12 +7224,12 @@ class ONNXGraphImporter:
then_seq = relax.SeqExpr(blocks=[], body=then_expr)
else_seq = relax.SeqExpr(blocks=[], body=else_expr)
op = bb.emit(relax.IfExpr(cond, then_seq, else_seq))
- outputs = node.output
if len(outputs) == 1:
self._nodes[outputs[0]] = op
else:
for i, k in enumerate(outputs):
- self._nodes[k] = bb.emit(relax.TupleGetItem(op, i))
+ if k:
+ self._nodes[k] =
bb.emit(relax.TupleGetItem(op, i))
continue
op = self._convert_operator(op_name, inputs, attr, self.opset)
@@ -7207,12 +7239,12 @@ class ONNXGraphImporter:
tuple_items = [relax.TupleGetItem(op, i) for i in
range(len(op.ty.fields))]
op = relax.Tuple(tuple_items)
- outputs = node.output
if len(outputs) == 1:
self._nodes[outputs[0]] = op
else:
for i, k in enumerate(outputs):
- self._nodes[k] = op[i]
+ if k:
+ self._nodes[k] = op[i]
branch_outputs = [self._nodes[o.name] for o in graph.output]
result = branch_outputs[0] if len(branch_outputs) == 1 else
relax.Tuple(branch_outputs)
diff --git a/src/relax/transform/decompose_ops.cc
b/src/relax/transform/decompose_ops.cc
index 2e0684e216..b31c6af7fd 100644
--- a/src/relax/transform/decompose_ops.cc
+++ b/src/relax/transform/decompose_ops.cc
@@ -30,6 +30,8 @@
#include <unordered_set>
+#include "../op/op_common.h"
+#include "../op/tensor/datatype.h"
#include "utils.h"
namespace tvm {
@@ -95,9 +97,10 @@ Expr MutateBatchNormForTraining(Call call) {
TensorType ty = MatchTensorType(data);
+ int axis = NormalizeAxis(call, ty->ndim, attrs->axis);
ffi::Array<int64_t> reduce_axes;
for (int i = 0; i < ty->ndim; ++i) {
- if (i != attrs->axis) {
+ if (i != axis) {
reduce_axes.push_back(i);
}
}
@@ -126,13 +129,19 @@ Expr DecomposeLayerNorm(const Call& call) {
Expr gamma = call->args[1];
Expr beta = call->args[2];
- Expr data_mean = mean(data, attrs->axes, true);
- Expr data_var = variance(data, attrs->axes, true);
+ const bool is_float16 = ty->dtype.value() == PrimType::Float(16);
+ Expr stats_data = is_float16 ? astype(data, PrimType::Float(32)->dtype) :
data;
+ Expr data_mean = mean(stats_data, attrs->axes, true);
+ Expr data_var = variance(stats_data, attrs->axes, true);
// output = (x - mean) / sqrt(var + epsilon) * gamma + beta
- Expr epsilon = MakeConstantScalar(attrs->epsilon, ty->dtype.value()->dtype);
+ Expr epsilon = MakeConstantScalar(
+ attrs->epsilon, is_float16 ? PrimType::Float(32)->dtype :
ty->dtype.value()->dtype);
Expr sqrt_var = sqrt(add(data_var, epsilon));
- Expr out = divide(subtract(data, data_mean), sqrt_var);
+ Expr out = divide(subtract(stats_data, data_mean), sqrt_var);
+ if (is_float16) {
+ out = astype(out, ty->dtype.value()->dtype);
+ }
if (attrs->scale) {
out = multiply(out, gamma);
diff --git a/tests/python/relax/test_frontend_onnx.py
b/tests/python/relax/test_frontend_onnx.py
index 1a5113ca2d..bb015f9ca1 100644
--- a/tests/python/relax/test_frontend_onnx.py
+++ b/tests/python/relax/test_frontend_onnx.py
@@ -6243,6 +6243,171 @@ def test_layer_norm():
from_onnx(model, opset=17, keep_params_in_input=True)
+def test_layer_norm_optional_outputs():
+ node = helper.make_node(
+ "LayerNormalization",
+ ["input", "scale", "bias"],
+ ["Y", "Mean", "InvStdDev"],
+ axis=-1,
+ epsilon=1e-5,
+ )
+ graph = helper.make_graph(
+ [node],
+ "layer_norm_optional_outputs_test",
+ inputs=[
+ helper.make_tensor_value_info("input", TensorProto.FLOAT, [2, 3,
4]),
+ helper.make_tensor_value_info("scale", TensorProto.FLOAT, [4]),
+ helper.make_tensor_value_info("bias", TensorProto.FLOAT, [4]),
+ ],
+ outputs=[
+ helper.make_tensor_value_info("Y", TensorProto.FLOAT, [2, 3, 4]),
+ helper.make_tensor_value_info("Mean", TensorProto.FLOAT, [2, 3,
1]),
+ helper.make_tensor_value_info("InvStdDev", TensorProto.FLOAT, [2,
3, 1]),
+ ],
+ )
+ model = helper.make_model(
+ graph,
+ producer_name="layer_norm_optional_outputs_test",
+ opset_imports=[helper.make_opsetid("", 17)],
+ )
+ check_correctness(
+ model,
+ inputs={
+ "input": np.arange(24, dtype="float32").reshape(2, 3, 4),
+ "scale": np.ones(4, dtype="float32"),
+ "bias": np.zeros(4, dtype="float32"),
+ },
+ opset=17,
+ check_dtypes=True,
+ )
+
+
[email protected]("dtype", ["float32", "float16"])
[email protected]("axis", [1, -2])
[email protected](
+ "scale_shape, bias_shape",
+ [((4,), (4,)), ((1, 4), (4,)), ((4, 4), None), ((4, 4), (4, 4))],
+)
[email protected]("num_outputs", [1, 2, 3])
+def test_layer_norm_broadcast_parameters(dtype, axis, scale_shape, bias_shape,
num_outputs):
+ tensor_type = TensorProto.FLOAT if dtype == "float32" else
TensorProto.FLOAT16
+ input_names = ["input", "scale"]
+ input_infos = [
+ helper.make_tensor_value_info("input", tensor_type, [2, 4, 4]),
+ helper.make_tensor_value_info("scale", tensor_type, scale_shape),
+ ]
+ inputs = {
+ "input": np.linspace(-3.0, 5.0, 32).reshape(2, 4, 4).astype(dtype),
+ "scale": np.linspace(0.5, 1.5, int(np.prod(scale_shape)))
+ .reshape(scale_shape)
+ .astype(dtype),
+ }
+ if bias_shape is not None:
+ input_names.append("bias")
+ input_infos.append(helper.make_tensor_value_info("bias", tensor_type,
bias_shape))
+ inputs["bias"] = (
+ np.linspace(-0.25, 0.25,
int(np.prod(bias_shape))).reshape(bias_shape).astype(dtype)
+ )
+ output_infos = [
+ helper.make_tensor_value_info("Y", tensor_type, [2, 4, 4]),
+ helper.make_tensor_value_info("Mean", TensorProto.FLOAT, [2, 1, 1]),
+ helper.make_tensor_value_info("InvStdDev", TensorProto.FLOAT, [2, 1,
1]),
+ ][:num_outputs]
+ node = helper.make_node(
+ "LayerNormalization", input_names, [info.name for info in
output_infos], axis=axis
+ )
+ model = helper.make_model(
+ helper.make_graph([node], "layer_norm_broadcast_parameters",
input_infos, output_infos),
+ opset_imports=[helper.make_opsetid("", 17)],
+ )
+ # Some ONNX Runtime versions reject valid broadcasted scale and bias
shapes.
+ x = inputs["input"].astype("float32")
+ axes = tuple(range(axis % x.ndim, x.ndim))
+ mean = np.mean(x, axis=axes, keepdims=True)
+ inv_std_dev = 1 / np.sqrt(np.var(x, axis=axes, keepdims=True) +
np.float32(1e-5))
+ expected_y = ((x - mean) * inv_std_dev).astype(dtype) * inputs["scale"]
+ if bias_shape is not None:
+ expected_y += inputs["bias"]
+ expected = [expected_y, mean, inv_std_dev][:num_outputs]
+
+ actual = run_in_tvm(model, inputs=inputs, opset=17)
+ actual = [actual] if num_outputs == 1 else actual
+ assert len(actual) == num_outputs
+ tol = 1e-3 if dtype == "float16" else 1e-5
+ for value, reference in zip(actual, expected):
+ value = value.numpy()
+ assert value.dtype == reference.dtype
+ assert value.shape == reference.shape
+ np.testing.assert_allclose(value, reference, rtol=tol, atol=tol)
+
+
[email protected]("dtype", ["float32", "float16"])
[email protected](
+ "outputs", [["Y"], ["Y", "", ""], ["Y", "Mean", ""], ["Y", "",
"InvStdDev"]]
+)
[email protected]("in_subgraph", [False, True])
+def test_layer_norm_optional_output_names(dtype, outputs, in_subgraph):
+ tensor_type = TensorProto.FLOAT if dtype == "float32" else
TensorProto.FLOAT16
+ input_infos = [
+ helper.make_tensor_value_info("input", tensor_type, [2, 4]),
+ helper.make_tensor_value_info("scale", tensor_type, [4]),
+ ]
+ output_infos = [
+ helper.make_tensor_value_info(
+ name,
+ tensor_type if name == "Y" else TensorProto.FLOAT,
+ [2, 4] if name == "Y" else [2, 1],
+ )
+ for name in outputs
+ if name
+ ]
+ node = helper.make_node("LayerNormalization", ["input", "scale"], outputs)
+ inputs = {
+ "input": np.array([[1, 2, 4, 8], [10000, 10016, 10032, 10048]],
dtype=dtype),
+ "scale": np.array([0.5, 1.5, 2.0, 0.75], dtype=dtype),
+ }
+ if in_subgraph:
+ branch = helper.make_graph([node], "branch", [], output_infos)
+ node = helper.make_node(
+ "If",
+ ["cond"],
+ [info.name for info in output_infos],
+ then_branch=branch,
+ else_branch=branch,
+ )
+ input_infos.append(helper.make_tensor_value_info("cond",
TensorProto.BOOL, []))
+ inputs["cond"] = np.array(True)
+ model = helper.make_model(
+ helper.make_graph([node], "layer_norm_optional_outputs", input_infos,
output_infos),
+ opset_imports=[helper.make_opsetid("", 17)],
+ )
+ onnx.checker.check_model(model)
+ ops = collect_relax_call_ops(from_onnx(model)["main"])
+ if len(output_infos) == 1:
+ assert "relax.nn.layer_norm" in ops
+ assert "relax.mean" not in ops
+ assert "relax.rsqrt" not in ops
+
+ x = inputs["input"].astype("float32")
+ mean = np.mean(x, axis=-1, keepdims=True)
+ inv_std_dev = 1 / np.sqrt(np.var(x, axis=-1, keepdims=True) +
np.float32(1e-5))
+ expected = {
+ "Y": ((x - mean) * inv_std_dev).astype(dtype) * inputs["scale"],
+ "Mean": mean,
+ "InvStdDev": inv_std_dev,
+ }
+ actual = run_in_tvm(model, inputs=inputs, opset=17)
+ actual = [actual] if len(output_infos) == 1 else actual
+ assert len(actual) == len(output_infos)
+ for value, info in zip(actual, output_infos):
+ value = value.numpy()
+ reference = expected[info.name]
+ assert value.dtype == reference.dtype
+ assert value.shape == reference.shape
+ tol = 1e-3 if reference.dtype == np.float16 else 1e-5
+ np.testing.assert_allclose(value, reference, rtol=tol, atol=tol)
+
+
def test_layer_norm_with_nd_gamma_beta():
layer_norm_node = helper.make_node(
"LayerNormalization", ["input", "scale", "bias"], ["Y"], axis=1,
epsilon=1e-12
@@ -10341,6 +10506,7 @@ def test_batch_norm_training_preserves_output_dtypes():
assert len(batch_norm_calls) == 1
assert [str(arg.ty.dtype) for arg in batch_norm_calls[0].args] ==
["float32"] * 5
+ assert batch_norm_calls[0].attrs.momentum == pytest.approx(0.1)
def get_pool_padding(shape, auto_pad, kernel_shape, strides, pads):
diff --git a/tests/python/relax/test_transform_decompose_ops.py
b/tests/python/relax/test_transform_decompose_ops.py
index 9d872554ad..998f986c03 100644
--- a/tests/python/relax/test_transform_decompose_ops.py
+++ b/tests/python/relax/test_transform_decompose_ops.py
@@ -18,6 +18,9 @@
from typing import Union
+import numpy as np
+import pytest
+
import tvm
import tvm_tirx
import tvm_tirx.script
@@ -182,6 +185,31 @@ def test_batch_norm_training():
tvm_tirx.ir.assert_structural_equal(Expected, After)
+def test_batch_norm_training_negative_axis():
+ @I.ir_module
+ class Before:
+ @R.function
+ def main(
+ x: R.Tensor((2, 3, 4), "float32"),
+ gamma: R.Tensor((4,), "float32"),
+ beta: R.Tensor((4,), "float32"),
+ moving_mean: R.Tensor((4,), "float32"),
+ moving_var: R.Tensor((4,), "float32"),
+ ):
+ return R.nn.batch_norm(
+ x,
+ gamma,
+ beta,
+ moving_mean,
+ moving_var,
+ axis=-1,
+ training=True,
+ )
+
+ After = relax.transform.DecomposeOpsForTraining()(Before)
+ relax.analysis.check_well_formed(After, check_ty=True)
+
+
def test_batch_norm_multiple_functions():
@I.ir_module
class Before:
@@ -361,6 +389,61 @@ def test_layer_norm():
tvm_tirx.ir.assert_structural_equal(Expected, After)
[email protected](not tvm_tirx.runtime.enabled("llvm"), reason="LLVM is
required")
+def test_layer_norm_fp16_training():
+ @I.ir_module
+ class Before:
+ @R.function
+ def main(
+ x: R.Tensor((2, 4), "float16"),
+ gamma: R.Tensor((4,), "float16"),
+ beta: R.Tensor((4,), "float16"),
+ ):
+ with R.dataflow():
+ output = R.nn.layer_norm(x, gamma, beta, axes=[-1],
epsilon=1e-5)
+ R.output(output)
+ return output
+
+ @I.ir_module
+ class Expected:
+ @R.function
+ def main(
+ x: R.Tensor((2, 4), "float16"),
+ gamma: R.Tensor((4,), "float16"),
+ beta: R.Tensor((4,), "float16"),
+ ):
+ with R.dataflow():
+ stats_data = R.astype(x, "float32")
+ mean = R.mean(stats_data, axis=[-1], keepdims=True)
+ centered = R.subtract(stats_data, mean)
+ variance = R.variance(stats_data, axis=[-1], keepdims=True)
+ adjusted = R.add(variance, R.const(1e-5, "float32"))
+ stddev = R.sqrt(adjusted)
+ normalized = R.divide(centered, stddev)
+ normalized_fp16 = R.astype(normalized, "float16")
+ scaled = R.multiply(normalized_fp16, gamma)
+ output = R.add(scaled, beta)
+ R.output(output)
+ return output
+
+ after = relax.transform.DecomposeOpsForTraining()(Before)
+ tvm_tirx.ir.assert_structural_equal(Expected, after)
+
+ x = np.array([[1, 2, 4, 8], [30000, 30016, 30032, 30048]], dtype="float16")
+ gamma = np.array([0.5, 1.5, 2.0, 0.75], dtype="float16")
+ beta = np.array([0.25, -0.5, 0.75, -1.0], dtype="float16")
+ stats_data = x.astype("float32")
+ mean = np.mean(stats_data, axis=-1, keepdims=True)
+ stddev = np.sqrt(np.var(stats_data, axis=-1, keepdims=True) +
np.float32(1e-5))
+ expected = ((stats_data - mean) / stddev).astype("float16") * gamma + beta
+ args = [tvm_tirx.runtime.tensor(value) for value in (x, gamma, beta)]
+ for mod in (Before, after):
+ vm = relax.VirtualMachine(tvm.compile(mod, target="llvm"),
tvm_tirx.cpu())
+ actual = vm["main"](*args).numpy()
+ assert actual.dtype == np.float16
+ np.testing.assert_allclose(actual, expected, rtol=1e-3, atol=1e-3)
+
+
def test_op_tensor_to_shape():
@I.ir_module
class Before: