AndrewZhaoLuo commented on a change in pull request #8952:
URL: https://github.com/apache/tvm/pull/8952#discussion_r725291267
##########
File path: python/tvm/relay/frontend/onnx.py
##########
@@ -3506,6 +3508,152 @@ def _impl_v10(cls, inputs, attr, params):
return _qnn.op.quantize(out, c_scale, c_zero_point, out_dtype=dtype)
+class QLinearMatMul(OnnxOpConverter):
+ """
+ Operator converter for QLinearMatMul from Microsoft onnxruntime contrib
opset.
+
+ Limitations:
+ - Only supports 2D input tensors.
+ - Not guaranteed to meet the integer-overflow behavior stipulated in the
+ ONNX documentation for this operator.
+ """
+
+ @classmethod
+ def _impl_v10(cls, inputs, attr, params):
+
+ # Some of the ops used below take scalar-like inputs, and may require
either
+ # of the following:
+ #
+ # - the input is Const node (not merely an expression that *could* be
reduced
+ # to a single Const at graph-compilation time)
+ #
+ # - the input has a specific dtype
+ #
+ # This function attempts to present 'x' in a form that meets both of
those
+ # requirements.
+ def try_resolve_to_const_scalar(x, dtype_override=None):
+ x2 = try_resolve_var_to_const(x, params)
+ x3 = ensure_scalar_shape(x2)
+
+ x_dtype = infer_type(x).checked_type.dtype
+ if (dtype_override is not None) and (dtype_override != x_dtype):
+ x4 = _op.cast(x3, dtype_override)
+ else:
+ x4 = x3
+
+ x5 = fold_constant(x4)
+ return x5
+
+ # Unpack the inputs and obtain some type info...
+ a, a_scale, a_zp, b, b_scale, b_zp, y_scale, y_zp = inputs
+
+ a_type = infer_type(a).checked_type # 'T1' in ONNX doc for this op
+ a_scale_type = infer_type(a_scale).checked_type
+ a_zp_type = infer_type(a_zp).checked_type
+
+ b_type = infer_type(b).checked_type # 'T2' in ONNX doc for this op
+ b_scale_type = infer_type(b_scale).checked_type
+ b_zp_type = infer_type(b_zp).checked_type
+
+ y_scale_type = infer_type(y_scale).checked_type
+ y_zp_type = infer_type(y_zp).checked_type # 'T3' in ONNX doc for this
op
+
+ a_shape = infer_shape(a)
+ b_shape = infer_shape(b)
+
+ # Verify type assumptions, based on the ONNX doc for this op...
+ assert a_type.dtype in ["int8", "uint8"]
+ assert a_scale_type.dtype == "float32"
+ assert a_zp_type.dtype == a_type.dtype
+
+ assert b_type.dtype in ["int8", "uint8"]
+ assert b_scale_type.dtype == "float32"
+ assert b_zp_type.dtype == b_type.dtype
+
+ assert y_scale_type.dtype == "float32"
+ assert y_zp_type.dtype in ["int8", "uint8"]
+
+ # TODO: relax this limitation in a future version of this importer.
+ a_rank = len(a_shape)
+ b_rank = len(b_shape)
+ assert (a_rank == 2) and (b_rank == 2), (
+ "QLinearMatMul importer currently requires both 'a' and 'b'
tensors to be 2D, but"
+ " rank(a)={}, rank(b)={}".format(a_rank, b_rank)
+ )
+
+ # _qnn.op.dense requires the zero-point values to have dtype int32.
+ a_scale_scalar = try_resolve_to_const_scalar(a_scale)
+ a_zp_scalar = try_resolve_to_const_scalar(a_zp, "int32")
+
+ b_scale_scalar = try_resolve_to_const_scalar(b_scale)
+ b_zp_scalar = try_resolve_to_const_scalar(b_zp, "int32")
+
+ y_scale_scalar = try_resolve_to_const_scalar(y_scale)
+ y_zp_scalar = try_resolve_to_const_scalar(y_zp, "int32")
+
+ # TODO: Confirm that we're using 'num_hidden_units' correctly / as
intended with
+ # the '_qnn.op.dense' instance below.
+ num_hidden_units = infer_shape(b)[-1]
+
+ # - Specify the matmul result dtype as int32, so that hopefully the
matmul will use
+ # a 32-bit accumulator as seems to be required by the ONNX op's
documentation.
+ #
+ # TL;DR:
+ # The ONNX documentation for this op is clear about acceptable overflow
+ # behavior during the matmul operation:
+ # - The scalar multiplication ops MAY NOT overflow.
+ # - The scalar addition ops, which sum the results of the scalar
multiplication,
+ # MAY overflow, but if they do so, it must behave as one would
expect during
+ # 32-bit integer-addition overflow.
+ # As of this writing, Relay's qnn.op.dense operator doesn't expose a
way for us to
+ # express these constraints.
+ matmul_result_dtype = "int32"
+
+ matmul_result = _qnn.op.dense(
Review comment:
I mean the Matmul code in onnx, which handles numpy style broadcasting.
That one is just a frontend change on how we generate relay from frontend code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
