maekawatoshiki commented on code in PR #14536:
URL: https://github.com/apache/tvm/pull/14536#discussion_r1193101620
##########
tests/python/relay/test_pass_fake_quantization_to_integer.py:
##########
@@ -1114,5 +1114,53 @@ def test_fake_quantize_take():
compare_fq_to_int(op, [x_np])
+def test_fake_quantize_softmax():
+ shape = [5, 10]
+ x_ = relay.var("x", shape=shape, dtype="int8")
+
+ is_sorted = lambda a: np.all(a[:-1] <= a[1:])
+
+ for scale in [1.0, 0.1, 0.01]:
+ x = relay.qnn.op.dequantize(x_, relay.const(scale), relay.const(0))
+ op = relay.op.nn.softmax(x, axis=1)
+ op = relay.qnn.op.quantize(
+ op, relay.const(1.0 / 256.0), relay.const(-128), out_dtype="int8"
+ )
+
+ x_np = np.random.randint(-128, 127, size=shape, dtype="int8")
+ x_np = np.sort(x_np)
+ args = [x_np]
+
+ mod = tvm.IRModule.from_expr(op)
+ mod = tvm.relay.transform.InferType()(mod)
+ mod_int = tvm.relay.transform.FakeQuantizationToInteger(
+ hard_fail=True, optional_qnn_ops=["nn.softmax"]
+ )(mod)
Review Comment:
The pass does not use the fully-integer implementation for `nn.softmax`,
unless it is specified as `optional_qnn_ops`.
##########
tests/python/relay/test_pass_fake_quantization_to_integer.py:
##########
@@ -1114,5 +1114,53 @@ def test_fake_quantize_take():
compare_fq_to_int(op, [x_np])
+def test_fake_quantize_softmax():
+ shape = [5, 10]
+ x_ = relay.var("x", shape=shape, dtype="int8")
+
+ is_sorted = lambda a: np.all(a[:-1] <= a[1:])
+
+ for scale in [1.0, 0.1, 0.01]:
+ x = relay.qnn.op.dequantize(x_, relay.const(scale), relay.const(0))
+ op = relay.op.nn.softmax(x, axis=1)
+ op = relay.qnn.op.quantize(
+ op, relay.const(1.0 / 256.0), relay.const(-128), out_dtype="int8"
+ )
+
+ x_np = np.random.randint(-128, 127, size=shape, dtype="int8")
+ x_np = np.sort(x_np)
+ args = [x_np]
+
+ mod = tvm.IRModule.from_expr(op)
+ mod = tvm.relay.transform.InferType()(mod)
+ mod_int = tvm.relay.transform.FakeQuantizationToInteger(
+ hard_fail=True, optional_qnn_ops=["nn.softmax"]
+ )(mod)
+ assert not tvm.ir.structural_equal(mod, mod_int)
+
+ result = (
+ relay.create_executor("vm", mod=mod, device=tvm.cpu(),
target="llvm")
+ .evaluate()(*args)
+ .numpy()
+ )
+ result_int = (
+ relay.create_executor("vm", mod=mod_int, device=tvm.cpu(),
target="llvm")
+ .evaluate()(*args)
+ .numpy()
+ )
+
+ # Check at least the softmax output is in ascending order,
+ # since it is difficult to use allclose due to not-so-good accuracy.
+ for qdq, qop in zip(result, result_int):
+ assert is_sorted(qdq)
+ assert is_sorted(qop)
+
+ try:
+ np.testing.assert_allclose(result_int, result, atol=1)
+ except AssertionError as e:
+ # To see the difference
+ print(e)
Review Comment:
What do you think about this?
I checked the max absolute difference here is, in most cases, 0~6.
Also the overall trend of the softmax output didn't differ much between the
QOp and QDQ implementation.
##########
src/relay/qnn/op/softmax.cc:
##########
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/relay/qnn/op/softmax.cc
+ * \brief QNN softmax operator.
+ */
+#include <tvm/relay/analysis.h>
+#include <tvm/relay/op_attr_types.h>
+
+#include "op_common.h"
+#include "tvm/ir/expr.h"
+#include "tvm/relay/attrs/nn.h"
+#include "tvm/relay/type.h"
+#include "tvm/runtime/data_type.h"
+#include "tvm/runtime/logging.h"
+#include "tvm/topi/reduction.h"
+
+namespace tvm {
+namespace relay {
+namespace qnn {
+
+bool QnnSoftmaxRel(const Array<Type>& types, int num_inputs, const Attrs&
attrs,
+ const TypeReporter& reporter) {
+ // Expected Types: input, scale, zero_point, output_scale,
output_zero_point, output
+ ICHECK_EQ(types.size(), 6);
+ const auto* x = types[0].as<TensorTypeNode>();
+ if (x == nullptr) return false;
+ ICHECK(x->dtype == DataType::Int(8))
+ << "Expected quantized softmax type(int8) for input but was " <<
x->dtype;
+
+ // Check the types of scale and zero points.
+ for (size_t i = 1; i < 5; ++i) {
+ if (types[i].as<IncompleteTypeNode>()) {
+ return false;
+ }
+ }
+
+ ICHECK(IsScalarType(types[1], DataType::Float(32))); // scale
+ ICHECK(IsScalarType(types[2], DataType::Int(32))); // zero_point
+ ICHECK(IsScalarType(types[3], DataType::Float(32))); // scale
+ ICHECK(IsScalarType(types[4], DataType::Int(32))); // zero_point
+
+ // Assign types for scale and zero points.
+ reporter->Assign(types[1], TensorType({}, DataType::Float(32))); // scale
+ reporter->Assign(types[2], TensorType({}, DataType::Int(32))); //
zero_point
+ reporter->Assign(types[3], TensorType({}, DataType::Float(32))); // scale
+ reporter->Assign(types[4], TensorType({}, DataType::Int(32))); //
zero_point
+
+ // Collect the input tensor and output tensor devoid of scale and zero
points to reuse Relay
+ // IdentityRel infer type function.
+ Array<Type> tensor_types = {types[0], types[5]};
+ return IdentityRel(tensor_types, 2, attrs, reporter);
+}
+
+// Positional relay function to create quantized softmax operator used by
frontend FFI.
+Expr MakeQuantizedSoftmax(Expr x, int axis, Expr scale, Expr zero_point, Expr
output_scale,
+ Expr output_zero_point) {
+ auto attrs = make_object<SoftmaxAttrs>();
+ attrs->axis = axis;
+ static const Op& op = Op::Get("qnn.softmax");
+ return Call(op, {x, scale, zero_point, output_scale, output_zero_point},
Attrs(attrs), {});
+}
+
+/*
+ * \brief Canonicalizes the QNN softmax op.
+ */
+Expr QnnSoftmaxCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
+ const Array<tvm::relay::Type>& arg_types) {
+ // Expected: input, scale, zero_point, output_scale, output_zero_point
+ ICHECK_EQ(new_args.size(), 5);
+
+ const auto const_i32 = [&](int32_t val) { return
MakeConstantScalar(DataType::Int(32), val); };
+ const auto const_f32 = [&](float val) { return
MakeConstantScalar(DataType::Float(32), val); };
+
+ const auto const_input_scale = new_args[1].as<ConstantNode>();
+ ICHECK(const_input_scale) << "Input scale should be constant.";
+ ICHECK(const_input_scale->is_scalar()) << "Input scale should be scalar.";
+ const float input_scale =
static_cast<float*>(const_input_scale->data->data)[0];
+ ICHECK(input_scale <= 1.f) << "Input scale should be less than or equal to
1.";
Review Comment:
The assertion fails when the input scale does not meet the condition.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
