ashutosh-arm commented on a change in pull request #8951: URL: https://github.com/apache/tvm/pull/8951#discussion_r704235132
########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { + const OpNode* op = call->op.as<OpNode>(); + if (op == nullptr || op->name != "qnn.quantize") { + return false; + } + const CallNode* softmax = call->args[0].as<CallNode>(); + op = softmax->op.as<OpNode>(); + if (op->name != "nn.softmax") { + return false; + } + const CallNode* dequantize = softmax->args[0].as<CallNode>(); + op = dequantize->op.as<OpNode>(); + if (op->name != "qnn.dequantize") { + return false; + } + return true; + } + + void emit_softmax_tir(const CallNode* call) { + auto* softmax_call = call->args[0].as<CallNode>(); + auto* dequant_call = softmax_call->args[0].as<CallNode>(); + auto* scale_const = dequant_call->args[1].as<ConstantNode>(); + const float quant_scale = static_cast<const float*>(scale_const->data->data)[0]; + + // assuming layout as NHWC + auto shape = call->type_as<TensorTypeNode>()->shape; + int trailing_dim = shape.size() - 1; + int row_size = shape[trailing_dim].as<tir::IntImmNode>()->value; + int num_rows = 1; + for (int i = 0; i < trailing_dim; ++i) { + num_rows *= shape[i].as<tir::IntImmNode>()->value; + } + + // calculate multiplier and shift for CMSIS-NN softmax API + // Note: TensorFlow Lite Micro assumptions + // Output zero point and scale are fixed to -128 and 1 / 256 + double beta = 1.0; + int32_t input_bits = 5; + double beta_multiplier = (beta * quant_scale * (1 << (31 - input_bits))); + beta_multiplier = std::min<double>(beta_multiplier, (1ll << 31) - 1.0); + auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(beta_multiplier); + int32_t mult = std::get<0>(mult_shift_pair); + int32_t shift = std::get<1>(mult_shift_pair); + int32_t diff_min = (1 << 5) - 1; + diff_min <<= (31 - 5); + diff_min >>= shift; + diff_min *= -1; + + auto in_var = tir::Var("input", DataType::Handle(8)); + auto out_var = tir::Var("output", DataType::Handle(8)); + + Array<tir::Var> main_signature{in_var, out_var}; Review comment: ACK ########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { + const OpNode* op = call->op.as<OpNode>(); + if (op == nullptr || op->name != "qnn.quantize") { + return false; + } + const CallNode* softmax = call->args[0].as<CallNode>(); + op = softmax->op.as<OpNode>(); + if (op->name != "nn.softmax") { + return false; + } + const CallNode* dequantize = softmax->args[0].as<CallNode>(); + op = dequantize->op.as<OpNode>(); + if (op->name != "qnn.dequantize") { + return false; + } + return true; + } + + void emit_softmax_tir(const CallNode* call) { + auto* softmax_call = call->args[0].as<CallNode>(); + auto* dequant_call = softmax_call->args[0].as<CallNode>(); + auto* scale_const = dequant_call->args[1].as<ConstantNode>(); + const float quant_scale = static_cast<const float*>(scale_const->data->data)[0]; + + // assuming layout as NHWC + auto shape = call->type_as<TensorTypeNode>()->shape; + int trailing_dim = shape.size() - 1; + int row_size = shape[trailing_dim].as<tir::IntImmNode>()->value; + int num_rows = 1; + for (int i = 0; i < trailing_dim; ++i) { + num_rows *= shape[i].as<tir::IntImmNode>()->value; + } + + // calculate multiplier and shift for CMSIS-NN softmax API + // Note: TensorFlow Lite Micro assumptions + // Output zero point and scale are fixed to -128 and 1 / 256 + double beta = 1.0; + int32_t input_bits = 5; + double beta_multiplier = (beta * quant_scale * (1 << (31 - input_bits))); + beta_multiplier = std::min<double>(beta_multiplier, (1ll << 31) - 1.0); + auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(beta_multiplier); + int32_t mult = std::get<0>(mult_shift_pair); + int32_t shift = std::get<1>(mult_shift_pair); + int32_t diff_min = (1 << 5) - 1; + diff_min <<= (31 - 5); + diff_min >>= shift; + diff_min *= -1; + + auto in_var = tir::Var("input", DataType::Handle(8)); + auto out_var = tir::Var("output", DataType::Handle(8)); + + Array<tir::Var> main_signature{in_var, out_var}; + + tvm::Array<PrimExpr> args; + args.push_back(tir::StringImm("arm_softmax_s8")); + args.push_back(in_var); + args.push_back(IntImm(DataType::Int(32), num_rows)); + args.push_back(IntImm(DataType::Int(32), row_size)); + args.push_back(IntImm(DataType::Int(32), mult)); + args.push_back(IntImm(DataType::Int(32), shift)); + args.push_back(IntImm(DataType::Int(32), diff_min)); + args.push_back(out_var); Review comment: ACK ########## File path: src/relay/backend/contrib/cmsisnn/codegen_cmsisnn.cc ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <cmath> +#include <fstream> +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include "../../../../runtime/file_utils.h" +#include "../../../../target/source/codegen_c.h" + +namespace tvm { +namespace runtime { + +using namespace tir; + +class CodeGenCMSISNN : public tvm::codegen::CodeGenC { + public: + void Init(bool output_ssa) { + decl_stream << "#include <stdio.h>\n"; + decl_stream << "#include <stdlib.h>\n"; + decl_stream << "#include <dlpack/dlpack.h>\n"; + decl_stream << "#include <tvm/runtime/crt/module.h>\n"; + decl_stream << "#include <arm_nnfunctions.h>\n"; + CodeGenC::Init(output_ssa); + } + + /*! + * \brief Emit code that offloads a subgraph to the Cortex-M + * + * \return string of code that offloads a subgraph to the Cortex-M + */ + void AddFunction(const PrimFunc& prim_func) { + PrintExternCPrefix(stream); + CodeGenC::AddFunction(prim_func); + PrintExternCPostfix(stream); + } + + private: + void VisitExpr_(const CallNode* op, std::ostream& os) { // NOLINT(*) + if (!op->op.same_as(builtin::call_extern())) { + return; + } + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + if (cmsis_func_name == "arm_softmax_s8") { + EmitSoftmax(op); Review comment: Are you suggesting that we don't check for cmsis_func_name for this version? This check will be needed for upcoming operators anyway. Right? ########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { + const OpNode* op = call->op.as<OpNode>(); + if (op == nullptr || op->name != "qnn.quantize") { + return false; + } + const CallNode* softmax = call->args[0].as<CallNode>(); + op = softmax->op.as<OpNode>(); + if (op->name != "nn.softmax") { + return false; + } + const CallNode* dequantize = softmax->args[0].as<CallNode>(); + op = dequantize->op.as<OpNode>(); + if (op->name != "qnn.dequantize") { + return false; + } + return true; + } + + void emit_softmax_tir(const CallNode* call) { + auto* softmax_call = call->args[0].as<CallNode>(); + auto* dequant_call = softmax_call->args[0].as<CallNode>(); + auto* scale_const = dequant_call->args[1].as<ConstantNode>(); + const float quant_scale = static_cast<const float*>(scale_const->data->data)[0]; + + // assuming layout as NHWC + auto shape = call->type_as<TensorTypeNode>()->shape; + int trailing_dim = shape.size() - 1; + int row_size = shape[trailing_dim].as<tir::IntImmNode>()->value; + int num_rows = 1; + for (int i = 0; i < trailing_dim; ++i) { + num_rows *= shape[i].as<tir::IntImmNode>()->value; + } + + // calculate multiplier and shift for CMSIS-NN softmax API + // Note: TensorFlow Lite Micro assumptions + // Output zero point and scale are fixed to -128 and 1 / 256 + double beta = 1.0; + int32_t input_bits = 5; + double beta_multiplier = (beta * quant_scale * (1 << (31 - input_bits))); + beta_multiplier = std::min<double>(beta_multiplier, (1ll << 31) - 1.0); + auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(beta_multiplier); + int32_t mult = std::get<0>(mult_shift_pair); + int32_t shift = std::get<1>(mult_shift_pair); + int32_t diff_min = (1 << 5) - 1; + diff_min <<= (31 - 5); + diff_min >>= shift; + diff_min *= -1; + + auto in_var = tir::Var("input", DataType::Handle(8)); + auto out_var = tir::Var("output", DataType::Handle(8)); + + Array<tir::Var> main_signature{in_var, out_var}; + + tvm::Array<PrimExpr> args; + args.push_back(tir::StringImm("arm_softmax_s8")); + args.push_back(in_var); + args.push_back(IntImm(DataType::Int(32), num_rows)); + args.push_back(IntImm(DataType::Int(32), row_size)); + args.push_back(IntImm(DataType::Int(32), mult)); + args.push_back(IntImm(DataType::Int(32), shift)); + args.push_back(IntImm(DataType::Int(32), diff_min)); + args.push_back(out_var); + tir::Stmt body = + tir::Evaluate(tvm::tir::Call(DataType::Int(8), tir::builtin::call_extern(), args)); + + Map<String, ObjectRef> dict_attrs; + dict_attrs.Set("global_symbol", func_name_); + dict_attrs.Set("tir.noalias", Bool(true)); + + primfunc_ = tir::PrimFunc(main_signature, body, VoidType(), Map<tir::Var, tir::Buffer>(), + DictAttrs(dict_attrs)); + } + + void VisitExpr_(const CallNode* call) final { + if (is_quant_softmax(call)) { + emit_softmax_tir(call); + } + } + + public: + String func_name_; + tir::PrimFunc primfunc_; +}; + +IRModule GenerateTIR(IRModule mod) { + String func_name; + Function func; + + // Obtain external Relay Function that needs to be translated into TIR + ICHECK(mod->functions.size() == 1) << "Supports modules with single external Relay function."; + for (auto kv : mod->functions) { + func = Downcast<Function>(kv.second); + func_name = func->GetAttr<String>(tvm::attr::kGlobalSymbol).value(); + } + + // Prepare PrimFunc from Relay Function + auto relay_to_tir = RelayToTIR(func_name); + relay_to_tir.VisitExpr(func->body); + + // Build the TIR IRModule from the generated PrimFunc + Map<GlobalVar, BaseFunc> var_func_map; + var_func_map.Set(GlobalVar(func_name), relay_to_tir.primfunc_); + return IRModule(var_func_map); +} + +TVM_REGISTER_GLOBAL("cmsisnn.create.tir.module").set_body([](TVMArgs args, TVMRetValue* rv) { Review comment: ACK ########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { + const OpNode* op = call->op.as<OpNode>(); + if (op == nullptr || op->name != "qnn.quantize") { + return false; + } + const CallNode* softmax = call->args[0].as<CallNode>(); + op = softmax->op.as<OpNode>(); + if (op->name != "nn.softmax") { + return false; + } + const CallNode* dequantize = softmax->args[0].as<CallNode>(); + op = dequantize->op.as<OpNode>(); + if (op->name != "qnn.dequantize") { + return false; + } + return true; + } + + void emit_softmax_tir(const CallNode* call) { + auto* softmax_call = call->args[0].as<CallNode>(); + auto* dequant_call = softmax_call->args[0].as<CallNode>(); + auto* scale_const = dequant_call->args[1].as<ConstantNode>(); + const float quant_scale = static_cast<const float*>(scale_const->data->data)[0]; + + // assuming layout as NHWC + auto shape = call->type_as<TensorTypeNode>()->shape; + int trailing_dim = shape.size() - 1; + int row_size = shape[trailing_dim].as<tir::IntImmNode>()->value; + int num_rows = 1; + for (int i = 0; i < trailing_dim; ++i) { + num_rows *= shape[i].as<tir::IntImmNode>()->value; + } + + // calculate multiplier and shift for CMSIS-NN softmax API + // Note: TensorFlow Lite Micro assumptions + // Output zero point and scale are fixed to -128 and 1 / 256 + double beta = 1.0; + int32_t input_bits = 5; + double beta_multiplier = (beta * quant_scale * (1 << (31 - input_bits))); + beta_multiplier = std::min<double>(beta_multiplier, (1ll << 31) - 1.0); + auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(beta_multiplier); + int32_t mult = std::get<0>(mult_shift_pair); + int32_t shift = std::get<1>(mult_shift_pair); + int32_t diff_min = (1 << 5) - 1; + diff_min <<= (31 - 5); + diff_min >>= shift; + diff_min *= -1; + + auto in_var = tir::Var("input", DataType::Handle(8)); + auto out_var = tir::Var("output", DataType::Handle(8)); + + Array<tir::Var> main_signature{in_var, out_var}; + + tvm::Array<PrimExpr> args; + args.push_back(tir::StringImm("arm_softmax_s8")); + args.push_back(in_var); + args.push_back(IntImm(DataType::Int(32), num_rows)); + args.push_back(IntImm(DataType::Int(32), row_size)); + args.push_back(IntImm(DataType::Int(32), mult)); + args.push_back(IntImm(DataType::Int(32), shift)); + args.push_back(IntImm(DataType::Int(32), diff_min)); + args.push_back(out_var); + tir::Stmt body = + tir::Evaluate(tvm::tir::Call(DataType::Int(8), tir::builtin::call_extern(), args)); + + Map<String, ObjectRef> dict_attrs; + dict_attrs.Set("global_symbol", func_name_); + dict_attrs.Set("tir.noalias", Bool(true)); + + primfunc_ = tir::PrimFunc(main_signature, body, VoidType(), Map<tir::Var, tir::Buffer>(), + DictAttrs(dict_attrs)); + } + + void VisitExpr_(const CallNode* call) final { + if (is_quant_softmax(call)) { + emit_softmax_tir(call); + } + } + + public: + String func_name_; Review comment: I borrowed that style from codegen_c_host.cc. Is this a style requirement though? ########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { + const OpNode* op = call->op.as<OpNode>(); + if (op == nullptr || op->name != "qnn.quantize") { + return false; + } + const CallNode* softmax = call->args[0].as<CallNode>(); + op = softmax->op.as<OpNode>(); + if (op->name != "nn.softmax") { + return false; + } + const CallNode* dequantize = softmax->args[0].as<CallNode>(); + op = dequantize->op.as<OpNode>(); + if (op->name != "qnn.dequantize") { + return false; + } + return true; + } + + void emit_softmax_tir(const CallNode* call) { + auto* softmax_call = call->args[0].as<CallNode>(); + auto* dequant_call = softmax_call->args[0].as<CallNode>(); + auto* scale_const = dequant_call->args[1].as<ConstantNode>(); + const float quant_scale = static_cast<const float*>(scale_const->data->data)[0]; + + // assuming layout as NHWC + auto shape = call->type_as<TensorTypeNode>()->shape; + int trailing_dim = shape.size() - 1; + int row_size = shape[trailing_dim].as<tir::IntImmNode>()->value; + int num_rows = 1; + for (int i = 0; i < trailing_dim; ++i) { + num_rows *= shape[i].as<tir::IntImmNode>()->value; + } + + // calculate multiplier and shift for CMSIS-NN softmax API + // Note: TensorFlow Lite Micro assumptions + // Output zero point and scale are fixed to -128 and 1 / 256 + double beta = 1.0; + int32_t input_bits = 5; + double beta_multiplier = (beta * quant_scale * (1 << (31 - input_bits))); + beta_multiplier = std::min<double>(beta_multiplier, (1ll << 31) - 1.0); + auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(beta_multiplier); + int32_t mult = std::get<0>(mult_shift_pair); + int32_t shift = std::get<1>(mult_shift_pair); + int32_t diff_min = (1 << 5) - 1; + diff_min <<= (31 - 5); + diff_min >>= shift; + diff_min *= -1; + + auto in_var = tir::Var("input", DataType::Handle(8)); + auto out_var = tir::Var("output", DataType::Handle(8)); + + Array<tir::Var> main_signature{in_var, out_var}; + + tvm::Array<PrimExpr> args; + args.push_back(tir::StringImm("arm_softmax_s8")); + args.push_back(in_var); + args.push_back(IntImm(DataType::Int(32), num_rows)); + args.push_back(IntImm(DataType::Int(32), row_size)); + args.push_back(IntImm(DataType::Int(32), mult)); + args.push_back(IntImm(DataType::Int(32), shift)); + args.push_back(IntImm(DataType::Int(32), diff_min)); + args.push_back(out_var); + tir::Stmt body = + tir::Evaluate(tvm::tir::Call(DataType::Int(8), tir::builtin::call_extern(), args)); + + Map<String, ObjectRef> dict_attrs; + dict_attrs.Set("global_symbol", func_name_); + dict_attrs.Set("tir.noalias", Bool(true)); + + primfunc_ = tir::PrimFunc(main_signature, body, VoidType(), Map<tir::Var, tir::Buffer>(), + DictAttrs(dict_attrs)); + } + + void VisitExpr_(const CallNode* call) final { + if (is_quant_softmax(call)) { + emit_softmax_tir(call); + } + } + + public: + String func_name_; + tir::PrimFunc primfunc_; +}; + +IRModule GenerateTIR(IRModule mod) { + String func_name; + Function func; + + // Obtain external Relay Function that needs to be translated into TIR + ICHECK(mod->functions.size() == 1) << "Supports modules with single external Relay function."; + for (auto kv : mod->functions) { Review comment: Done. Thanks! ########## File path: src/relay/backend/contrib/cmsisnn/codegen_cmsisnn.cc ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <cmath> +#include <fstream> +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include "../../../../runtime/file_utils.h" +#include "../../../../target/source/codegen_c.h" + +namespace tvm { +namespace runtime { + +using namespace tir; + +class CodeGenCMSISNN : public tvm::codegen::CodeGenC { + public: + void Init(bool output_ssa) { + decl_stream << "#include <stdio.h>\n"; + decl_stream << "#include <stdlib.h>\n"; + decl_stream << "#include <dlpack/dlpack.h>\n"; + decl_stream << "#include <tvm/runtime/crt/module.h>\n"; + decl_stream << "#include <arm_nnfunctions.h>\n"; + CodeGenC::Init(output_ssa); + } + + /*! + * \brief Emit code that offloads a subgraph to the Cortex-M + * + * \return string of code that offloads a subgraph to the Cortex-M + */ + void AddFunction(const PrimFunc& prim_func) { + PrintExternCPrefix(stream); + CodeGenC::AddFunction(prim_func); + PrintExternCPostfix(stream); + } + + private: + void VisitExpr_(const CallNode* op, std::ostream& os) { // NOLINT(*) + if (!op->op.same_as(builtin::call_extern())) { + return; + } + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + if (cmsis_func_name == "arm_softmax_s8") { + EmitSoftmax(op); + } + return; + } + + /*! * \brief Creates a cplusplus guard prefix for extern "C" printing */ + void PrintExternCPrefix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "extern \"C\" {\n"; + ss << "#endif\n"; + } + + /*! * \brief Creates a cplusplus guard postfix for extern "C" printing */ + void PrintExternCPostfix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "}\n"; + ss << "#endif\n"; + } + + /*! * \brief Emits CMSIS-NN code block for softmax */ + void EmitSoftmax(const CallNode* op) { + // @tir.call_extern("arm_softmax_s8", buffer_0, num_rows, row_size, + // mult, shift, diff_min, buffer_1, dtype=int8) + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + int32_t num_rows = op->args[2].as<IntImmNode>()->value; + int32_t row_size = op->args[3].as<IntImmNode>()->value; + int32_t mult = op->args[4].as<IntImmNode>()->value; + int32_t shift = op->args[5].as<IntImmNode>()->value; + int32_t diff_min = op->args[6].as<IntImmNode>()->value; + + PrintIndent(); + stream << "int32_t num_rows = " << num_rows << ";\n"; + PrintIndent(); + stream << "int32_t row_size = " << row_size << ";\n"; + PrintIndent(); + stream << "int32_t mult = " << mult << ";\n"; + PrintIndent(); + stream << "int32_t shift = " << shift << ";\n"; + PrintIndent(); + stream << "int32_t diff_min = " << diff_min << ";\n"; + PrintIndent(); + stream << cmsis_func_name << "(input,"; + PrintIndent(); + stream << " num_rows, row_size, mult, shift, diff_min, output);\n"; + PrintIndent(); + stream << "return;\n"; + } +}; + +class CMSISNNModuleNode : public runtime::ModuleNode { + public: + CMSISNNModuleNode(const std::string& code, const std::string& fmt, + const Array<String>& func_names) + : code_(code), fmt_(fmt), func_names_(func_names) {} + + std::string GetSource(const std::string& format) final { return code_; } + + const char* type_key() const { return "c"; } + + PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final { + if (name == "get_symbol") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_[0]; }); + } else if (name == "get_func_names") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_; }); + } else { + return PackedFunc(nullptr); + } + } + + void SaveToFile(const std::string& file_name, const std::string& format) final { + std::string fmt = GetFileFormat(file_name, format); + std::string meta_file = GetMetaFilePath(file_name); + if (fmt == "c" || fmt == "cu") { + ICHECK_NE(code_.length(), 0); + SaveBinaryToFile(file_name, code_); + } else { + ICHECK_EQ(fmt, fmt_) << "Can only save to format=" << fmt_; + } + } + + protected: + std::string code_; + std::string fmt_; + Array<String> func_names_; +}; + +class CMSISNNModule : public Module { Review comment: Without this, python side access was not working. I will try removing it once we move the registration for external compiler to C++. ########## File path: src/relay/backend/contrib/cmsisnn/codegen_cmsisnn.cc ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <cmath> +#include <fstream> +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include "../../../../runtime/file_utils.h" +#include "../../../../target/source/codegen_c.h" + +namespace tvm { +namespace runtime { + +using namespace tir; + +class CodeGenCMSISNN : public tvm::codegen::CodeGenC { + public: + void Init(bool output_ssa) { + decl_stream << "#include <stdio.h>\n"; + decl_stream << "#include <stdlib.h>\n"; + decl_stream << "#include <dlpack/dlpack.h>\n"; + decl_stream << "#include <tvm/runtime/crt/module.h>\n"; + decl_stream << "#include <arm_nnfunctions.h>\n"; + CodeGenC::Init(output_ssa); + } + + /*! + * \brief Emit code that offloads a subgraph to the Cortex-M + * + * \return string of code that offloads a subgraph to the Cortex-M + */ + void AddFunction(const PrimFunc& prim_func) { + PrintExternCPrefix(stream); + CodeGenC::AddFunction(prim_func); + PrintExternCPostfix(stream); + } + + private: + void VisitExpr_(const CallNode* op, std::ostream& os) { // NOLINT(*) + if (!op->op.same_as(builtin::call_extern())) { + return; + } + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + if (cmsis_func_name == "arm_softmax_s8") { + EmitSoftmax(op); + } + return; + } + + /*! * \brief Creates a cplusplus guard prefix for extern "C" printing */ + void PrintExternCPrefix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "extern \"C\" {\n"; + ss << "#endif\n"; + } + + /*! * \brief Creates a cplusplus guard postfix for extern "C" printing */ + void PrintExternCPostfix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "}\n"; + ss << "#endif\n"; + } + + /*! * \brief Emits CMSIS-NN code block for softmax */ + void EmitSoftmax(const CallNode* op) { + // @tir.call_extern("arm_softmax_s8", buffer_0, num_rows, row_size, + // mult, shift, diff_min, buffer_1, dtype=int8) + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + int32_t num_rows = op->args[2].as<IntImmNode>()->value; + int32_t row_size = op->args[3].as<IntImmNode>()->value; + int32_t mult = op->args[4].as<IntImmNode>()->value; + int32_t shift = op->args[5].as<IntImmNode>()->value; + int32_t diff_min = op->args[6].as<IntImmNode>()->value; + + PrintIndent(); + stream << "int32_t num_rows = " << num_rows << ";\n"; + PrintIndent(); + stream << "int32_t row_size = " << row_size << ";\n"; + PrintIndent(); + stream << "int32_t mult = " << mult << ";\n"; + PrintIndent(); + stream << "int32_t shift = " << shift << ";\n"; + PrintIndent(); + stream << "int32_t diff_min = " << diff_min << ";\n"; + PrintIndent(); + stream << cmsis_func_name << "(input,"; + PrintIndent(); + stream << " num_rows, row_size, mult, shift, diff_min, output);\n"; + PrintIndent(); + stream << "return;\n"; + } +}; + +class CMSISNNModuleNode : public runtime::ModuleNode { + public: + CMSISNNModuleNode(const std::string& code, const std::string& fmt, + const Array<String>& func_names) + : code_(code), fmt_(fmt), func_names_(func_names) {} + + std::string GetSource(const std::string& format) final { return code_; } + + const char* type_key() const { return "c"; } + + PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final { + if (name == "get_symbol") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_[0]; }); + } else if (name == "get_func_names") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_; }); + } else { + return PackedFunc(nullptr); + } + } + + void SaveToFile(const std::string& file_name, const std::string& format) final { + std::string fmt = GetFileFormat(file_name, format); + std::string meta_file = GetMetaFilePath(file_name); + if (fmt == "c" || fmt == "cu") { + ICHECK_NE(code_.length(), 0); + SaveBinaryToFile(file_name, code_); + } else { + ICHECK_EQ(fmt, fmt_) << "Can only save to format=" << fmt_; + } + } + + protected: + std::string code_; + std::string fmt_; + Array<String> func_names_; +}; + +class CMSISNNModule : public Module { + public: + CMSISNNModule() {} + explicit CMSISNNModule(ObjectPtr<Object> n) : Module(n) {} + inline CMSISNNModuleNode* operator->(); + inline const CMSISNNModuleNode* operator->() const; +}; + +inline CMSISNNModuleNode* CMSISNNModule::operator->() { + return static_cast<CMSISNNModuleNode*>(get_mutable()); +} + +static Module CMSISNNModuleNodeCreate(IRModule mod) { + bool output_ssa = false; + CodeGenCMSISNN cg; + Array<String> function_names; + cg.Init(output_ssa); + ICHECK(mod->functions.size() == 1) << "Supports modules with single PrimFunc."; + for (auto kv : mod->functions) { + ICHECK(kv.second->IsInstance<PrimFuncNode>()) << "CodegenCMSISNN: Can only take PrimFunc"; + auto f = Downcast<PrimFunc>(kv.second); + auto global_symbol = f->GetAttr<String>(tvm::attr::kGlobalSymbol); + ICHECK(global_symbol.defined()) + << "CodeGenCHost: Expect PrimFunc to have the global_symbol attribute"; + function_names.push_back(global_symbol.value()); + cg.AddFunction(f); + } + std::string code = cg.Finish(); + auto n = make_object<CMSISNNModuleNode>(code, "c", function_names); + return Module(n); +} + +TVM_REGISTER_GLOBAL("cmsisnn.create.runtime.module").set_body([](TVMArgs args, TVMRetValue* rv) { Review comment: Just out of curiosity, what's the guideline? ########## File path: python/tvm/relay/backend/contrib/cmsisnn/codegen.py ########## @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Codegen for CMSIS-NN""" +import tvm + + [email protected]_func("relay.ext.cmsisnn") +def cmsisnn_compiler(relay_func): + """It compiles Relay's external function into equivalent TIR + and subsequently converts that into 'c' code. During the 'c' + code generation, it embeds CMSIS-NN APIs for the corresponding + operators. + """ + mod = tvm.IRModule() + mod["main"] = relay_func + mod = tvm.relay.transform.InferType()(mod) + generate_tir = tvm._ffi.get_global_func("cmsisnn.create.tir.module") Review comment: I will make that change. When do we decide to use a pass? ########## File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc ########## @@ -0,0 +1,147 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <tvm/relay/expr_functor.h> +#include <tvm/tir/builtin.h> +#include <tvm/tir/expr.h> +#include <tvm/tir/function.h> +#include <tvm/tir/op.h> +#include <tvm/tir/stmt_functor.h> + +#include "../../../qnn/utils.h" + +namespace tvm { +namespace relay { + +class RelayToTIR : public MixedModeVisitor { + public: + explicit RelayToTIR(String func_name) : func_name_(func_name) {} + + private: + bool is_quant_softmax(const CallNode* call) { Review comment: ok, will do. ########## File path: src/relay/backend/contrib/cmsisnn/codegen_cmsisnn.cc ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <cmath> +#include <fstream> +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include "../../../../runtime/file_utils.h" +#include "../../../../target/source/codegen_c.h" + +namespace tvm { +namespace runtime { + +using namespace tir; + +class CodeGenCMSISNN : public tvm::codegen::CodeGenC { + public: + void Init(bool output_ssa) { + decl_stream << "#include <stdio.h>\n"; + decl_stream << "#include <stdlib.h>\n"; + decl_stream << "#include <dlpack/dlpack.h>\n"; + decl_stream << "#include <tvm/runtime/crt/module.h>\n"; + decl_stream << "#include <arm_nnfunctions.h>\n"; + CodeGenC::Init(output_ssa); + } + + /*! + * \brief Emit code that offloads a subgraph to the Cortex-M + * + * \return string of code that offloads a subgraph to the Cortex-M + */ + void AddFunction(const PrimFunc& prim_func) { + PrintExternCPrefix(stream); + CodeGenC::AddFunction(prim_func); + PrintExternCPostfix(stream); + } + + private: + void VisitExpr_(const CallNode* op, std::ostream& os) { // NOLINT(*) + if (!op->op.same_as(builtin::call_extern())) { + return; + } + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + if (cmsis_func_name == "arm_softmax_s8") { + EmitSoftmax(op); + } + return; + } + + /*! * \brief Creates a cplusplus guard prefix for extern "C" printing */ + void PrintExternCPrefix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "extern \"C\" {\n"; + ss << "#endif\n"; + } + + /*! * \brief Creates a cplusplus guard postfix for extern "C" printing */ + void PrintExternCPostfix(std::ostringstream& ss) { + PrintIndent(); + ss << "#ifdef __cplusplus\n"; + ss << "}\n"; + ss << "#endif\n"; + } + + /*! * \brief Emits CMSIS-NN code block for softmax */ + void EmitSoftmax(const CallNode* op) { + // @tir.call_extern("arm_softmax_s8", buffer_0, num_rows, row_size, + // mult, shift, diff_min, buffer_1, dtype=int8) + std::string cmsis_func_name = op->args[0].as<StringImmNode>()->value; + int32_t num_rows = op->args[2].as<IntImmNode>()->value; + int32_t row_size = op->args[3].as<IntImmNode>()->value; + int32_t mult = op->args[4].as<IntImmNode>()->value; + int32_t shift = op->args[5].as<IntImmNode>()->value; + int32_t diff_min = op->args[6].as<IntImmNode>()->value; + + PrintIndent(); + stream << "int32_t num_rows = " << num_rows << ";\n"; + PrintIndent(); + stream << "int32_t row_size = " << row_size << ";\n"; + PrintIndent(); + stream << "int32_t mult = " << mult << ";\n"; + PrintIndent(); + stream << "int32_t shift = " << shift << ";\n"; + PrintIndent(); + stream << "int32_t diff_min = " << diff_min << ";\n"; + PrintIndent(); + stream << cmsis_func_name << "(input,"; + PrintIndent(); + stream << " num_rows, row_size, mult, shift, diff_min, output);\n"; + PrintIndent(); + stream << "return;\n"; + } +}; + +class CMSISNNModuleNode : public runtime::ModuleNode { + public: + CMSISNNModuleNode(const std::string& code, const std::string& fmt, + const Array<String>& func_names) + : code_(code), fmt_(fmt), func_names_(func_names) {} + + std::string GetSource(const std::string& format) final { return code_; } + + const char* type_key() const { return "c"; } + + PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final { + if (name == "get_symbol") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_[0]; }); + } else if (name == "get_func_names") { + return PackedFunc( + [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->func_names_; }); + } else { + return PackedFunc(nullptr); + } + } + + void SaveToFile(const std::string& file_name, const std::string& format) final { + std::string fmt = GetFileFormat(file_name, format); + std::string meta_file = GetMetaFilePath(file_name); + if (fmt == "c" || fmt == "cu") { Review comment: Thanks for pointing that out. I will remove it. Result of bad copy-n-paste from codegen_c_host.cc -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
