kparzysz-quic commented on a change in pull request #6261: URL: https://github.com/apache/incubator-tvm/pull/6261#discussion_r471419335
########## File path: src/target/llvm/codegen_hexagon.cc ########## @@ -0,0 +1,812 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#if defined(TVM_LLVM_VERSION) && TVM_LLVM_VERSION >= 70 + +#include <llvm/Bitcode/BitcodeWriter.h> +#if TVM_LLVM_VERSION <= 90 +#include <llvm/IR/Intrinsics.h> +#else +#include <llvm/IR/IntrinsicsHexagon.h> +#endif +#include <llvm/Support/CommandLine.h> +#include <tvm/runtime/module.h> +#include <tvm/target/codegen.h> +#include <tvm/tir/analysis.h> + +#include <cstdio> +#include <cstdlib> +#include <map> +#include <sstream> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "../../runtime/hexagon/hexagon_module.h" +#include "../build_common.h" +#include "codegen_llvm.h" + +namespace tvm { +namespace codegen { + +static std::string get_name(const PrimFunc& f) { + auto global_symbol = f->GetAttr<runtime::String>(tvm::attr::kGlobalSymbol); + CHECK(global_symbol.defined()) + << "CodeGenLLVM: Expect PrimFunc to have the global_symbol attribute"; + return std::string(global_symbol.value()); +} + +// Hexagon code generation +class CodeGenHexagon final : public CodeGenLLVM { + public: + void InitTarget(llvm::TargetMachine* tm) final; + void Init(const std::string& module_name, llvm::TargetMachine* tm, llvm::LLVMContext* ctx, + bool system_lib, bool dynamic_lookup, bool target_c_runtime) final; + + void VisitStmt_(const AssertStmtNode* op) override; + + llvm::Value* CreateIntrinsic(const CallNode* op) override; + llvm::Value* CreateCallExtern(Type ret_type, String global_symbol, const Array<PrimExpr>& args, + bool skip_first_arg) override; + llvm::Module* GetModulePtr() const { return module_.get(); } + + protected: + // meta data + llvm::MDNode* md_tbaa_ctx_ptr_{nullptr}; + llvm::FunctionType* ftype_tvm_func_call_{nullptr}; + llvm::FunctionType* ftype_tvm_get_func_from_env_{nullptr}; + llvm::FunctionType* ftype_tvm_api_set_last_error_{nullptr}; + + private: + llvm::Value* CreateStructRefPtr(DataType t, llvm::Value* buf, llvm::Value* index, int kind); + + // Check if the call to packed function is successful + // if not directly finalize function and pass on return code. + // return the end block after the check + llvm::BasicBlock* CheckCallSuccess(llvm::Value* retcode); + + // Get runtime functions + llvm::Value* RuntimeTVMFuncCall(); + llvm::Value* RuntimeTVMGetFuncFromEnv(); + llvm::Value* RuntimeTVMAPISetLastError(); + + void InitGlobalContext(bool dynamic_lookup); + llvm::GlobalVariable* InitContextPtr(llvm::Type* type, std::string name); + llvm::Value* GetContextPtr(llvm::GlobalVariable* gv); + std::vector<std::pair<std::string, llvm::Value*>> export_system_symbols_; + llvm::Value* GetPackedFuncHandle(const std::string& str); + + // global to packed function handle + std::unordered_map<std::string, llvm::GlobalVariable*> func_handle_map_; + + // Make packed call. + llvm::BasicBlock* MakeCallPacked(const Array<PrimExpr>& args, llvm::Value** rvalue, + llvm::Value** ret_tcode, const DataType& r_type, + const int64_t begin, const int64_t end); + // create call into tvm packed function. + llvm::Value* CreateCallPacked(const CallNode* op); + // Create trace call into tvm packed function. + llvm::Value* CreateCallTracePacked(const CallNode* op); + + std::map<std::string, llvm::Type*> types_for_alloca_; + + // Type definitions. + llvm::Type* t_tvm_func_handle_{nullptr}; + llvm::Type* t_tvm_value_{nullptr}; + llvm::Type* t_tvm_shape_index_{nullptr}; + llvm::Type* t_tvm_context_{nullptr}; + llvm::Type* t_tvm_type_{nullptr}; + llvm::Type* t_tvm_array_{nullptr}; + + // Context for injection lookup + llvm::GlobalVariable* gv_mod_ctx_{nullptr}; + llvm::GlobalVariable* gv_tvm_func_call_{nullptr}; + llvm::GlobalVariable* gv_tvm_get_func_from_env_{nullptr}; + llvm::GlobalVariable* gv_tvm_api_set_last_error_{nullptr}; + std::unordered_map<std::string, llvm::GlobalVariable*> gv_func_map_; + + // context for direct dynamic lookup + llvm::Function* f_tvm_func_call_{nullptr}; + llvm::Function* f_tvm_get_func_from_env_{nullptr}; + llvm::Function* f_tvm_api_set_last_error_{nullptr}; + llvm::Function* f_tvm_register_system_symbol_{nullptr}; +}; + +void CodeGenHexagon::InitTarget(llvm::TargetMachine* tm) { + native_vector_bits_ = 64; // Assume "scalar" vectors at first. + llvm::StringRef fs = tm->getTargetFeatureString(); + size_t npos = llvm::StringRef::npos; + const auto hvx_length_feature = "+hvx-length"; // +hvx-length{64|128}b + size_t len_begin = fs.find(hvx_length_feature); + size_t len_end = len_begin != npos ? fs.find('b', len_begin) : npos; + if (len_end != npos) { + int hvx_bytes = 0; + len_begin += std::strlen(hvx_length_feature); + CHECK(!fs.substr(len_begin, len_end - len_begin).getAsInteger(10, hvx_bytes)) + << "invalid HVX length in feature string: " << fs.str(); + CHECK(hvx_bytes == 64 || hvx_bytes == 128) + << "invalid HVX vector length: " << hvx_bytes << ", should be 64 or 128"; + native_vector_bits_ = hvx_bytes * 8; + } + CodeGenLLVM::InitTarget(tm); +} + +void CodeGenHexagon::Init(const std::string& module_name, llvm::TargetMachine* tm, + llvm::LLVMContext* ctx, bool system_lib, bool dynamic_lookup, + bool target_c_runtime) { + CodeGenLLVM::Init(module_name, tm, ctx, system_lib, dynamic_lookup, false); + + func_handle_map_.clear(); + t_tvm_value_ = llvm::StructType::create({t_float64_}, "t_tvm_value"); + t_tvm_shape_index_ = llvm::Type::getIntNTy(*ctx, DataType::ShapeIndex().bits()); + t_tvm_context_ = llvm::StructType::create({t_int_, t_int_}, "t_tvm_context"); + t_tvm_type_ = llvm::StructType::create({t_int8_, t_int8_, t_int16_}, "t_tvm_type"); + t_tvm_func_handle_ = t_void_p_; + // DLTensor + t_tvm_array_ = llvm::StructType::create( + {t_void_p_, t_tvm_context_, t_int_, t_tvm_type_, t_tvm_shape_index_->getPointerTo(), + t_tvm_shape_index_->getPointerTo(), t_int64_}, + "t_tvm_array"); + + types_for_alloca_ = { + {"shape", t_tvm_shape_index_}, + {"arg_value", t_tvm_value_}, + {"arg_tcode", t_int_}, + {"array", t_tvm_array_}, + }; + + // Runtime functions. + ftype_tvm_func_call_ = llvm::FunctionType::get( + t_int_, + {t_tvm_func_handle_, t_tvm_value_->getPointerTo(), t_int_->getPointerTo(), t_int_, + t_tvm_value_->getPointerTo(), t_int_->getPointerTo()}, + false); + ftype_tvm_get_func_from_env_ = llvm::FunctionType::get( + t_int_, {t_void_p_, t_char_->getPointerTo(), t_tvm_func_handle_->getPointerTo()}, false); + ftype_tvm_api_set_last_error_ = + llvm::FunctionType::get(t_void_, {t_char_->getPointerTo()}, false); + md_tbaa_ctx_ptr_ = md_builder_->createTBAAScalarTypeNode("ctx_ptr", md_tbaa_root_); + + // initialize TVM runtime API + if (system_lib) { + // We will need this in environment for backward registration. + f_tvm_register_system_symbol_ = llvm::Function::Create( + llvm::FunctionType::get(t_int_, {t_char_->getPointerTo(), t_void_p_}, false), + llvm::Function::ExternalLinkage, "TVMBackendRegisterSystemLibSymbol", module_.get()); + } else { + f_tvm_register_system_symbol_ = nullptr; + } + this->InitGlobalContext(dynamic_lookup); +} + +llvm::Value* CodeGenHexagon::CreateCallExtern(Type ret_type, String global_symbol, + const Array<PrimExpr>& args, bool skip_first_arg) { + std::vector<llvm::Value*> arg_values; + for (size_t i = skip_first_arg; i < args.size(); ++i) { + arg_values.push_back(MakeValue(args[i])); + } + std::vector<llvm::Type*> arg_types; + for (llvm::Value* v : arg_values) { + arg_types.push_back(v->getType()); + } + llvm::FunctionType* ftype = llvm::FunctionType::get(GetLLVMType(ret_type), arg_types, false); + // Check if it is available in global function table as injected function. + auto it = gv_func_map_.find(global_symbol); + if (it != gv_func_map_.end()) { + if (it->second == nullptr) { + gv_func_map_[global_symbol] = InitContextPtr(ftype->getPointerTo(), "__" + global_symbol); + it = gv_func_map_.find(global_symbol); + } +#if TVM_LLVM_VERSION >= 90 + auto ext_callee = llvm::FunctionCallee(ftype, GetContextPtr(it->second)); +#else + auto ext_callee = GetContextPtr(it->second); +#endif + return builder_->CreateCall(ext_callee, arg_values); + } else { + llvm::Function* f = module_->getFunction(global_symbol); + if (f == nullptr) { + f = llvm::Function::Create(ftype, llvm::Function::ExternalLinkage, + global_symbol.operator llvm::StringRef(), module_.get()); + } +#if TVM_LLVM_VERSION >= 90 + auto ext_callee = llvm::FunctionCallee(f); +#else + auto ext_callee = f; +#endif + return builder_->CreateCall(ext_callee, arg_values); + } +} + +llvm::GlobalVariable* CodeGenHexagon::InitContextPtr(llvm::Type* p_type, std::string name) { + llvm::GlobalVariable* gv = new llvm::GlobalVariable( + *module_, p_type, false, llvm::GlobalValue::LinkOnceAnyLinkage, 0, name); +#if TVM_LLVM_VERSION >= 100 + gv->setAlignment(llvm::Align(data_layout_->getTypeAllocSize(p_type))); +#else + gv->setAlignment(data_layout_->getTypeAllocSize(p_type)); +#endif + gv->setInitializer(llvm::Constant::getNullValue(p_type)); + gv->setDLLStorageClass(llvm::GlobalValue::DLLStorageClassTypes::DLLExportStorageClass); + return gv; +} + +llvm::Value* CodeGenHexagon::GetContextPtr(llvm::GlobalVariable* gv) { + CHECK(gv != nullptr); +#if TVM_LLVM_VERSION >= 110 + llvm::LoadInst* faddr = builder_->CreateAlignedLoad(gv, llvm::Align(gv->getAlignment())); +#else + llvm::LoadInst* faddr = builder_->CreateAlignedLoad(gv, gv->getAlignment()); +#endif + faddr->setMetadata("tbaa", + md_builder_->createTBAAStructTagNode(md_tbaa_ctx_ptr_, md_tbaa_ctx_ptr_, 0)); + return faddr; +} + +void CodeGenHexagon::InitGlobalContext(bool dynamic_lookup) { + // Module context + gv_mod_ctx_ = InitContextPtr(t_void_p_, tvm::runtime::symbol::tvm_module_ctx); + // Register back the locations. + if (f_tvm_register_system_symbol_ != nullptr) { + export_system_symbols_.emplace_back( + std::make_pair(tvm::runtime::symbol::tvm_module_ctx, gv_mod_ctx_)); + } else { + if (!dynamic_lookup) { + gv_tvm_func_call_ = InitContextPtr(ftype_tvm_func_call_->getPointerTo(), "__TVMFuncCall"); + gv_tvm_get_func_from_env_ = InitContextPtr(ftype_tvm_get_func_from_env_->getPointerTo(), + "__TVMBackendGetFuncFromEnv"); + gv_tvm_api_set_last_error_ = + InitContextPtr(ftype_tvm_api_set_last_error_->getPointerTo(), "__TVMAPISetLastError"); + // Mark as context functions + gv_func_map_["TVMBackendAllocWorkspace"] = nullptr; + gv_func_map_["TVMBackendFreeWorkspace"] = nullptr; + } + } +} + +llvm::Value* CodeGenHexagon::RuntimeTVMFuncCall() { + if (f_tvm_func_call_ != nullptr) return f_tvm_func_call_; + return GetContextPtr(gv_tvm_func_call_); +} + +llvm::Value* CodeGenHexagon::RuntimeTVMGetFuncFromEnv() { + if (f_tvm_get_func_from_env_ != nullptr) return f_tvm_get_func_from_env_; + return GetContextPtr(gv_tvm_get_func_from_env_); +} + +llvm::Value* CodeGenHexagon::RuntimeTVMAPISetLastError() { + if (f_tvm_api_set_last_error_ != nullptr) return f_tvm_api_set_last_error_; + return GetContextPtr(gv_tvm_api_set_last_error_); +} + +llvm::BasicBlock* CodeGenHexagon::MakeCallPacked(const Array<PrimExpr>& args, llvm::Value** rvalue, + llvm::Value** ret_tcode, const DataType& r_type, + const int64_t begin, const int64_t end) { + using llvm::BasicBlock; + // using namespace tir; + std::string func_name = args[0].as<StringImmNode>()->value; + llvm::Value* handle = GetPackedFuncHandle(func_name); + // call the function + int64_t nargs = end - begin; + CHECK_GE(nargs, 0); + llvm::Value* stack_value = MakeValue(args[1]); + llvm::Value* stack_tcode = MakeValue(args[2]); + llvm::Value* arg_value = builder_->CreateInBoundsGEP( + builder_->CreatePointerCast(stack_value, t_tvm_value_->getPointerTo()), ConstInt32(begin)); + llvm::Value* arg_tcode = CreateBufferPtr(DataType::Int(32), stack_tcode, ConstInt32(begin)); + llvm::Value* ret_value = builder_->CreateInBoundsGEP( + builder_->CreatePointerCast(stack_value, t_tvm_value_->getPointerTo()), ConstInt32(end)); + *ret_tcode = CreateBufferPtr(DataType::Int(32), stack_tcode, ConstInt32(end)); +#if TVM_LLVM_VERSION >= 90 + auto call_callee = llvm::FunctionCallee(ftype_tvm_func_call_, RuntimeTVMFuncCall()); +#else + auto call_callee = RuntimeTVMFuncCall(); +#endif + BasicBlock* end_block = CheckCallSuccess(builder_->CreateCall( + call_callee, {handle, arg_value, arg_tcode, ConstInt32(nargs), ret_value, *ret_tcode})); + DataType r_api_type = tir::APIType(r_type); +#if TVM_LLVM_VERSION >= 110 + *rvalue = builder_->CreateAlignedLoad( + builder_->CreatePointerCast(ret_value, DTypeToLLVMType(r_api_type)->getPointerTo()), + llvm::Align(8)); +#else + *rvalue = builder_->CreateAlignedLoad( + builder_->CreatePointerCast(ret_value, DTypeToLLVMType(r_api_type)->getPointerTo()), 8); +#endif + *rvalue = CreateCast(r_api_type, r_type, *rvalue); + return end_block; +} + +llvm::Value* CodeGenHexagon::GetPackedFuncHandle(const std::string& fname) { + using llvm::BasicBlock; + // We will store the packed function handle in global space. + // Initialize it during the first call. + llvm::DataLayout layout(module_.get()); + uint64_t align = layout.getTypeAllocSize(t_tvm_func_handle_); + auto it = func_handle_map_.find(fname); + + llvm::GlobalVariable* hptr; + if (it == func_handle_map_.end()) { + // create global location for the handle + // create the function handle + hptr = + new llvm::GlobalVariable(*module_, t_tvm_func_handle_, false, + llvm::GlobalValue::InternalLinkage, nullptr, ".tvm_func." + fname); +#if TVM_LLVM_VERSION >= 100 + hptr->setAlignment(llvm::Align(align)); +#else + hptr->setAlignment(align); +#endif + hptr->setInitializer(llvm::Constant::getNullValue(t_tvm_func_handle_)); + func_handle_map_[fname] = hptr; + } else { + hptr = it->second; + } + // create emit codes that checks and load the function. + BasicBlock* pre_block = builder_->GetInsertBlock(); + BasicBlock* init_block = BasicBlock::Create(*ctx_, "handle_init", function_); + BasicBlock* end_block = BasicBlock::Create(*ctx_, "handle_init_end", function_); +#if TVM_LLVM_VERSION >= 110 + llvm::Value* handle = builder_->CreateAlignedLoad(hptr, llvm::Align(align)); +#else + llvm::Value* handle = builder_->CreateAlignedLoad(hptr, align); +#endif + llvm::Value* handle_not_null = + builder_->CreateICmpNE(handle, llvm::Constant::getNullValue(t_tvm_func_handle_)); + builder_->CreateCondBr(handle_not_null, end_block, init_block, md_very_likely_branch_); + // Initialize the handle if needed. + builder_->SetInsertPoint(init_block); + llvm::Value* out = + WithFunctionEntry([&]() { return builder_->CreateAlloca(t_tvm_func_handle_); }); +#if TVM_LLVM_VERSION >= 110 + llvm::LoadInst* ctx = + builder_->CreateAlignedLoad(gv_mod_ctx_, llvm::Align(gv_mod_ctx_->getAlignment())); +#else + llvm::LoadInst* ctx = builder_->CreateAlignedLoad(gv_mod_ctx_, gv_mod_ctx_->getAlignment()); +#endif + ctx->setMetadata("tbaa", + md_builder_->createTBAAStructTagNode(md_tbaa_ctx_ptr_, md_tbaa_ctx_ptr_, 0)); +#if TVM_LLVM_VERSION >= 90 + auto env_callee = llvm::FunctionCallee(ftype_tvm_get_func_from_env_, RuntimeTVMGetFuncFromEnv()); +#else + auto env_callee = RuntimeTVMGetFuncFromEnv(); +#endif + llvm::Value* retcode = builder_->CreateCall(env_callee, {ctx, GetConstString(fname), out}); + init_block = CheckCallSuccess(retcode); +#if TVM_LLVM_VERSION >= 110 + llvm::Value* loaded_handle = builder_->CreateAlignedLoad(out, llvm::Align(align)); +#else + llvm::Value* loaded_handle = builder_->CreateAlignedLoad(out, align); +#endif + // Store the handle + builder_->CreateStore(loaded_handle, hptr); + builder_->CreateBr(end_block); + // end block + builder_->SetInsertPoint(end_block); + llvm::PHINode* phi = builder_->CreatePHI(t_tvm_func_handle_, 2); + phi->addIncoming(handle, pre_block); + phi->addIncoming(loaded_handle, init_block); + return phi; +} + +llvm::Value* CodeGenHexagon::CreateCallPacked(const CallNode* op) { + // There is always a call to __tvm_set_device in a standalone op, + // and we can't have calls to packed functions, because they need + // a Module object to work (or at least TVMBackendGetFuncFromEnv + // function). + const std::string& name = op->args[0].as<StringImmNode>()->value; + if (name == "__tvm_set_device") { + return ConstInt32(0); + } + + CHECK_EQ(op->args.size(), 5U); + llvm::Value* rvalue = nullptr; + llvm::Value* ret_tcode = nullptr; + MakeCallPacked(op->args, &rvalue, &ret_tcode, op->dtype, op->args[3].as<IntImmNode>()->value, + op->args[4].as<IntImmNode>()->value); + return rvalue; +} + +llvm::Value* CodeGenHexagon::CreateCallTracePacked(const CallNode* op) { + using llvm::BasicBlock; + CHECK_EQ(op->args.size(), 6U); + llvm::Value* rvalue = nullptr; + llvm::Value* ret_tcode = nullptr; + BasicBlock* end_block = + MakeCallPacked(op->args, &rvalue, &ret_tcode, op->dtype, op->args[3].as<IntImmNode>()->value, + op->args[4].as<IntImmNode>()->value); + // Get traced value. + llvm::Value* traced_value = MakeValue(op->args[5]); + // The update_block handles case when we need to update the return value. + BasicBlock* update_block = BasicBlock::Create(*ctx_, "update_block", function_); + // The continue_block handles case when we need to return original + // traced value. + BasicBlock* continue_block = BasicBlock::Create(*ctx_, "continue_block", function_); +#if TVM_LLVM_VERSION >= 110 + llvm::Value* ret_tcode_value = builder_->CreateAlignedLoad(ret_tcode, llvm::Align(8)); +#else + llvm::Value* ret_tcode_value = builder_->CreateAlignedLoad(ret_tcode, 8); +#endif + // Check the ret_type_code and create cmp instruction. + llvm::Value* cmp = + builder_->CreateICmpNE(ret_tcode_value, llvm::ConstantInt::get(t_int_, kTVMNullptr)); + builder_->CreateCondBr(cmp, update_block, continue_block); + builder_->SetInsertPoint(update_block); + builder_->CreateBr(continue_block); + builder_->SetInsertPoint(continue_block); + // The return value depends on from what bb we come from. + llvm::PHINode* phi_rvalue = builder_->CreatePHI(traced_value->getType(), 2); + phi_rvalue->addIncoming(rvalue, update_block); + phi_rvalue->addIncoming(traced_value, end_block); + return phi_rvalue; +} + +llvm::BasicBlock* CodeGenHexagon::CheckCallSuccess(llvm::Value* retcode) { + // create emit codes that checks and load the function. + using llvm::BasicBlock; + BasicBlock* fail_block = BasicBlock::Create(*ctx_, "call_fail", function_); + BasicBlock* end_block = BasicBlock::Create(*ctx_, "call_end", function_); + llvm::Value* succ = builder_->CreateICmpEQ(retcode, llvm::ConstantInt::get(t_int_, 0)); + builder_->CreateCondBr(succ, end_block, fail_block, md_very_likely_branch_); + builder_->SetInsertPoint(fail_block); + // return the code. + builder_->CreateRet(retcode); + // otherwise set it to be new end. + builder_->SetInsertPoint(end_block); + return end_block; +} + +void CodeGenHexagon::VisitStmt_(const AssertStmtNode* op) { + using llvm::BasicBlock; + llvm::Value* cond = MakeValue(op->condition); + std::ostringstream os; + os << "Assert fail: " << op->condition; + if (op->message.as<StringImmNode>()) { + os << ", " << op->message.as<StringImmNode>()->value; + } + llvm::Value* msg = GetConstString(os.str()); + BasicBlock* fail_block = BasicBlock::Create(*ctx_, "assert_fail", function_); + BasicBlock* end_block = BasicBlock::Create(*ctx_, "assert_end", function_); + builder_->CreateCondBr(cond, end_block, fail_block, md_very_likely_branch_); + // fail condition. + builder_->SetInsertPoint(fail_block); +#if TVM_LLVM_VERSION >= 90 + auto err_callee = + llvm::FunctionCallee(ftype_tvm_api_set_last_error_, RuntimeTVMAPISetLastError()); +#else + auto err_callee = RuntimeTVMAPISetLastError(); +#endif + builder_->CreateCall(err_callee, {msg}); + builder_->CreateRet(ConstInt32(-1)); + // otherwise set it to be new end. + builder_->SetInsertPoint(end_block); + CodeGenLLVM::VisitStmt_(op); +} + +llvm::Value* CodeGenHexagon::CreateIntrinsic(const CallNode* op) { + if (op->op.same_as(builtin::tvm_call_packed_lowered())) { + return CreateCallPacked(op); + } else if (op->op.same_as(builtin::tvm_call_trace_packed_lowered())) { + return CreateCallTracePacked(op); + } else if (op->op.same_as(builtin::tvm_struct_get())) { + CHECK_EQ(op->args.size(), 3); + int kind = op->args[2].as<IntImmNode>()->value; + llvm::Value* ref = + CreateStructRefPtr(op->dtype, MakeValue(op->args[0]), MakeValue(op->args[1]), kind); + if (kind == builtin::kArrAddr) { + return builder_->CreatePointerCast(ref, t_void_p_); + } + return builder_->CreateLoad(ref); + } else if (op->op.same_as(builtin::tvm_struct_set())) { + CHECK_EQ(op->args.size(), 4); + int kind = op->args[2].as<IntImmNode>()->value; + CHECK(kind != builtin::kArrAddr); + llvm::Value* ref = CreateStructRefPtr(op->args[3].dtype(), MakeValue(op->args[0]), + MakeValue(op->args[1]), kind); + llvm::Value* value = MakeValue(op->args[3]); + if (value->getType()->isPointerTy()) { + value = builder_->CreatePointerCast(value, ref->getType()->getPointerElementType()); + } + builder_->CreateStore(value, ref); + return ConstInt32(0); + } else if (op->op.same_as(builtin::tvm_stack_alloca())) { + CHECK_EQ(op->args.size(), 2); + const std::string& name = op->args[0].as<StringImmNode>()->value; + llvm::Value* size = ConstInt32(op->args[1].as<IntImmNode>()->value); + return builder_->CreateAlloca(types_for_alloca_.at(name), size); + } else if (op->op.same_as(builtin::tvm_throw_last_error())) { + llvm::Value* neg_1 = ConstInt32(-1); + builder_->CreateRet(neg_1); + auto next_block = std::next(builder_->GetInsertBlock()->getIterator()); + llvm::BasicBlock* new_bb = llvm::BasicBlock::Create(*ctx_, "cont", function_, &*next_block); + builder_->SetInsertPoint(new_bb); + return neg_1; + } + + return CodeGenLLVM::CreateIntrinsic(op); +} + +llvm::Value* CodeGenHexagon::CreateStructRefPtr(DataType t, llvm::Value* buf, llvm::Value* index, + int kind) { + static const std::map<int, int> field_index = { + {builtin::kArrData, 0}, {builtin::kArrDeviceType, 1}, {builtin::kArrDeviceId, 1}, + {builtin::kArrNDim, 2}, {builtin::kArrTypeCode, 3}, {builtin::kArrTypeBits, 3}, + {builtin::kArrTypeLanes, 3}, {builtin::kArrShape, 4}, {builtin::kArrStrides, 5}, + {builtin::kArrByteOffset, 6}}; + static const std::map<int, int> subfield_index = { + {builtin::kArrDeviceType, 0}, {builtin::kArrDeviceId, 1}, {builtin::kArrTypeCode, 0}, + {builtin::kArrTypeBits, 1}, {builtin::kArrTypeLanes, 2}, + }; + + if (kind < builtin::kArrKindBound_) { + if (buf->getType() == t_void_p_) { + buf = builder_->CreatePointerCast(buf, t_tvm_array_->getPointerTo()); + } else { + CHECK_EQ(buf->getType(), t_tvm_array_->getPointerTo()); + } + /* The following "kinds" are accessing the members of DLTensor: + typedef struct { + void* data; kArrData + DLContext ctx; kArrDeviceType (ctx.device_type) + kArrDeviceId (ctx.device_id) + int ndim; kArrNDim + DLDataType dtype; kArrTypeCode (dtype.code) + kArrTypeBits (dtype.bits) + kArrTypeLanes (dtype.lanes) + int64_t* shape; kArrShape + int64_t* strides; kArrStrides + uint64_t byte_offset; kArrByteOffset + } DLTensor; + */ + llvm::Value* base_gep = builder_->CreateInBoundsGEP(buf, index, "base_gep"); + if (kind == builtin::kArrAddr) { + return base_gep; + } + llvm::Value* field_gep = builder_->CreateInBoundsGEP( + base_gep, {ConstInt32(0), ConstInt32(field_index.at(kind))}, "field_gep"); + switch (kind) { + // These fields have no sub-fields. + case builtin::kArrData: + case builtin::kArrNDim: + case builtin::kArrShape: + case builtin::kArrStrides: + case builtin::kArrByteOffset: + return field_gep; + } + return builder_->CreateInBoundsGEP( + field_gep, {ConstInt32(0), ConstInt32(subfield_index.at(kind))}, "subfield_gep"); + } + + if (kind == builtin::kTVMValueContent) { + /* TVMValue is a union: + typedef union { + int64_t v_int64; + double v_float64; + void* v_handle; + const char* v_str; + TVMType v_type; + TVMContext v_ctx; + } TVMValue; + */ + CHECK_EQ(t.lanes(), 1); + CHECK(t.is_handle() || t.bits() == 64); + if (t.is_int()) { + buf = builder_->CreatePointerCast(buf, t_int64_->getPointerTo()); + return builder_->CreateInBoundsGEP(buf, index); + } else if (t.is_float()) { + buf = builder_->CreatePointerCast(buf, t_float64_->getPointerTo()); + return builder_->CreateInBoundsGEP(buf, index); + } else { + CHECK(t.is_handle()); + buf = builder_->CreatePointerCast(buf, t_tvm_value_->getPointerTo()); + buf = builder_->CreateInBoundsGEP(buf, index); + return builder_->CreatePointerCast(buf, t_void_p_->getPointerTo()); + } + } + + assert(!"Unknown kind"); + return nullptr; +} + +namespace { +// Check if the function matches the TVMBackendPackedCFunc prototype. +bool UsesExportABI(const PrimFunc& f) { + if (f->attrs.defined()) { + auto it = f->attrs->dict.find("calling_conv"); + return it != f->attrs->dict.end() && + Downcast<Integer>((*it).second) == CallingConv::kCPackedFunc; + } + return false; +} + +__attribute__((unused)) std::ostream& operator<<(std::ostream& os, const llvm::Module& m) { + std::string ms; + llvm::raw_string_ostream sos(ms); + sos << m; + os << sos.str(); + return os; +} Review comment: There is a diagnostic code at [line 729](https://github.com/apache/incubator-tvm/pull/6261/files#diff-ed6a1469ccdc4ff4ac97357c972035dfR728-R729) that uses this. It's commented out, but we find it very useful, so I kept it in the code. Maybe it should be enabled with an environment variable instead, I'm not sure if TVM has an established convention for things like this. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
