llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-flang-fir-hlfir Author: Krzysztof Parzyszek (kparzysz) <details> <summary>Changes</summary> The OpenMP implementation of the ATOMIC construct will change in the near future to accommodate OpenMP 6.0. This patch separates the shared implementations to avoid interfering with OpenACC. --- Patch is 64.93 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/137517.diff 3 Files Affected: - (modified) flang/include/flang/Lower/DirectivesCommon.h (-514) - (modified) flang/lib/Lower/OpenACC.cpp (+308-12) - (modified) flang/lib/Lower/OpenMP/OpenMP.cpp (+459-14) ``````````diff diff --git a/flang/include/flang/Lower/DirectivesCommon.h b/flang/include/flang/Lower/DirectivesCommon.h index d1dbaefcd81d0..93ab2e350d035 100644 --- a/flang/include/flang/Lower/DirectivesCommon.h +++ b/flang/include/flang/Lower/DirectivesCommon.h @@ -46,520 +46,6 @@ namespace Fortran { namespace lower { -/// Populates \p hint and \p memoryOrder with appropriate clause information -/// if present on atomic construct. -static inline void genOmpAtomicHintAndMemoryOrderClauses( - Fortran::lower::AbstractConverter &converter, - const Fortran::parser::OmpAtomicClauseList &clauseList, - mlir::IntegerAttr &hint, - mlir::omp::ClauseMemoryOrderKindAttr &memoryOrder) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - for (const Fortran::parser::OmpAtomicClause &clause : clauseList.v) { - common::visit( - common::visitors{ - [&](const parser::OmpMemoryOrderClause &s) { - auto kind = common::visit( - common::visitors{ - [&](const parser::OmpClause::AcqRel &) { - return mlir::omp::ClauseMemoryOrderKind::Acq_rel; - }, - [&](const parser::OmpClause::Acquire &) { - return mlir::omp::ClauseMemoryOrderKind::Acquire; - }, - [&](const parser::OmpClause::Relaxed &) { - return mlir::omp::ClauseMemoryOrderKind::Relaxed; - }, - [&](const parser::OmpClause::Release &) { - return mlir::omp::ClauseMemoryOrderKind::Release; - }, - [&](const parser::OmpClause::SeqCst &) { - return mlir::omp::ClauseMemoryOrderKind::Seq_cst; - }, - [&](auto &&) -> mlir::omp::ClauseMemoryOrderKind { - llvm_unreachable("Unexpected clause"); - }, - }, - s.v.u); - memoryOrder = mlir::omp::ClauseMemoryOrderKindAttr::get( - firOpBuilder.getContext(), kind); - }, - [&](const parser::OmpHintClause &s) { - const auto *expr = Fortran::semantics::GetExpr(s.v); - uint64_t hintExprValue = *Fortran::evaluate::ToInt64(*expr); - hint = firOpBuilder.getI64IntegerAttr(hintExprValue); - }, - [&](const parser::OmpFailClause &) {}, - }, - clause.u); - } -} - -template <typename AtomicListT> -static void processOmpAtomicTODO(mlir::Type elementType, - [[maybe_unused]] mlir::Location loc) { - if (!elementType) - return; - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - assert(fir::isa_trivial(fir::unwrapRefType(elementType)) && - "is supported type for omp atomic"); - } -} - -/// Used to generate atomic.read operation which is created in existing -/// location set by builder. -template <typename AtomicListT> -static inline void genOmpAccAtomicCaptureStatement( - Fortran::lower::AbstractConverter &converter, mlir::Value fromAddress, - mlir::Value toAddress, - [[maybe_unused]] const AtomicListT *leftHandClauseList, - [[maybe_unused]] const AtomicListT *rightHandClauseList, - mlir::Type elementType, mlir::Location loc) { - // Generate `atomic.read` operation for atomic assigment statements - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - - processOmpAtomicTODO<AtomicListT>(elementType, loc); - - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // If no hint clause is specified, the effect is as if - // hint(omp_sync_hint_none) had been specified. - mlir::IntegerAttr hint = nullptr; - - mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr; - if (leftHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, - hint, memoryOrder); - if (rightHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, - hint, memoryOrder); - firOpBuilder.create<mlir::omp::AtomicReadOp>( - loc, fromAddress, toAddress, mlir::TypeAttr::get(elementType), hint, - memoryOrder); - } else { - firOpBuilder.create<mlir::acc::AtomicReadOp>( - loc, fromAddress, toAddress, mlir::TypeAttr::get(elementType)); - } -} - -/// Used to generate atomic.write operation which is created in existing -/// location set by builder. -template <typename AtomicListT> -static inline void genOmpAccAtomicWriteStatement( - Fortran::lower::AbstractConverter &converter, mlir::Value lhsAddr, - mlir::Value rhsExpr, [[maybe_unused]] const AtomicListT *leftHandClauseList, - [[maybe_unused]] const AtomicListT *rightHandClauseList, mlir::Location loc, - mlir::Value *evaluatedExprValue = nullptr) { - // Generate `atomic.write` operation for atomic assignment statements - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - - mlir::Type varType = fir::unwrapRefType(lhsAddr.getType()); - // Create a conversion outside the capture block. - auto insertionPoint = firOpBuilder.saveInsertionPoint(); - firOpBuilder.setInsertionPointAfter(rhsExpr.getDefiningOp()); - rhsExpr = firOpBuilder.createConvert(loc, varType, rhsExpr); - firOpBuilder.restoreInsertionPoint(insertionPoint); - - processOmpAtomicTODO<AtomicListT>(varType, loc); - - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // If no hint clause is specified, the effect is as if - // hint(omp_sync_hint_none) had been specified. - mlir::IntegerAttr hint = nullptr; - mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr; - if (leftHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, - hint, memoryOrder); - if (rightHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, - hint, memoryOrder); - firOpBuilder.create<mlir::omp::AtomicWriteOp>(loc, lhsAddr, rhsExpr, hint, - memoryOrder); - } else { - firOpBuilder.create<mlir::acc::AtomicWriteOp>(loc, lhsAddr, rhsExpr); - } -} - -/// Used to generate atomic.update operation which is created in existing -/// location set by builder. -template <typename AtomicListT> -static inline void genOmpAccAtomicUpdateStatement( - Fortran::lower::AbstractConverter &converter, mlir::Value lhsAddr, - mlir::Type varType, const Fortran::parser::Variable &assignmentStmtVariable, - const Fortran::parser::Expr &assignmentStmtExpr, - [[maybe_unused]] const AtomicListT *leftHandClauseList, - [[maybe_unused]] const AtomicListT *rightHandClauseList, mlir::Location loc, - mlir::Operation *atomicCaptureOp = nullptr) { - // Generate `atomic.update` operation for atomic assignment statements - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - mlir::Location currentLocation = converter.getCurrentLocation(); - - // Create the omp.atomic.update or acc.atomic.update operation - // - // func.func @_QPsb() { - // %0 = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFsbEa"} - // %1 = fir.alloca i32 {bindc_name = "b", uniq_name = "_QFsbEb"} - // %2 = fir.load %1 : !fir.ref<i32> - // omp.atomic.update %0 : !fir.ref<i32> { - // ^bb0(%arg0: i32): - // %3 = arith.addi %arg0, %2 : i32 - // omp.yield(%3 : i32) - // } - // return - // } - - auto getArgExpression = - [](std::list<parser::ActualArgSpec>::const_iterator it) { - const auto &arg{std::get<parser::ActualArg>((*it).t)}; - const auto *parserExpr{ - std::get_if<common::Indirection<parser::Expr>>(&arg.u)}; - return parserExpr; - }; - - // Lower any non atomic sub-expression before the atomic operation, and - // map its lowered value to the semantic representation. - Fortran::lower::ExprToValueMap exprValueOverrides; - // Max and min intrinsics can have a list of Args. Hence we need a list - // of nonAtomicSubExprs to hoist. Currently, only the load is hoisted. - llvm::SmallVector<const Fortran::lower::SomeExpr *> nonAtomicSubExprs; - Fortran::common::visit( - Fortran::common::visitors{ - [&](const common::Indirection<parser::FunctionReference> &funcRef) - -> void { - const auto &args{std::get<std::list<parser::ActualArgSpec>>( - funcRef.value().v.t)}; - std::list<parser::ActualArgSpec>::const_iterator beginIt = - args.begin(); - std::list<parser::ActualArgSpec>::const_iterator endIt = args.end(); - const auto *exprFirst{getArgExpression(beginIt)}; - if (exprFirst && exprFirst->value().source == - assignmentStmtVariable.GetSource()) { - // Add everything except the first - beginIt++; - } else { - // Add everything except the last - endIt--; - } - std::list<parser::ActualArgSpec>::const_iterator it; - for (it = beginIt; it != endIt; it++) { - const common::Indirection<parser::Expr> *expr = - getArgExpression(it); - if (expr) - nonAtomicSubExprs.push_back(Fortran::semantics::GetExpr(*expr)); - } - }, - [&](const auto &op) -> void { - using T = std::decay_t<decltype(op)>; - if constexpr (std::is_base_of< - Fortran::parser::Expr::IntrinsicBinary, - T>::value) { - const auto &exprLeft{std::get<0>(op.t)}; - const auto &exprRight{std::get<1>(op.t)}; - if (exprLeft.value().source == assignmentStmtVariable.GetSource()) - nonAtomicSubExprs.push_back( - Fortran::semantics::GetExpr(exprRight)); - else - nonAtomicSubExprs.push_back( - Fortran::semantics::GetExpr(exprLeft)); - } - }, - }, - assignmentStmtExpr.u); - StatementContext nonAtomicStmtCtx; - if (!nonAtomicSubExprs.empty()) { - // Generate non atomic part before all the atomic operations. - auto insertionPoint = firOpBuilder.saveInsertionPoint(); - if (atomicCaptureOp) - firOpBuilder.setInsertionPoint(atomicCaptureOp); - mlir::Value nonAtomicVal; - for (auto *nonAtomicSubExpr : nonAtomicSubExprs) { - nonAtomicVal = fir::getBase(converter.genExprValue( - currentLocation, *nonAtomicSubExpr, nonAtomicStmtCtx)); - exprValueOverrides.try_emplace(nonAtomicSubExpr, nonAtomicVal); - } - if (atomicCaptureOp) - firOpBuilder.restoreInsertionPoint(insertionPoint); - } - - mlir::Operation *atomicUpdateOp = nullptr; - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // If no hint clause is specified, the effect is as if - // hint(omp_sync_hint_none) had been specified. - mlir::IntegerAttr hint = nullptr; - mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr; - if (leftHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, - hint, memoryOrder); - if (rightHandClauseList) - genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, - hint, memoryOrder); - atomicUpdateOp = firOpBuilder.create<mlir::omp::AtomicUpdateOp>( - currentLocation, lhsAddr, hint, memoryOrder); - } else { - atomicUpdateOp = firOpBuilder.create<mlir::acc::AtomicUpdateOp>( - currentLocation, lhsAddr); - } - - processOmpAtomicTODO<AtomicListT>(varType, loc); - - llvm::SmallVector<mlir::Type> varTys = {varType}; - llvm::SmallVector<mlir::Location> locs = {currentLocation}; - firOpBuilder.createBlock(&atomicUpdateOp->getRegion(0), {}, varTys, locs); - mlir::Value val = - fir::getBase(atomicUpdateOp->getRegion(0).front().getArgument(0)); - - exprValueOverrides.try_emplace( - Fortran::semantics::GetExpr(assignmentStmtVariable), val); - { - // statement context inside the atomic block. - converter.overrideExprValues(&exprValueOverrides); - Fortran::lower::StatementContext atomicStmtCtx; - mlir::Value rhsExpr = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(assignmentStmtExpr), atomicStmtCtx)); - mlir::Value convertResult = - firOpBuilder.createConvert(currentLocation, varType, rhsExpr); - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - firOpBuilder.create<mlir::omp::YieldOp>(currentLocation, convertResult); - } else { - firOpBuilder.create<mlir::acc::YieldOp>(currentLocation, convertResult); - } - converter.resetExprOverrides(); - } - firOpBuilder.setInsertionPointAfter(atomicUpdateOp); -} - -/// Processes an atomic construct with write clause. -template <typename AtomicT, typename AtomicListT> -void genOmpAccAtomicWrite(Fortran::lower::AbstractConverter &converter, - const AtomicT &atomicWrite, mlir::Location loc) { - const AtomicListT *rightHandClauseList = nullptr; - const AtomicListT *leftHandClauseList = nullptr; - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // Get the address of atomic read operands. - rightHandClauseList = &std::get<2>(atomicWrite.t); - leftHandClauseList = &std::get<0>(atomicWrite.t); - } - - const Fortran::parser::AssignmentStmt &stmt = - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicWrite.t) - .statement; - const Fortran::evaluate::Assignment &assign = *stmt.typedAssignment->v; - Fortran::lower::StatementContext stmtCtx; - // Get the value and address of atomic write operands. - mlir::Value rhsExpr = - fir::getBase(converter.genExprValue(assign.rhs, stmtCtx)); - mlir::Value lhsAddr = - fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx)); - genOmpAccAtomicWriteStatement(converter, lhsAddr, rhsExpr, leftHandClauseList, - rightHandClauseList, loc); -} - -/// Processes an atomic construct with read clause. -template <typename AtomicT, typename AtomicListT> -void genOmpAccAtomicRead(Fortran::lower::AbstractConverter &converter, - const AtomicT &atomicRead, mlir::Location loc) { - const AtomicListT *rightHandClauseList = nullptr; - const AtomicListT *leftHandClauseList = nullptr; - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // Get the address of atomic read operands. - rightHandClauseList = &std::get<2>(atomicRead.t); - leftHandClauseList = &std::get<0>(atomicRead.t); - } - - const auto &assignmentStmtExpr = std::get<Fortran::parser::Expr>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicRead.t) - .statement.t); - const auto &assignmentStmtVariable = std::get<Fortran::parser::Variable>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicRead.t) - .statement.t); - - Fortran::lower::StatementContext stmtCtx; - const Fortran::semantics::SomeExpr &fromExpr = - *Fortran::semantics::GetExpr(assignmentStmtExpr); - mlir::Type elementType = converter.genType(fromExpr); - mlir::Value fromAddress = - fir::getBase(converter.genExprAddr(fromExpr, stmtCtx)); - mlir::Value toAddress = fir::getBase(converter.genExprAddr( - *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx)); - genOmpAccAtomicCaptureStatement(converter, fromAddress, toAddress, - leftHandClauseList, rightHandClauseList, - elementType, loc); -} - -/// Processes an atomic construct with update clause. -template <typename AtomicT, typename AtomicListT> -void genOmpAccAtomicUpdate(Fortran::lower::AbstractConverter &converter, - const AtomicT &atomicUpdate, mlir::Location loc) { - const AtomicListT *rightHandClauseList = nullptr; - const AtomicListT *leftHandClauseList = nullptr; - if constexpr (std::is_same<AtomicListT, - Fortran::parser::OmpAtomicClauseList>()) { - // Get the address of atomic read operands. - rightHandClauseList = &std::get<2>(atomicUpdate.t); - leftHandClauseList = &std::get<0>(atomicUpdate.t); - } - - const auto &assignmentStmtExpr = std::get<Fortran::parser::Expr>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicUpdate.t) - .statement.t); - const auto &assignmentStmtVariable = std::get<Fortran::parser::Variable>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicUpdate.t) - .statement.t); - - Fortran::lower::StatementContext stmtCtx; - mlir::Value lhsAddr = fir::getBase(converter.genExprAddr( - *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx)); - mlir::Type varType = fir::unwrapRefType(lhsAddr.getType()); - genOmpAccAtomicUpdateStatement<AtomicListT>( - converter, lhsAddr, varType, assignmentStmtVariable, assignmentStmtExpr, - leftHandClauseList, rightHandClauseList, loc); -} - -/// Processes an atomic construct with no clause - which implies update clause. -template <typename AtomicT, typename AtomicListT> -void genOmpAtomic(Fortran::lower::AbstractConverter &converter, - const AtomicT &atomicConstruct, mlir::Location loc) { - const AtomicListT &atomicClauseList = - std::get<AtomicListT>(atomicConstruct.t); - const auto &assignmentStmtExpr = std::get<Fortran::parser::Expr>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicConstruct.t) - .statement.t); - const auto &assignmentStmtVariable = std::get<Fortran::parser::Variable>( - std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>( - atomicConstruct.t) - .statement.t); - Fortran::lower::StatementContext stmtCtx; - mlir::Value lhsAddr = fir::getBase(converter.genExprAddr( - *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx)); - mlir::Type varType = fir::unwrapRefType(lhsAddr.getType()); - // If atomic-clause is not present on the construct, the behaviour is as if - // the update clause is specified (for both OpenMP and OpenACC). - genOmpAccAtomicUpdateStatement<AtomicListT>( - converter, lhsAddr, varType, assignmentStmtVariable, assignmentStmtExpr, - &atomicClauseList, nullptr, loc); -} - -/// Processes an atomic construct with capture clause. -template <typename AtomicT, typename AtomicListT> -void genOmpAccAtomicCapture(Fortran::lower::AbstractConverter &converter, - const AtomicT &atomicCapture, mlir::Location loc) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - - const Fortran::parser::AssignmentStmt &stmt1 = - std::get<typename AtomicT::Stmt1>(atomicCapture.t).v.statement; - const Fortran::evaluate::Assignment &assign1 = *stmt1.typedAssignment->v; - const auto &stmt1Var{std::get<Fortran::parser::Variable>(stmt1.t)}; - const auto &stmt1Expr{std::get<Fortran::parser::Expr>(stmt1.t)}; - const Fortran::parser::AssignmentStmt &stmt2 = - std::get<typename AtomicT::Stmt2>(atomicCapture.t).v.statement; - const Fortran::evaluate::Assignment &assign2 = *stmt2.typedAssignment->v; - const auto &stmt2Var{std::get<Fortran::parser::Variable>(stmt2.t)}; - const auto &stmt2Expr{std::get<Fortran::parser::Expr>(stmt2.t)}; - - // Pre-evaluate expressions to be used in the various operations inside - // `atomic.capture` since it is not desirable to have anything other than - // a `atomic.read`, `atomic.write`, or `atomic.updat... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/137517 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
