diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h index bbe3b01fdb29d..dad841863ac00 100644 --- a/flang/include/flang/Lower/OpenACC.h +++ b/flang/include/flang/Lower/OpenACC.h @@ -85,7 +85,7 @@ void genOpenACCRoutineConstruct( /// Get a acc.private.recipe op for the given type or create it if it does not /// exist yet. -mlir::acc::PrivateRecipeOp createOrGetPrivateRecipe(mlir::OpBuilder &, +mlir::acc::PrivateRecipeOp createOrGetPrivateRecipe(fir::FirOpBuilder &, llvm::StringRef, mlir::Location, mlir::Type); @@ -99,7 +99,7 @@ createOrGetReductionRecipe(fir::FirOpBuilder &, llvm::StringRef, mlir::Location, /// Get a acc.firstprivate.recipe op for the given type or create it if it does /// not exist yet. mlir::acc::FirstprivateRecipeOp -createOrGetFirstprivateRecipe(mlir::OpBuilder &, llvm::StringRef, +createOrGetFirstprivateRecipe(fir::FirOpBuilder &, llvm::StringRef, mlir::Location, mlir::Type, llvm::SmallVector &); diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index e1918288d6de3..bc94e860ff10b 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -843,22 +843,147 @@ fir::ShapeOp genShapeOp(mlir::OpBuilder &builder, fir::SequenceType seqTy, return builder.create(loc, extents); } +/// Get the initial value for reduction operator. +template +static R getReductionInitValue(mlir::acc::ReductionOperator op, mlir::Type ty) { + if (op == mlir::acc::ReductionOperator::AccMin) { + // min init value -> largest + if constexpr (std::is_same_v) { + assert(ty.isIntOrIndex() && "expect integer or index type"); + return llvm::APInt::getSignedMaxValue(ty.getIntOrFloatBitWidth()); + } + if constexpr (std::is_same_v) { + auto floatTy = mlir::dyn_cast_or_null(ty); + assert(floatTy && "expect float type"); + return llvm::APFloat::getLargest(floatTy.getFloatSemantics(), + /*negative=*/false); + } + } else if (op == mlir::acc::ReductionOperator::AccMax) { + // max init value -> smallest + if constexpr (std::is_same_v) { + assert(ty.isIntOrIndex() && "expect integer or index type"); + return llvm::APInt::getSignedMinValue(ty.getIntOrFloatBitWidth()); + } + if constexpr (std::is_same_v) { + auto floatTy = mlir::dyn_cast_or_null(ty); + assert(floatTy && "expect float type"); + return llvm::APFloat::getSmallest(floatTy.getFloatSemantics(), + /*negative=*/true); + } + } else if (op == mlir::acc::ReductionOperator::AccIand) { + if constexpr (std::is_same_v) { + assert(ty.isIntOrIndex() && "expect integer type"); + unsigned bits = ty.getIntOrFloatBitWidth(); + return llvm::APInt::getAllOnes(bits); + } + } else { + assert(op != mlir::acc::ReductionOperator::AccNone); + // +, ior, ieor init value -> 0 + // * init value -> 1 + int64_t value = (op == mlir::acc::ReductionOperator::AccMul) ? 1 : 0; + if constexpr (std::is_same_v) { + assert(ty.isIntOrIndex() && "expect integer or index type"); + return llvm::APInt(ty.getIntOrFloatBitWidth(), value, true); + } + + if constexpr (std::is_same_v) { + assert(mlir::isa(ty) && "expect float type"); + auto floatTy = mlir::dyn_cast(ty); + return llvm::APFloat(floatTy.getFloatSemantics(), value); + } + + if constexpr (std::is_same_v) + return value; + } + llvm_unreachable("OpenACC reduction unsupported type"); +} + +/// Return a constant with the initial value for the reduction operator and +/// type combination. +static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder, + mlir::Location loc, mlir::Type ty, + mlir::acc::ReductionOperator op) { + if (op == mlir::acc::ReductionOperator::AccLand || + op == mlir::acc::ReductionOperator::AccLor || + op == mlir::acc::ReductionOperator::AccEqv || + op == mlir::acc::ReductionOperator::AccNeqv) { + assert(mlir::isa(ty) && "expect fir.logical type"); + bool value = true; // .true. for .and. and .eqv. + if (op == mlir::acc::ReductionOperator::AccLor || + op == mlir::acc::ReductionOperator::AccNeqv) + value = false; // .false. for .or. and .neqv. + return builder.createBool(loc, value); + } + if (ty.isIntOrIndex()) + return builder.create( + loc, ty, + builder.getIntegerAttr(ty, getReductionInitValue(op, ty))); + if (op == mlir::acc::ReductionOperator::AccMin || + op == mlir::acc::ReductionOperator::AccMax) { + if (mlir::isa(ty)) + llvm::report_fatal_error( + "min/max reduction not supported for complex type"); + if (auto floatTy = mlir::dyn_cast_or_null(ty)) + return builder.create( + loc, ty, + builder.getFloatAttr(ty, + getReductionInitValue(op, ty))); + } else if (auto floatTy = mlir::dyn_cast_or_null(ty)) { + return builder.create( + loc, ty, + builder.getFloatAttr(ty, getReductionInitValue(op, ty))); + } else if (auto cmplxTy = mlir::dyn_cast_or_null(ty)) { + mlir::Type floatTy = cmplxTy.getElementType(); + mlir::Value realInit = builder.createRealConstant( + loc, floatTy, getReductionInitValue(op, cmplxTy)); + mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0); + return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit, + imagInit); + } + + if (auto seqTy = mlir::dyn_cast(ty)) + return getReductionInitValue(builder, loc, seqTy.getEleTy(), op); + + if (auto boxTy = mlir::dyn_cast(ty)) + return getReductionInitValue(builder, loc, boxTy.getEleTy(), op); + + if (auto heapTy = mlir::dyn_cast(ty)) + return getReductionInitValue(builder, loc, heapTy.getEleTy(), op); + + if (auto ptrTy = mlir::dyn_cast(ty)) + return getReductionInitValue(builder, loc, ptrTy.getEleTy(), op); + + llvm::report_fatal_error("Unsupported OpenACC reduction type"); +} + template -static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe, - mlir::Type argTy, mlir::Location loc) { +static void genPrivateLikeInitRegion(fir::FirOpBuilder &builder, + RecipeOp recipe, mlir::Type argTy, + mlir::Location loc, + mlir::Value initValue) { mlir::Value retVal = recipe.getInitRegion().front().getArgument(0); mlir::Type unwrappedTy = fir::unwrapRefType(argTy); + llvm::StringRef initName; + if constexpr (std::is_same_v) + initName = accReductionInitName; + else + initName = accPrivateInitName; + auto getDeclareOpForType = [&](mlir::Type ty) -> hlfir::DeclareOp { auto alloca = builder.create(loc, ty); return builder.create( - loc, alloca, accPrivateInitName, /*shape=*/nullptr, - llvm::ArrayRef{}, /*dummy_scope=*/nullptr, - fir::FortranVariableFlagsAttr{}); + loc, alloca, initName, /*shape=*/nullptr, llvm::ArrayRef{}, + /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{}); }; if (fir::isa_trivial(unwrappedTy)) { - retVal = getDeclareOpForType(unwrappedTy).getBase(); + auto declareOp = getDeclareOpForType(unwrappedTy); + if (initValue) { + auto convert = builder.createConvert(loc, unwrappedTy, initValue); + builder.create(loc, convert, declareOp.getBase()); + } + retVal = declareOp.getBase(); } else if (auto seqTy = mlir::dyn_cast_or_null(unwrappedTy)) { if (fir::isa_trivial(seqTy.getEleTy())) { @@ -877,8 +1002,34 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe, auto alloca = builder.create( loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents); auto declareOp = builder.create( - loc, alloca, accPrivateInitName, shape, llvm::ArrayRef{}, + loc, alloca, initName, shape, llvm::ArrayRef{}, /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{}); + + if (initValue) { + mlir::Type idxTy = builder.getIndexType(); + mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy()); + llvm::SmallVector loops; + llvm::SmallVector ivs; + + if (seqTy.hasDynamicExtents()) { + builder.create(loc, initValue, declareOp.getBase()); + } else { + for (auto ext : seqTy.getShape()) { + auto lb = builder.createIntegerConstant(loc, idxTy, 0); + auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1); + auto step = builder.createIntegerConstant(loc, idxTy, 1); + auto loop = builder.create(loc, lb, ub, step, + /*unordered=*/false); + builder.setInsertionPointToStart(loop.getBody()); + loops.push_back(loop); + ivs.push_back(loop.getInductionVar()); + } + auto coord = builder.create( + loc, refTy, declareOp.getBase(), ivs); + builder.create(loc, initValue, coord); + builder.setInsertionPointAfter(loops[0]); + } + } retVal = declareOp.getBase(); } } else if (auto boxTy = @@ -909,25 +1060,29 @@ static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe, retVal = temp; } } else { - TODO(loc, "Unsupported boxed type in OpenACC privatization"); + TODO(loc, "Unsupported boxed type for OpenACC private-like recipe"); + } + if (initValue) { + builder.create(loc, initValue, retVal); } } builder.create(loc, retVal); } -mlir::acc::PrivateRecipeOp -Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder, - llvm::StringRef recipeName, - mlir::Location loc, mlir::Type ty) { - mlir::ModuleOp mod = - builder.getBlock()->getParent()->getParentOfType(); - if (auto recipe = mod.lookupSymbol(recipeName)) - return recipe; - - auto crtPos = builder.saveInsertionPoint(); +template +static RecipeOp genRecipeOp( + fir::FirOpBuilder &builder, mlir::ModuleOp mod, llvm::StringRef recipeName, + mlir::Location loc, mlir::Type ty, + mlir::acc::ReductionOperator op = mlir::acc::ReductionOperator::AccNone) { mlir::OpBuilder modBuilder(mod.getBodyRegion()); - auto recipe = - modBuilder.create(loc, recipeName, ty); + RecipeOp recipe; + if constexpr (std::is_same_v) { + recipe = modBuilder.create(loc, recipeName, + ty, op); + } else { + recipe = modBuilder.create(loc, recipeName, ty); + } + llvm::SmallVector argsTy{ty}; llvm::SmallVector argsLoc{loc}; if (auto refTy = mlir::dyn_cast_or_null(ty)) { @@ -945,9 +1100,28 @@ Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder, builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), argsTy, argsLoc); builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); - genPrivateLikeInitRegion(builder, recipe, ty, - loc); - builder.restoreInsertionPoint(crtPos); + mlir::Value initValue; + if constexpr (std::is_same_v) { + assert(op != mlir::acc::ReductionOperator::AccNone); + initValue = getReductionInitValue(builder, loc, fir::unwrapRefType(ty), op); + } + genPrivateLikeInitRegion(builder, recipe, ty, loc, initValue); + return recipe; +} + +mlir::acc::PrivateRecipeOp +Fortran::lower::createOrGetPrivateRecipe(fir::FirOpBuilder &builder, + llvm::StringRef recipeName, + mlir::Location loc, mlir::Type ty) { + mlir::ModuleOp mod = + builder.getBlock()->getParent()->getParentOfType(); + if (auto recipe = mod.lookupSymbol(recipeName)) + return recipe; + + auto ip = builder.saveInsertionPoint(); + auto recipe = genRecipeOp(builder, mod, + recipeName, loc, ty); + builder.restoreInsertionPoint(ip); return recipe; } @@ -1064,7 +1238,7 @@ static hlfir::Entity genDesignateWithTriplets( } mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( - mlir::OpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc, + fir::FirOpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc, mlir::Type ty, llvm::SmallVector &bounds) { mlir::ModuleOp mod = builder.getBlock()->getParent()->getParentOfType(); @@ -1072,28 +1246,9 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( mod.lookupSymbol(recipeName)) return recipe; - auto crtPos = builder.saveInsertionPoint(); - mlir::OpBuilder modBuilder(mod.getBodyRegion()); - auto recipe = - modBuilder.create(loc, recipeName, ty); - llvm::SmallVector initArgsTy{ty}; - llvm::SmallVector initArgsLoc{loc}; - auto refTy = fir::unwrapRefType(ty); - if (auto seqTy = mlir::dyn_cast_or_null(refTy)) { - if (seqTy.hasDynamicExtents()) { - mlir::Type idxTy = builder.getIndexType(); - for (unsigned i = 0; i < seqTy.getDimension(); ++i) { - initArgsTy.push_back(idxTy); - initArgsLoc.push_back(loc); - } - } - } - builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), - initArgsTy, initArgsLoc); - builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); - genPrivateLikeInitRegion(builder, recipe, ty, - loc); - + auto ip = builder.saveInsertionPoint(); + auto recipe = genRecipeOp( + builder, mod, recipeName, loc, ty); bool allConstantBound = areAllBoundConstant(bounds); llvm::SmallVector argsTy{ty, ty}; llvm::SmallVector argsLoc{loc, loc}; @@ -1167,7 +1322,7 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( } builder.create(loc); - builder.restoreInsertionPoint(crtPos); + builder.restoreInsertionPoint(ip); return recipe; } @@ -1326,188 +1481,6 @@ getReductionOperator(const Fortran::parser::ReductionOperator &op) { llvm_unreachable("unexpected reduction operator"); } -/// Get the initial value for reduction operator. -template -static R getReductionInitValue(mlir::acc::ReductionOperator op, mlir::Type ty) { - if (op == mlir::acc::ReductionOperator::AccMin) { - // min init value -> largest - if constexpr (std::is_same_v) { - assert(ty.isIntOrIndex() && "expect integer or index type"); - return llvm::APInt::getSignedMaxValue(ty.getIntOrFloatBitWidth()); - } - if constexpr (std::is_same_v) { - auto floatTy = mlir::dyn_cast_or_null(ty); - assert(floatTy && "expect float type"); - return llvm::APFloat::getLargest(floatTy.getFloatSemantics(), - /*negative=*/false); - } - } else if (op == mlir::acc::ReductionOperator::AccMax) { - // max init value -> smallest - if constexpr (std::is_same_v) { - assert(ty.isIntOrIndex() && "expect integer or index type"); - return llvm::APInt::getSignedMinValue(ty.getIntOrFloatBitWidth()); - } - if constexpr (std::is_same_v) { - auto floatTy = mlir::dyn_cast_or_null(ty); - assert(floatTy && "expect float type"); - return llvm::APFloat::getSmallest(floatTy.getFloatSemantics(), - /*negative=*/true); - } - } else if (op == mlir::acc::ReductionOperator::AccIand) { - if constexpr (std::is_same_v) { - assert(ty.isIntOrIndex() && "expect integer type"); - unsigned bits = ty.getIntOrFloatBitWidth(); - return llvm::APInt::getAllOnes(bits); - } - } else { - // +, ior, ieor init value -> 0 - // * init value -> 1 - int64_t value = (op == mlir::acc::ReductionOperator::AccMul) ? 1 : 0; - if constexpr (std::is_same_v) { - assert(ty.isIntOrIndex() && "expect integer or index type"); - return llvm::APInt(ty.getIntOrFloatBitWidth(), value, true); - } - - if constexpr (std::is_same_v) { - assert(mlir::isa(ty) && "expect float type"); - auto floatTy = mlir::dyn_cast(ty); - return llvm::APFloat(floatTy.getFloatSemantics(), value); - } - - if constexpr (std::is_same_v) - return value; - } - llvm_unreachable("OpenACC reduction unsupported type"); -} - -/// Return a constant with the initial value for the reduction operator and -/// type combination. -static mlir::Value getReductionInitValue(fir::FirOpBuilder &builder, - mlir::Location loc, mlir::Type ty, - mlir::acc::ReductionOperator op) { - if (op == mlir::acc::ReductionOperator::AccLand || - op == mlir::acc::ReductionOperator::AccLor || - op == mlir::acc::ReductionOperator::AccEqv || - op == mlir::acc::ReductionOperator::AccNeqv) { - assert(mlir::isa(ty) && "expect fir.logical type"); - bool value = true; // .true. for .and. and .eqv. - if (op == mlir::acc::ReductionOperator::AccLor || - op == mlir::acc::ReductionOperator::AccNeqv) - value = false; // .false. for .or. and .neqv. - return builder.createBool(loc, value); - } - if (ty.isIntOrIndex()) - return builder.create( - loc, ty, - builder.getIntegerAttr(ty, getReductionInitValue(op, ty))); - if (op == mlir::acc::ReductionOperator::AccMin || - op == mlir::acc::ReductionOperator::AccMax) { - if (mlir::isa(ty)) - llvm::report_fatal_error( - "min/max reduction not supported for complex type"); - if (auto floatTy = mlir::dyn_cast_or_null(ty)) - return builder.create( - loc, ty, - builder.getFloatAttr(ty, - getReductionInitValue(op, ty))); - } else if (auto floatTy = mlir::dyn_cast_or_null(ty)) { - return builder.create( - loc, ty, - builder.getFloatAttr(ty, getReductionInitValue(op, ty))); - } else if (auto cmplxTy = mlir::dyn_cast_or_null(ty)) { - mlir::Type floatTy = cmplxTy.getElementType(); - mlir::Value realInit = builder.createRealConstant( - loc, floatTy, getReductionInitValue(op, cmplxTy)); - mlir::Value imagInit = builder.createRealConstant(loc, floatTy, 0.0); - return fir::factory::Complex{builder, loc}.createComplex(cmplxTy, realInit, - imagInit); - } - - if (auto seqTy = mlir::dyn_cast(ty)) - return getReductionInitValue(builder, loc, seqTy.getEleTy(), op); - - if (auto boxTy = mlir::dyn_cast(ty)) - return getReductionInitValue(builder, loc, boxTy.getEleTy(), op); - - if (auto heapTy = mlir::dyn_cast(ty)) - return getReductionInitValue(builder, loc, heapTy.getEleTy(), op); - - if (auto ptrTy = mlir::dyn_cast(ty)) - return getReductionInitValue(builder, loc, ptrTy.getEleTy(), op); - - llvm::report_fatal_error("Unsupported OpenACC reduction type"); -} - -static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder, - mlir::Location loc, mlir::Type ty, - mlir::acc::ReductionOperator op) { - ty = fir::unwrapRefType(ty); - mlir::Value initValue = getReductionInitValue(builder, loc, ty, op); - if (fir::isa_trivial(ty)) { - mlir::Value alloca = builder.create(loc, ty); - auto declareOp = builder.create( - loc, alloca, accReductionInitName, /*shape=*/nullptr, - llvm::ArrayRef{}, /*dummy_scope=*/nullptr, - fir::FortranVariableFlagsAttr{}); - builder.create(loc, builder.createConvert(loc, ty, initValue), - declareOp.getBase()); - return declareOp.getBase(); - } else if (auto seqTy = mlir::dyn_cast_or_null(ty)) { - if (fir::isa_trivial(seqTy.getEleTy())) { - mlir::Value shape; - auto extents = builder.getBlock()->getArguments().drop_front(1); - if (seqTy.hasDynamicExtents()) - shape = builder.create(loc, extents); - else - shape = genShapeOp(builder, seqTy, loc); - mlir::Value alloca = builder.create( - loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents); - auto declareOp = builder.create( - loc, alloca, accReductionInitName, shape, - llvm::ArrayRef{}, /*dummy_scope=*/nullptr, - fir::FortranVariableFlagsAttr{}); - mlir::Type idxTy = builder.getIndexType(); - mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy()); - llvm::SmallVector loops; - llvm::SmallVector ivs; - - if (seqTy.hasDynamicExtents()) { - builder.create(loc, initValue, declareOp.getBase()); - return declareOp.getBase(); - } - for (auto ext : seqTy.getShape()) { - auto lb = builder.createIntegerConstant(loc, idxTy, 0); - auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1); - auto step = builder.createIntegerConstant(loc, idxTy, 1); - auto loop = builder.create(loc, lb, ub, step, - /*unordered=*/false); - builder.setInsertionPointToStart(loop.getBody()); - loops.push_back(loop); - ivs.push_back(loop.getInductionVar()); - } - auto coord = builder.create(loc, refTy, - declareOp.getBase(), ivs); - builder.create(loc, initValue, coord); - builder.setInsertionPointAfter(loops[0]); - return declareOp.getBase(); - } - } else if (auto boxTy = mlir::dyn_cast_or_null(ty)) { - mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy()); - if (!fir::isa_trivial(innerTy) && !mlir::isa(innerTy)) - TODO(loc, "Unsupported boxed type for reduction"); - // Create the private copy from the initial fir.box. - hlfir::Entity source = hlfir::Entity{builder.getBlock()->getArgument(0)}; - auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source); - mlir::Value newBox = temp; - if (!mlir::isa(temp.getType())) { - newBox = builder.create(loc, boxTy, temp); - } - builder.create(loc, initValue, newBox); - return newBox; - } - llvm::report_fatal_error("Unsupported OpenACC reduction type"); -} - template static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value value1, @@ -1799,27 +1772,10 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe( if (auto recipe = mod.lookupSymbol(recipeName)) return recipe; - auto crtPos = builder.saveInsertionPoint(); - mlir::OpBuilder modBuilder(mod.getBodyRegion()); - auto recipe = - modBuilder.create(loc, recipeName, ty, op); - llvm::SmallVector initArgsTy{ty}; - llvm::SmallVector initArgsLoc{loc}; - mlir::Type refTy = fir::unwrapRefType(ty); - if (auto seqTy = mlir::dyn_cast_or_null(refTy)) { - if (seqTy.hasDynamicExtents()) { - mlir::Type idxTy = builder.getIndexType(); - for (unsigned i = 0; i < seqTy.getDimension(); ++i) { - initArgsTy.push_back(idxTy); - initArgsLoc.push_back(loc); - } - } - } - builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), - initArgsTy, initArgsLoc); - builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); - mlir::Value initValue = genReductionInitRegion(builder, loc, ty, op); - builder.create(loc, initValue); + auto ip = builder.saveInsertionPoint(); + + auto recipe = genRecipeOp( + builder, mod, recipeName, loc, ty, op); // The two first block arguments are the two values to be combined. // The next arguments are the iteration ranges (lb, ub, step) to be used @@ -1846,7 +1802,7 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe( mlir::Value v2 = recipe.getCombinerRegion().front().getArgument(1); genCombiner(builder, loc, op, ty, v1, v2, recipe, bounds, allConstantBound); builder.create(loc, v1); - builder.restoreInsertionPoint(crtPos); + builder.restoreInsertionPoint(ip); return recipe; } diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90 index 0d97c298f8d24..20b5ad28f78a1 100644 --- a/flang/test/Lower/OpenACC/acc-reduction.f90 +++ b/flang/test/Lower/OpenACC/acc-reduction.f90 @@ -39,9 +39,11 @@ ! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box>>, index) -> (index, index, index) ! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> ! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} -! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box> -! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[STORAGE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: %[[BOXTEMP:.*]] = fir.alloca !fir.box>> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[BOXTEMP]] {uniq_name = "acc.reduction.init"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.ref>>> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref>>> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>>>, %[[ARG1:.*]]: !fir.ref>>>): ! CHECK: %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref>>> @@ -74,9 +76,11 @@ ! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[BOX]], %[[C0]] : (!fir.box>>, index) -> (index, index, index) ! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> ! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} -! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box> -! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[STORAGE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: %[[BOXTEMP:.*]] = fir.alloca !fir.box>> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[BOXTEMP]] {uniq_name = "acc.reduction.init"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.ref>>> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref>>> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>>>, %[[ARG1:.*]]: !fir.ref>>>): ! CHECK: %[[BOX0:.*]] = fir.load %[[ARG0]] : !fir.ref>>> diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 3c22aeb9a1ff7..b9148dc088a6a 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -35,22 +35,23 @@ class OpenACC_Op traits = []> : Op; // Reduction operation enumeration. -def OpenACC_ReductionOperatorAdd : I32EnumAttrCase<"AccAdd", 0, "add">; -def OpenACC_ReductionOperatorMul : I32EnumAttrCase<"AccMul", 1, "mul">; -def OpenACC_ReductionOperatorMax : I32EnumAttrCase<"AccMax", 2, "max">; -def OpenACC_ReductionOperatorMin : I32EnumAttrCase<"AccMin", 3, "min">; -def OpenACC_ReductionOperatorAnd : I32EnumAttrCase<"AccIand", 4, "iand">; -def OpenACC_ReductionOperatorOr : I32EnumAttrCase<"AccIor", 5, "ior">; -def OpenACC_ReductionOperatorXor : I32EnumAttrCase<"AccXor", 6, "xor">; -def OpenACC_ReductionOperatorLogEqv : I32EnumAttrCase<"AccEqv", 7, "eqv">; -def OpenACC_ReductionOperatorLogNeqv : I32EnumAttrCase<"AccNeqv", 8, "neqv">; -def OpenACC_ReductionOperatorLogAnd : I32EnumAttrCase<"AccLand", 9, "land">; -def OpenACC_ReductionOperatorLogOr : I32EnumAttrCase<"AccLor", 10, "lor">; +def OpenACC_ReductionOperatorNone : I32EnumAttrCase<"AccNone", 0, "none">; +def OpenACC_ReductionOperatorAdd : I32EnumAttrCase<"AccAdd", 1, "add">; +def OpenACC_ReductionOperatorMul : I32EnumAttrCase<"AccMul", 2, "mul">; +def OpenACC_ReductionOperatorMax : I32EnumAttrCase<"AccMax", 3, "max">; +def OpenACC_ReductionOperatorMin : I32EnumAttrCase<"AccMin", 4, "min">; +def OpenACC_ReductionOperatorAnd : I32EnumAttrCase<"AccIand", 5, "iand">; +def OpenACC_ReductionOperatorOr : I32EnumAttrCase<"AccIor", 6, "ior">; +def OpenACC_ReductionOperatorXor : I32EnumAttrCase<"AccXor", 7, "xor">; +def OpenACC_ReductionOperatorLogEqv : I32EnumAttrCase<"AccEqv", 8, "eqv">; +def OpenACC_ReductionOperatorLogNeqv : I32EnumAttrCase<"AccNeqv", 9, "neqv">; +def OpenACC_ReductionOperatorLogAnd : I32EnumAttrCase<"AccLand", 10, "land">; +def OpenACC_ReductionOperatorLogOr : I32EnumAttrCase<"AccLor", 11, "lor">; def OpenACC_ReductionOperator : I32EnumAttr<"ReductionOperator", "built-in reduction operations supported by OpenACC", - [OpenACC_ReductionOperatorAdd, OpenACC_ReductionOperatorMul, - OpenACC_ReductionOperatorMax, OpenACC_ReductionOperatorMin, + [OpenACC_ReductionOperatorNone, OpenACC_ReductionOperatorAdd, + OpenACC_ReductionOperatorMul, OpenACC_ReductionOperatorMax, OpenACC_ReductionOperatorMin, OpenACC_ReductionOperatorAnd, OpenACC_ReductionOperatorOr, OpenACC_ReductionOperatorXor, OpenACC_ReductionOperatorLogEqv, OpenACC_ReductionOperatorLogNeqv, OpenACC_ReductionOperatorLogAnd,