diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index a48a99ba95e0..8db92f967214 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -4003,19 +4003,14 @@ def CopyOp : CIR_Op<"copy", // MemCpyOp && MemMoveOp //===----------------------------------------------------------------------===// -class CIR_MemCpyOp: CIR_Op]> { - let arguments = (ins Arg:$dst, - Arg:$src, - PrimitiveUInt:$len); +class CIR_MemOp + : CIR_Op]> { + dag commonArgs = (ins Arg:$dst, + Arg:$src); let hasVerifier = 0; - - let extraClassDeclaration = [{ - /// Returns the byte length type. - cir::IntType getLenTy() { return getLen().getType(); } - }]; } -def MemCpyOp : CIR_MemCpyOp<"libc.memcpy"> { +def MemCpyOp : CIR_MemOp<"libc.memcpy"> { let summary = "Equivalent to libc's `memcpy`"; let description = [{ Given two CIR pointers, `src` and `dst`, `cir.libc.memcpy` will copy `len` @@ -4034,13 +4029,20 @@ def MemCpyOp : CIR_MemCpyOp<"libc.memcpy"> { ``` }]; + let arguments = !con(commonArgs, (ins PrimitiveUInt:$len)); + let assemblyFormat = [{ $len `bytes` `from` $src `to` $dst attr-dict `:` type($len) `` `,` qualified(type($src)) `->` qualified(type($dst)) }]; + + let extraClassDeclaration = [{ + /// Returns the byte length type. + cir::IntType getLenTy() { return getLen().getType(); } + }]; } -def MemMoveOp : CIR_MemCpyOp<"libc.memmove"> { +def MemMoveOp : CIR_MemOp<"libc.memmove"> { let summary = "Equivalent to libc's `memmove`"; let description = [{ Given two CIR pointers, `src` and `dst`, `cir.libc.memmove` will copy `len` @@ -4057,12 +4059,49 @@ def MemMoveOp : CIR_MemCpyOp<"libc.memmove"> { ``` }]; + let arguments = !con(commonArgs, (ins PrimitiveUInt:$len)); let assemblyFormat = [{ $len `bytes` `from` $src `to` $dst attr-dict `:` qualified(type($dst)) `,` type($len) }]; + + let extraClassDeclaration = [{ + /// Returns the byte length type. + cir::IntType getLenTy() { return getLen().getType(); } + }]; } + +//===----------------------------------------------------------------------===// +// MemCpyInlineOp +//===----------------------------------------------------------------------===// + +def MemCpyInlineOp : CIR_MemOp<"memcpy_inline"> { + let summary = "Memory copy with constant length without calling" + "any external function"; + let description = [{ + Given two CIR pointers, `src` and `dst`, `memcpy_inline` will copy `len` + bytes from the memory pointed by `src` to the memory pointed by `dst`. + + Unlike `cir.libc.memcpy`, this Op guarantees that no external functions + are called, and length of copied bytes is a constant. + + Examples: + + ```mlir + // Copying 2 bytes from one array to a struct: + cir.memcpy_inline 2 bytes from %arr to %struct : !cir.ptr -> !cir.ptr + ``` + }]; + + let arguments = !con(commonArgs, (ins I64Attr:$len)); + + let assemblyFormat = [{ + $len `bytes` `from` $src `to` $dst attr-dict + `:` qualified(type($src)) `->` qualified(type($dst)) + }]; +} + //===----------------------------------------------------------------------===// // MemSetOp //===----------------------------------------------------------------------===// diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index 6809c3ad5ce3..b15549547ea1 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -1408,8 +1408,22 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, return RValue::get(Dest.getPointer()); } - case Builtin::BI__builtin_memcpy_inline: - llvm_unreachable("BI__builtin_memcpy_inline NYI"); + case Builtin::BI__builtin_memcpy_inline: { + Address dest = emitPointerWithAlignment(E->getArg(0)); + Address src = emitPointerWithAlignment(E->getArg(1)); + emitNonNullArgCheck(RValue::get(dest.getPointer()), E->getArg(0)->getType(), + E->getArg(0)->getExprLoc(), FD, 0); + emitNonNullArgCheck(RValue::get(src.getPointer()), E->getArg(1)->getType(), + E->getArg(1)->getExprLoc(), FD, 1); + uint64_t size = + E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue(); + builder.create( + getLoc(E->getSourceRange()), dest.getPointer(), src.getPointer(), + mlir::IntegerAttr::get(mlir::IntegerType::get(builder.getContext(), 64), + size)); + // __builtin_memcpy_inline has no return value + return RValue::get(nullptr); + } case Builtin::BI__builtin_char_memchr: case Builtin::BI__builtin_memchr: { diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 2ae12be6efbd..d0fa5fe9afa1 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -778,6 +778,21 @@ class CIRMemChrOpLowering : public mlir::OpConversionPattern { } }; +class CIRMemCpyInlineOpLowering + : public mlir::OpConversionPattern { +public: + using mlir::OpConversionPattern::OpConversionPattern; + + mlir::LogicalResult + matchAndRewrite(cir::MemCpyInlineOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp( + op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLenAttr(), + /*isVolatile=*/false); + return mlir::success(); + } +}; + class CIRMemMoveOpLowering : public mlir::OpConversionPattern { public: using mlir::OpConversionPattern::OpConversionPattern; @@ -4331,8 +4346,8 @@ void populateCIRToLLVMConversionPatterns( CIRVAStartLowering, CIRVAEndLowering, CIRVACopyLowering, CIRVAArgLowering, CIRBrOpLowering, CIRGetMemberOpLowering, CIRGetRuntimeMemberOpLowering, CIRSwitchFlatOpLowering, CIRPtrDiffOpLowering, CIRCopyOpLowering, - CIRMemCpyOpLowering, CIRMemChrOpLowering, CIRFAbsOpLowering, - CIRExpectOpLowering, CIRVTableAddrPointOpLowering, + CIRMemCpyOpLowering, CIRMemChrOpLowering, CIRMemCpyInlineOpLowering, + CIRFAbsOpLowering, CIRExpectOpLowering, CIRVTableAddrPointOpLowering, CIRVectorCreateLowering, CIRVectorCmpOpLowering, CIRVectorSplatLowering, CIRVectorTernaryLowering, CIRVectorShuffleIntsLowering, CIRVectorShuffleVecLowering, CIRStackSaveLowering, CIRUnreachableLowering, @@ -4376,27 +4391,27 @@ std::unique_ptr prepareLowerModule(mlir::ModuleOp module) { void prepareTypeConverter(mlir::LLVMTypeConverter &converter, mlir::DataLayout &dataLayout, cir::LowerModule *lowerModule) { - converter.addConversion([&, - lowerModule](cir::PointerType type) -> mlir::Type { - // Drop pointee type since LLVM dialect only allows opaque pointers. - - auto addrSpace = - mlir::cast_if_present(type.getAddrSpace()); - // Null addrspace attribute indicates the default addrspace. - if (!addrSpace) - return mlir::LLVM::LLVMPointerType::get(type.getContext()); - - assert(lowerModule && "CIR AS map is not available"); - // Pass through target addrspace and map CIR addrspace to LLVM addrspace by - // querying the target info. - unsigned targetAS = - addrSpace.isTarget() - ? addrSpace.getTargetValue() - : lowerModule->getTargetLoweringInfo() - .getTargetAddrSpaceFromCIRAddrSpace(addrSpace); - - return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS); - }); + converter.addConversion( + [&, lowerModule](cir::PointerType type) -> mlir::Type { + // Drop pointee type since LLVM dialect only allows opaque pointers. + + auto addrSpace = + mlir::cast_if_present(type.getAddrSpace()); + // Null addrspace attribute indicates the default addrspace. + if (!addrSpace) + return mlir::LLVM::LLVMPointerType::get(type.getContext()); + + assert(lowerModule && "CIR AS map is not available"); + // Pass through target addrspace and map CIR addrspace to LLVM addrspace + // by querying the target info. + unsigned targetAS = + addrSpace.isTarget() + ? addrSpace.getTargetValue() + : lowerModule->getTargetLoweringInfo() + .getTargetAddrSpaceFromCIRAddrSpace(addrSpace); + + return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS); + }); converter.addConversion([&](cir::DataMemberType type) -> mlir::Type { return mlir::IntegerType::get(type.getContext(), dataLayout.getTypeSizeInBits(type)); diff --git a/clang/test/CIR/CodeGen/builtins-memory.c b/clang/test/CIR/CodeGen/builtins-memory.c index 439b82e98d33..940e09a8ed6d 100644 --- a/clang/test/CIR/CodeGen/builtins-memory.c +++ b/clang/test/CIR/CodeGen/builtins-memory.c @@ -1,5 +1,8 @@ // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir // RUN: FileCheck %s --check-prefix=CIR --input-file=%t.cir +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o - \ +// RUN: | opt -S -passes=instcombine,mem2reg,simplifycfg -o %t.ll +// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s typedef __SIZE_TYPE__ size_t; void test_memcpy_chk(void *dest, const void *src, size_t n) { @@ -107,3 +110,38 @@ void test_memset_chk(void *dest, int ch, size_t n) { // CIR: cir.call @__memset_chk(%[[#DEST_LOAD]], %[[#CH_LOAD]], %[[#N_LOAD1]], %[[#N_LOAD2]]) __builtin___memset_chk(dest, ch, n, n); } + +// FIXME: The test should test intrinsic argument alignment, however, +// currently we lack support for argument attributes. +// Thus, added `COM: LLVM:` lines so we can easily flip the test +// when the support of argument attributes is in. +void test_memcpy_inline(void *dst, const void *src, size_t n) { + + // CIR-LABEL: test_memcpy_inline + // CIR: cir.memcpy_inline 0 bytes from {{%.*}} to {{%.*}} : !cir.ptr -> !cir.ptr + + // LLVM-LABEL: test_memcpy_inline + // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 0, i1 false) + // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 0, i1 false) + __builtin_memcpy_inline(dst, src, 0); + + // CIR: cir.memcpy_inline 1 bytes from {{%.*}} to {{%.*}} : !cir.ptr -> !cir.ptr + + // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 1, i1 false) + // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 1, i1 false) + __builtin_memcpy_inline(dst, src, 1); + + // CIR: cir.memcpy_inline 4 bytes from {{%.*}} to {{%.*}} : !cir.ptr -> !cir.ptr + + // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 4, i1 false) + // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 {{%.*}}, ptr align 1 {{%.*}}, i64 4, i1 false) + __builtin_memcpy_inline(dst, src, 4); +} + +void test_memcpy_inline_aligned_buffers(unsigned long long *dst, const unsigned long long *src) { + + // LLVM-LABEL: test_memcpy_inline_aligned_buffers + // LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr {{%.*}}, ptr {{%.*}}, i64 4, i1 false) + // COM: LLVM: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 {{%.*}}, ptr align 8 {{%.*}}, i64 4, i1 false) + __builtin_memcpy_inline(dst, src, 4); +}