New "reverse C" calling convention in LLVM and Clang.

haberman · haberman · commit e8d9c75bb35c · 2021-11-03T15:17:48.000-07:00
Test case:
  #define PARAMS \
        const char *ptr, long p2, \
      long p3, long p4, long p5, long p6
  #define ARGS ptr, p2, p3, p4, p5, p6
  #define UNLIKELY(x) \
      __builtin_expect(x, 0)

  #if __has_attribute(reverse_call)
  #define CC __attribute__((reverse_call))
  #else
  #define CC
  #endif

  CC const char *Next(PARAMS);
  const char *Fallback(PARAMS);

  CC const char *Parse(PARAMS) {
    if (UNLIKELY(!ptr)) {
      // This ruins everything.
      ptr = Fallback(ARGS);
    }
    ptr++;
    __attribute__((musttail)) return Next(ARGS);
  }

Before (existing calling convention):
  0000000000000000 &lt;Parse&gt;:
     0:   41 57                   push   r15
     2:   41 56                   push   r14
     4:   41 55                   push   r13
     6:   41 54                   push   r12
     8:   53                      push   rbx
     9:   4d 89 ce                mov    r14,r9
     c:   4d 89 c7                mov    r15,r8
     f:   49 89 cc                mov    r12,rcx
    12:   49 89 d5                mov    r13,rdx
    15:   48 89 f3                mov    rbx,rsi
    18:   48 85 ff                test   rdi,rdi
    1b:   74 21                   je     3e &lt;Parse+0x3e&gt;
    1d:   48 83 c7 01             add    rdi,0x1
    21:   48 89 de                mov    rsi,rbx
    24:   4c 89 ea                mov    rdx,r13
    27:   4c 89 e1                mov    rcx,r12
    2a:   4d 89 f8                mov    r8,r15
    2d:   4d 89 f1                mov    r9,r14
    30:   5b                      pop    rbx
    31:   41 5c                   pop    r12
    33:   41 5d                   pop    r13
    35:   41 5e                   pop    r14
    37:   41 5f                   pop    r15
    39:   e9 00 00 00 00          jmp    3e &lt;Parse+0x3e&gt;
                          3a: R_X86_64_PLT32      Next-0x4
    3e:   31 ff                   xor    edi,edi
    40:   48 89 de                mov    rsi,rbx
    43:   4c 89 ea                mov    rdx,r13
    46:   4c 89 e1                mov    rcx,r12
    49:   4d 89 f8                mov    r8,r15
    4c:   4d 89 f1                mov    r9,r14
    4f:   e8 00 00 00 00          call   54 &lt;Parse+0x54&gt;
                          50: R_X86_64_PLT32      Fallback-0x4
    54:   48 89 c7                mov    rdi,rax
    57:   eb c4                   jmp    1d &lt;Parse+0x1d&gt;

After (new reverse_call calling convention):
  0000000000000000 &lt;Parse&gt;:
     0:   48 85 db                test   rbx,rbx
     3:   74 09                   je     e &lt;Parse+0xe&gt;
     5:   48 83 c3 01             add    rbx,0x1
     9:   e9 00 00 00 00          jmp    e &lt;Parse+0xe&gt;
                          a: R_X86_64_PLT32       Next-0x4
     e:   50                      push   rax
     f:   31 ff                   xor    edi,edi
    11:   48 89 ee                mov    rsi,rbp
    14:   4c 89 e2                mov    rdx,r12
    17:   4c 89 e9                mov    rcx,r13
    1a:   4d 89 f0                mov    r8,r14
    1d:   4d 89 f9                mov    r9,r15
    20:   e8 00 00 00 00          call   25 &lt;Parse+0x25&gt;
                          21: R_X86_64_PLT32      Fallback-0x4
    25:   48 89 c3                mov    rbx,rax
    28:   48 83 c4 08             add    rsp,0x8
    2c:   48 83 c3 01             add    rbx,0x1
    30:   e9 00 00 00 00          jmp    35 &lt;Parse+0x35&gt;

The fast path has gone from 24 instructions (including 5 spills) to
4 instructions (0 spills).
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
@@ -3424,6 +3424,7 @@ enum CXCallingConv {
   CXCallingConv_PreserveAll = 15,
   CXCallingConv_AArch64VectorCall = 16,
   CXCallingConv_SwiftAsync = 17,
+  CXCallingConv_ReverseCall = 18,
 
   CXCallingConv_Invalid = 100,
   CXCallingConv_Unexposed = 200
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
@@ -2567,6 +2567,11 @@ def PreserveMost : DeclOrTypeAttr {
   let Documentation = [PreserveMostDocs];
 }
 
+def ReverseCall : DeclOrTypeAttr {
+  let Spellings = [Clang<"reverse_call">];
+  let Documentation = [ReverseCallDocs];
+}
+
 def PreserveAll : DeclOrTypeAttr {
   let Spellings = [Clang<"preserve_all">];
   let Documentation = [PreserveAllDocs];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
@@ -4864,6 +4864,20 @@ in the future.
   }];
 }
 
+def ReverseCallDocs : Documentation {
+  let Category = DocCatCallingConvs;
+  let Content = [{
+This calling convention makes all registers available for
+passing arguments, but assigns them in reverse order from the default calling
+convention to reduce the chance of overlap. There are no callee-saved
+registers, and the registers that are callee-save under ``ccc`` are the first
+to be assigned as arguments. This makes it very efficient to perform a series
+of tail calls to other ``reverse_call`` functions in threaded code style, while
+reducing the chance of spills or register shuffles when performing non-tail
+calls to regular fallback functions.
+  }];
+}
+
 def PreserveAllDocs : Documentation {
   let Category = DocCatCallingConvs;
   let Content = [{
diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
@@ -280,6 +280,7 @@ namespace clang {
     CC_PreserveMost, // __attribute__((preserve_most))
     CC_PreserveAll,  // __attribute__((preserve_all))
     CC_AArch64VectorCall, // __attribute__((aarch64_vector_pcs))
+    CC_ReverseCall, // __attribute__((reverse_call))
   };
 
   /// Checks whether the given calling convention supports variadic
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
@@ -3098,6 +3098,7 @@ StringRef CXXNameMangler::getCallingConvQualifierName(CallingConv CC) {
   case CC_OpenCLKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
+  case CC_ReverseCall:
     // FIXME: we should be mangling all of the above.
     return "";
 
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
@@ -3156,6 +3156,7 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   case CC_SwiftAsync: return "swiftasynccall";
   case CC_PreserveMost: return "preserve_most";
   case CC_PreserveAll: return "preserve_all";
+  case CC_ReverseCall: return "reverse_call";
   }
 
   llvm_unreachable("Invalid calling convention.");
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
@@ -983,6 +983,9 @@ void TypePrinter::printFunctionAfter(const FunctionType::ExtInfo &Info,
     case CC_PreserveMost:
       OS << " __attribute__((preserve_most))";
       break;
+    case CC_ReverseCall:
+      OS << " __attribute__((reverse_call))";
+      break;
     case CC_PreserveAll:
       OS << " __attribute__((preserve_all))";
       break;
@@ -1718,6 +1721,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::MSABI: OS << "ms_abi"; break;
   case attr::SysVABI: OS << "sysv_abi"; break;
   case attr::RegCall: OS << "regcall"; break;
+  case attr::ReverseCall: OS << "reverse_call"; break;
   case attr::Pcs: {
     OS << "pcs(";
    QualType t = T->getEquivalentType();
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
@@ -358,6 +358,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
     case CC_X86Pascal:
     case CC_IntelOclBicc:
     case CC_OpenCLKernel:
+    case CC_ReverseCall:
       return CCCR_OK;
     case CC_SwiftAsync:
       return CCCR_Error;
@@ -729,6 +730,7 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
     case CC_PreserveAll:
     case CC_X86RegCall:
     case CC_OpenCLKernel:
+    case CC_ReverseCall:
       return CCCR_OK;
     default:
       return CCCR_Warning;
@@ -806,6 +808,7 @@ class LLVM_LIBRARY_VISIBILITY WindowsX86_64TargetInfo
     case CC_SwiftAsync:
     case CC_X86RegCall:
     case CC_OpenCLKernel:
+    case CC_ReverseCall:
       return CCCR_OK;
     default:
       return CCCR_Warning;
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
@@ -67,6 +67,7 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
   case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
   case CC_Swift: return llvm::CallingConv::Swift;
   case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
+  case CC_ReverseCall: return llvm::CallingConv::ReverseC;
   }
 }
 
@@ -242,6 +243,9 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
   if (D->hasAttr<PreserveAllAttr>())
     return CC_PreserveAll;
 
+  if (D->hasAttr<ReverseCallAttr>())
+    return CC_ReverseCall;
+
   return CC_C;
 }
 
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -4742,6 +4742,9 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   case ParsedAttr::AT_PreserveAll:
     D->addAttr(::new (S.Context) PreserveAllAttr(S.Context, AL));
     return;
+  case ParsedAttr::AT_ReverseCall:
+    D->addAttr(::new (S.Context) ReverseCallAttr(S.Context, AL));
+    return;
   default:
     llvm_unreachable("unexpected attribute kind");
   }
@@ -4913,6 +4916,9 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
   case ParsedAttr::AT_PreserveAll:
     CC = CC_PreserveAll;
     break;
+  case ParsedAttr::AT_ReverseCall:
+    CC = CC_ReverseCall;
+    break;
   default: llvm_unreachable("unexpected attribute kind");
   }
 
@@ -8290,6 +8296,7 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
   case ParsedAttr::AT_PreserveMost:
   case ParsedAttr::AT_PreserveAll:
   case ParsedAttr::AT_AArch64VectorPcs:
+  case ParsedAttr::AT_ReverseCall:
     handleCallConvAttr(S, D, AL);
     break;
   case ParsedAttr::AT_Suppress:
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
@@ -125,7 +125,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   case ParsedAttr::AT_Pcs:                                                     \
   case ParsedAttr::AT_IntelOclBicc:                                            \
   case ParsedAttr::AT_PreserveMost:                                            \
-  case ParsedAttr::AT_PreserveAll
+  case ParsedAttr::AT_PreserveAll:                                             \
+  case ParsedAttr::AT_ReverseCall
 
 // Function type attributes.
 #define FUNCTION_TYPE_ATTRS_CASELIST                                           \
@@ -7409,6 +7410,8 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     return createSimpleAttr<SysVABIAttr>(Ctx, Attr);
   case ParsedAttr::AT_PreserveMost:
     return createSimpleAttr<PreserveMostAttr>(Ctx, Attr);
+  case ParsedAttr::AT_ReverseCall:
+    return createSimpleAttr<ReverseCallAttr>(Ctx, Attr);
   case ParsedAttr::AT_PreserveAll:
     return createSimpleAttr<PreserveAllAttr>(Ctx, Attr);
   }
diff --git a/clang/tools/libclang/CXType.cpp b/clang/tools/libclang/CXType.cpp
@@ -669,6 +669,7 @@ CXCallingConv clang_getFunctionTypeCallingConv(CXType X) {
       TCALLINGCONV(SwiftAsync);
       TCALLINGCONV(PreserveMost);
       TCALLINGCONV(PreserveAll);
+      TCALLINGCONV(ReverseCall);
     case CC_SpirFunction: return CXCallingConv_Unexposed;
     case CC_OpenCLKernel: return CXCallingConv_Unexposed;
       break;
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
@@ -440,6 +440,14 @@ added in the future:
     the GHC or the HiPE convention is used. <CodeGenerator.html#id80>`_ This
     calling convention does not support varargs and requires the prototype of
     all callees to exactly match the prototype of the function definition.
+"``reverse_ccc``" - This calling convention makes all registers available for
+    passing arguments, but assigns them in reverse order from ``ccc`` to reduce
+    the chance of overlap. There are no callee-saved registers, and the
+    registers that are callee-save under ``ccc`` are the first to be assigned
+    as arguments. This makes it very efficient to perform a series of tail
+    calls to other ``reverse_ccc`` functions in threaded code style, while
+    reducing the chance of spills or register shuffles when performing non-tail
+    calls to regular ``ccc`` fallback functions.
 "``swiftcc``" - This calling convention is used for Swift language.
     - On X86-64 RCX and R8 are available for additional integer returns, and
       XMM2 and XMM3 are available for additional FP/vector returns.
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
@@ -174,6 +174,7 @@ enum Kind {
   kw_amdgpu_kernel,
   kw_amdgpu_gfx,
   kw_tailcc,
+  kw_reverse_ccc,
 
   // Attributes:
   kw_attributes,
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
@@ -91,6 +91,11 @@ namespace CallingConv {
     /// clean up their stack.
     SwiftTail = 20,
 
+    /// ReverseC - This calling convention makes all registers available for
+    /// arguments, but assigns them in reverse order from CallingConv::C, and
+    /// uses no callee-saved registers.
+    ReverseC = 21,
+
     // Target - This is the start of the target-specific calling conventions,
     // e.g. fastcall and thiscall on X86.
     FirstTargetCC = 64,
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
@@ -627,6 +627,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(amdgpu_kernel);
   KEYWORD(amdgpu_gfx);
   KEYWORD(tailcc);
+  KEYWORD(reverse_ccc);
 
   KEYWORD(cc);
   KEYWORD(c);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
@@ -1811,6 +1811,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) {
 ///   ::= 'amdgpu_cs'
 ///   ::= 'amdgpu_kernel'
 ///   ::= 'tailcc'
+///   ::= 'reverse_ccc'
 ///   ::= 'cc' UINT
 ///
 bool LLParser::parseOptionalCallingConv(unsigned &CC) {
@@ -1863,6 +1864,7 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) {
   case lltok::kw_amdgpu_cs:      CC = CallingConv::AMDGPU_CS; break;
   case lltok::kw_amdgpu_kernel:  CC = CallingConv::AMDGPU_KERNEL; break;
   case lltok::kw_tailcc:         CC = CallingConv::Tail; break;
+  case lltok::kw_reverse_ccc:    CC = CallingConv::ReverseC; break;
   case lltok::kw_cc: {
       Lex.Lex();
       return parseUInt32(CC);
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
@@ -299,6 +299,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) {
   case CallingConv::CXX_FAST_TLS:  Out << "cxx_fast_tlscc"; break;
   case CallingConv::GHC:           Out << "ghccc"; break;
   case CallingConv::Tail:          Out << "tailcc"; break;
+  case CallingConv::ReverseC:      Out << "reverse_ccc"; break;
   case CallingConv::CFGuard_Check: Out << "cfguard_checkcc"; break;
   case CallingConv::X86_StdCall:   Out << "x86_stdcallcc"; break;
   case CallingConv::X86_FastCall:  Out << "x86_fastcallcc"; break;
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1030,6 +1030,40 @@ def CC_X86_32_HiPE : CallingConv<[
   CCIfType<[i32, f32], CCAssignToStack<4, 4>>
 ]>;
 
+// X86-64 ReverseC calling convention.
+def CC_X86_64_ReverseC : CallingConv<[
+  // Promote i1/i8/i16 arguments to i32.
+  CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
+
+  // Use all registers, starting with callee-save, ending with ccc
+  // arguments in reverse order.
+  CCIfType<[i32], CCAssignToReg<[
+    // Normally callee-save registers.
+    EBX,
+    //EBP,  // Conflicts with frame pointer when -fno-omit-frame-pointer
+    R12D, R13D, R14D, R15D,
+    // Normally scratch registers.
+    R10D, R11D,
+    // Normally argument registers, in reverse order.
+    R9D, R8D, ECX, EDX, ESI, EDI,
+    // Return register
+    EAX]>>,
+  CCIfType<[i64], CCAssignToReg<[
+    // Normally callee-save registers.
+    RBX,
+    //RBP,  // Conflicts with frame pointer when -fno-omit-frame-pointer
+    R12, R13, R14, R15,
+    // Normally scratch registers.
+    R10, R11,
+    // Normally argument registers, in reverse order.
+    R9, R8, RCX, RDX, RSI, RDI,
+    // Return register
+    RAX]>>,
+
+  // Otherwise it's the same as the regular C calling convention.
+  CCDelegateTo<CC_X86_64_C>
+]>;
+
 // X86-64 Intel OpenCL built-ins calling convention.
 def CC_Intel_OCL_BI : CallingConv<[
 
@@ -1099,8 +1133,10 @@ def CC_X86_64 : CallingConv<[
   CCIfCC<"CallingConv::X86_RegCall",
     CCIfSubtarget<"isTargetWin64()", CCDelegateTo<CC_X86_Win64_RegCall>>>,
   CCIfCC<"CallingConv::X86_RegCall", CCDelegateTo<CC_X86_SysV64_RegCall>>,
+  CCIfCC<"CallingConv::ReverseC", CCDelegateTo<CC_X86_64_ReverseC>>,
   CCIfCC<"CallingConv::X86_INTR", CCCustom<"CC_X86_Intr">>,
 
+
   // Mingw64 and native Win64 use Win64 CC
   CCIfSubtarget<"isTargetWin64()", CCDelegateTo<CC_X86_Win64_C>>,
 
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -298,6 +298,7 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   switch (CC) {
   case CallingConv::GHC:
   case CallingConv::HiPE:
+  case CallingConv::ReverseC:
     return CSR_NoRegs_SaveList;
   case CallingConv::AnyReg:
     if (HasAVX)
@@ -421,6 +422,7 @@ X86RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
   switch (CC) {
   case CallingConv::GHC:
   case CallingConv::HiPE:
+  case CallingConv::ReverseC:
     return CSR_NoRegs_RegMask;
   case CallingConv::AnyReg:
     if (HasAVX)

Original file line number	Diff line number	Diff line change
`@@ -3156,6 +3156,7 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {`
`3156`	`3156`	`case CC_SwiftAsync: return "swiftasynccall";`
`3157`	`3157`	`case CC_PreserveMost: return "preserve_most";`
`3158`	`3158`	`case CC_PreserveAll: return "preserve_all";`
	`3159`	`+ case CC_ReverseCall: return "reverse_call";`
`3159`	`3160`	`}`
`3160`	`3161`
`3161`	`3162`	`llvm_unreachable("Invalid calling convention.");`
Original file line number	Diff line number	Diff line change
`@@ -67,6 +67,7 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {`
`67`	`67`	`case CC_PreserveAll: return llvm::CallingConv::PreserveAll;`
`68`	`68`	`case CC_Swift: return llvm::CallingConv::Swift;`
`69`	`69`	`case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;`
	`70`	`+ case CC_ReverseCall: return llvm::CallingConv::ReverseC;`
`70`	`71`	`}`
`71`	`72`	`}`
`72`	`73`
`@@ -242,6 +243,9 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,`
`242`	`243`	`if (D->hasAttr<PreserveAllAttr>())`
`243`	`244`	`return CC_PreserveAll;`
`244`	`245`
	`246`	`+ if (D->hasAttr<ReverseCallAttr>())`
	`247`	`+ return CC_ReverseCall;`
	`248`	`+`
`245`	`249`	`return CC_C;`
`246`	`250`	`}`
`247`	`251`