Treat std::move, forward, and move_if_noexcept as builtins.

We still require these functions to be declared before they can be used,
but don't instantiate their definitions unless their addresses are
taken. Instead, code generation, constant evaluation, and static
analysis are given direct knowledge of their effect.

This change aims to reduce various costs associated with these functions
-- per-instantiation memory costs, compile time and memory costs due to
creating out-of-line copies and inlining them, code size at -O0, and so
on -- so that they are not substantially more expensive than a cast.
Most of these improvements are very small, but I measured a 3% decrease
in -O0 object file size for a simple C++ source file using the standard
library after this change.

We now automatically infer the `const` and `nothrow` attributes on these
now-builtin functions, in particular meaning that we get a warning for
an unused call to one of these functions.

In C++20 onwards, we disallow taking the addresses of these functions,
per the C++20 "addressable function" rule. In earlier language modes, a
compatibility warning is produced but the address can still be taken.

The same infrastructure is extended to the existing MSVC builtin
`__GetExceptionInfo`, which is now only recognized in namespace `std`
like it always should have been.

Reviewed By: aaron.ballman

Differential Revision: https://reviews.llvm.org/D123345

Author: zygoloid

clang/docs/CommandGuide/clang.rst

@@ -252,8 +252,24 @@ Language Selection and Mode Options
 
 .. option:: -fno-builtin
 
- Disable special handling and optimizations of builtin functions like
- :c:func:`strlen` and :c:func:`malloc`.
+ Disable special handling and optimizations of well-known library functions,
+ like :c:func:`strlen` and :c:func:`malloc`.
+
+.. option:: -fno-builtin-<function>
+
+ Disable special handling and optimizations for the specific library function.
+ For example, ``-fno-builtin-strlen`` removes any special handling for the
+ :c:func:`strlen` library function.
+
+.. option:: -fno-builtin-std-<function>
+
+ Disable special handling and optimizations for the specific C++ standard
+ library function in namespace ``std``. For example,
+ ``-fno-builtin-std-move_if_noexcept`` removes any special handling for the
+ :cpp:func:`std::move_if_noexcept` library function.
+
+ For C standard library functions that the C++ standard library also provides
+ in namespace ``std``, use :option:`-fno-builtin-\<function\>` instead.
 
 .. option:: -fmath-errno
 

clang/docs/ReleaseNotes.rst

@@ -230,7 +230,10 @@ C2x Feature Support
 C++ Language Changes in Clang
 -----------------------------
 
-- ...
+- Improved ``-O0`` code generation for calls to ``std::move``, ``std::forward``,
+  and ``std::move_if_noexcept``. These are now treated as compiler builtins and
+  implemented directly, rather than instantiating the definition from the
+  standard library.
 
 C++20 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^

clang/include/clang/Basic/Builtins.def

@@ -81,7 +81,9 @@
 //       builtin even if type doesn't match signature, and don't warn if we
 //       can't be sure the type is right
 //  F -> this is a libc/libm function with a '__builtin_' prefix added.
-//  f -> this is a libc/libm function without the '__builtin_' prefix.
+//  f -> this is a libc/libm function without a '__builtin_' prefix, or with
+//       'z', a C++ standard library function in namespace std::. This builtin
+//       is disableable by '-fno-builtin-foo' / '-fno-builtin-std-foo'.
 //  h -> this function requires a specific header or an explicit declaration.
 //  i -> this is a runtime library implemented function without the
 //       '__builtin_' prefix. It will be implemented in compiler-rt or libgcc.
@@ -101,6 +103,8 @@
 //  V:N: -> requires vectors of at least N bits to be legal
 //  C<N,M_0,...,M_k> -> callback behavior: argument N is called with argument
 //                      M_0, ..., M_k as payload
+//  z -> this is a C++ standard library function in (possibly-versioned)
+//       namespace std; implied by STDBUILTIN
 //  FIXME: gcc has nonnull
 
 #if defined(BUILTIN) && !defined(LIBBUILTIN)
@@ -111,6 +115,10 @@
 #  define LANGBUILTIN(ID, TYPE, ATTRS, BUILTIN_LANG) BUILTIN(ID, TYPE, ATTRS)
 #endif
 
+#if defined(BUILTIN) && !defined(STDBUILTIN)
+#  define STDBUILTIN(ID, TYPE, ATTRS, HEADER) LIBBUILTIN(ID, TYPE, "zf" ATTRS, HEADER, CXX_LANG)
+#endif
+
 // Standard libc/libm functions:
 BUILTIN(__builtin_atan2 , "ddd"  , "Fne")
 BUILTIN(__builtin_atan2f, "fff"  , "Fne")
@@ -919,7 +927,7 @@ LANGBUILTIN(__exception_info, "v*",  "n", ALL_MS_LANGUAGES)
 LANGBUILTIN(_exception_info,  "v*",  "n", ALL_MS_LANGUAGES)
 LANGBUILTIN(__abnormal_termination, "i", "n", ALL_MS_LANGUAGES)
 LANGBUILTIN(_abnormal_termination,  "i", "n", ALL_MS_LANGUAGES)
-LANGBUILTIN(__GetExceptionInfo, "v*.", "ntu", ALL_MS_LANGUAGES)
+LANGBUILTIN(__GetExceptionInfo, "v*.", "zntu", ALL_MS_LANGUAGES)
 LANGBUILTIN(_InterlockedAnd8,   "ccD*c",        "n", ALL_MS_LANGUAGES)
 LANGBUILTIN(_InterlockedAnd16,  "ssD*s",        "n", ALL_MS_LANGUAGES)
 LANGBUILTIN(_InterlockedAnd,    "NiNiD*Ni",     "n", ALL_MS_LANGUAGES)
@@ -1543,6 +1551,11 @@ LIBBUILTIN(_Block_object_assign, "vv*vC*iC", "f", "Blocks.h", ALL_LANGUAGES)
 LIBBUILTIN(_Block_object_dispose, "vvC*iC", "f", "Blocks.h", ALL_LANGUAGES)
 // FIXME: Also declare NSConcreteGlobalBlock and NSConcreteStackBlock.
 
+// C++11
+STDBUILTIN(move, "v&v&", "ncTh", "utility")
+STDBUILTIN(move_if_noexcept, "v&v&", "ncTh", "utility")
+STDBUILTIN(forward, "v&v&", "ncTh", "utility")
+
 // Annotation function
 BUILTIN(__builtin_annotation, "v.", "tn")
 

clang/include/clang/Basic/Builtins.h

@@ -138,6 +138,10 @@ class Context {
   /// Determines whether this builtin is a predefined libc/libm
   /// function, such as "malloc", where we know the signature a
   /// priori.
+  /// In C, such functions behave as if they are predeclared,
+  /// possibly with a warning on first use. In Objective-C and C++,
+  /// they do not, but they are recognized as builtins once we see
+  /// a declaration.
   bool isPredefinedLibFunction(unsigned ID) const {
     return strchr(getRecord(ID).Attributes, 'f') != nullptr;
   }
@@ -156,6 +160,23 @@ class Context {
     return strchr(getRecord(ID).Attributes, 'i') != nullptr;
   }
 
+  /// Determines whether this builtin is a C++ standard library function
+  /// that lives in (possibly-versioned) namespace std, possibly a template
+  /// specialization, where the signature is determined by the standard library
+  /// declaration.
+  bool isInStdNamespace(unsigned ID) const {
+    return strchr(getRecord(ID).Attributes, 'z') != nullptr;
+  }
+
+  /// Determines whether this builtin can have its address taken with no
+  /// special action required.
+  bool isDirectlyAddressable(unsigned ID) const {
+    // Most standard library functions can have their addresses taken. C++
+    // standard library functions formally cannot in C++20 onwards, and when
+    // we allow it, we need to ensure we instantiate a definition.
+    return isPredefinedLibFunction(ID) && !isInStdNamespace(ID);
+  }
+
   /// Determines whether this builtin has custom typechecking.
   bool hasCustomTypechecking(unsigned ID) const {
     return strchr(getRecord(ID).Attributes, 't') != nullptr;
@@ -237,10 +258,6 @@ class Context {
 private:
   const Info &getRecord(unsigned ID) const;
 
-  /// Is this builtin supported according to the given language options?
-  bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
-                          const LangOptions &LangOpts);
-
   /// Helper function for isPrintfLike and isScanfLike.
   bool isLike(unsigned ID, unsigned &FormatIdx, bool &HasVAListArg,
               const char *Fmt) const;

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -6586,6 +6586,15 @@ def warn_self_move : Warning<
   "explicitly moving variable of type %0 to itself">,
   InGroup<SelfMove>, DefaultIgnore;
 
+def err_builtin_move_forward_unsupported : Error<
+  "unsupported signature for '%select{std::move|std::forward}0'">;
+def err_use_of_unaddressable_function : Error<
+  "taking address of non-addressable standard library function">;
+// FIXME: This should also be in -Wc++23-compat once we have it.
+def warn_cxx20_compat_use_of_unaddressable_function : Warning<
+  "taking address of non-addressable standard library function "
+  "is incompatible with C++20">, InGroup<CXX20Compat>;
+
 def warn_redundant_move_on_return : Warning<
   "redundant move in return statement">,
   InGroup<RedundantMove>, DefaultIgnore;

clang/lib/AST/ExprConstant.cpp

@@ -8127,6 +8127,7 @@ class LValueExprEvaluator
   bool VisitVarDecl(const Expr *E, const VarDecl *VD);
   bool VisitUnaryPreIncDec(const UnaryOperator *UO);
 
+  bool VisitCallExpr(const CallExpr *E);
   bool VisitDeclRefExpr(const DeclRefExpr *E);
   bool VisitPredefinedExpr(const PredefinedExpr *E) { return Success(E); }
   bool VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
@@ -8292,6 +8293,19 @@ bool LValueExprEvaluator::VisitVarDecl(const Expr *E, const VarDecl *VD) {
   return Success(*V, E);
 }
 
+bool LValueExprEvaluator::VisitCallExpr(const CallExpr *E) {
+  switch (unsigned BuiltinOp = E->getBuiltinCallee()) {
+  case Builtin::BImove:
+  case Builtin::BImove_if_noexcept:
+  case Builtin::BIforward:
+    if (cast<FunctionDecl>(E->getCalleeDecl())->isConstexpr())
+      return Visit(E->getArg(0));
+    break;
+  }
+
+  return ExprEvaluatorBaseTy::VisitCallExpr(E);
+}
+
 bool LValueExprEvaluator::VisitMaterializeTemporaryExpr(
     const MaterializeTemporaryExpr *E) {
   // Walk through the expression to find the materialized temporary itself.

clang/lib/Analysis/BodyFarm.cpp

@@ -20,6 +20,7 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/Analysis/CodeInjector.h"
+#include "clang/Basic/Builtins.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Debug.h"
@@ -86,6 +87,9 @@ class ASTMaker {
   ImplicitCastExpr *makeImplicitCast(const Expr *Arg, QualType Ty,
                                      CastKind CK = CK_LValueToRValue);
 
+  /// Create a cast to reference type.
+  CastExpr *makeReferenceCast(const Expr *Arg, QualType Ty);
+
   /// Create an Objective-C bool literal.
   ObjCBoolLiteralExpr *makeObjCBool(bool Val);
 
@@ -173,6 +177,16 @@ ImplicitCastExpr *ASTMaker::makeImplicitCast(const Expr *Arg, QualType Ty,
                                   /* FPFeatures */ FPOptionsOverride());
 }
 
+CastExpr *ASTMaker::makeReferenceCast(const Expr *Arg, QualType Ty) {
+  assert(Ty->isReferenceType());
+  return CXXStaticCastExpr::Create(
+      C, Ty.getNonReferenceType(),
+      Ty->isLValueReferenceType() ? VK_LValue : VK_XValue, CK_NoOp,
+      const_cast<Expr *>(Arg), /*CXXCastPath=*/nullptr,
+      /*Written=*/C.getTrivialTypeSourceInfo(Ty), FPOptionsOverride(),
+      SourceLocation(), SourceLocation(), SourceRange());
+}
+
 Expr *ASTMaker::makeIntegralCast(const Expr *Arg, QualType Ty) {
   if (Arg->getType() == Ty)
     return const_cast<Expr*>(Arg);
@@ -296,6 +310,22 @@ static CallExpr *create_call_once_lambda_call(ASTContext &C, ASTMaker M,
       /*FPFeatures=*/FPOptionsOverride());
 }
 
+/// Create a fake body for 'std::move' or 'std::forward'. This is just:
+///
+/// \code
+/// return static_cast<return_type>(param);
+/// \endcode
+static Stmt *create_std_move_forward(ASTContext &C, const FunctionDecl *D) {
+  LLVM_DEBUG(llvm::dbgs() << "Generating body for std::move / std::forward\n");
+
+  ASTMaker M(C);
+
+  QualType ReturnType = D->getType()->castAs<FunctionType>()->getReturnType();
+  Expr *Param = M.makeDeclRefExpr(D->getParamDecl(0));
+  Expr *Cast = M.makeReferenceCast(Param, ReturnType);
+  return M.makeReturn(Cast);
+}
+
 /// Create a fake body for std::call_once.
 /// Emulates the following function body:
 ///
@@ -681,8 +711,19 @@ Stmt *BodyFarm::getBody(const FunctionDecl *D) {
 
   FunctionFarmer FF;
 
-  if (Name.startswith("OSAtomicCompareAndSwap") ||
-      Name.startswith("objc_atomicCompareAndSwap")) {
+  if (unsigned BuiltinID = D->getBuiltinID()) {
+    switch (BuiltinID) {
+    case Builtin::BImove:
+    case Builtin::BImove_if_noexcept:
+    case Builtin::BIforward:
+      FF = create_std_move_forward;
+      break;
+    default:
+      FF = nullptr;
+      break;
+    }
+  } else if (Name.startswith("OSAtomicCompareAndSwap") ||
+             Name.startswith("objc_atomicCompareAndSwap")) {
     FF = create_OSAtomicCompareAndSwap;
   } else if (Name == "call_once" && D->getDeclContext()->isStdNamespace()) {
     FF = create_call_once;

clang/lib/Basic/Builtins.cpp

@@ -48,18 +48,22 @@ void Builtin::Context::InitializeTarget(const TargetInfo &Target,
 }
 
 bool Builtin::Context::isBuiltinFunc(llvm::StringRef FuncName) {
-  for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin; ++i)
-    if (FuncName.equals(BuiltinInfo[i].Name))
+  bool InStdNamespace = FuncName.consume_front("std-");
+  for (unsigned i = Builtin::NotBuiltin + 1; i != Builtin::FirstTSBuiltin;
+       ++i) {
+    if (FuncName.equals(BuiltinInfo[i].Name) &&
+        (bool)strchr(BuiltinInfo[i].Attributes, 'z') == InStdNamespace)
       return strchr(BuiltinInfo[i].Attributes, 'f') != nullptr;
+  }
 
   return false;
 }
 
-bool Builtin::Context::builtinIsSupported(const Builtin::Info &BuiltinInfo,
-                                          const LangOptions &LangOpts) {
+/// Is this builtin supported according to the given language options?
+static bool builtinIsSupported(const Builtin::Info &BuiltinInfo,
+                               const LangOptions &LangOpts) {
   bool BuiltinsUnsupported =
-      (LangOpts.NoBuiltin || LangOpts.isNoBuiltinFunc(BuiltinInfo.Name)) &&
-      strchr(BuiltinInfo.Attributes, 'f');
+      LangOpts.NoBuiltin && strchr(BuiltinInfo.Attributes, 'f') != nullptr;
   bool CorBuiltinsUnsupported =
       !LangOpts.Coroutines && (BuiltinInfo.Langs & COR_LANG);
   bool MathBuiltinsUnsupported =
@@ -111,6 +115,19 @@ void Builtin::Context::initializeBuiltins(IdentifierTable &Table,
   for (unsigned i = 0, e = AuxTSRecords.size(); i != e; ++i)
     Table.get(AuxTSRecords[i].Name)
         .setBuiltinID(i + Builtin::FirstTSBuiltin + TSRecords.size());
+
+  // Step #4: Unregister any builtins specified by -fno-builtin-foo.
+  for (llvm::StringRef Name : LangOpts.NoBuiltinFuncs) {
+    bool InStdNamespace = Name.consume_front("std-");
+    auto NameIt = Table.find(Name);
+    if (NameIt != Table.end()) {
+      unsigned ID = NameIt->second->getBuiltinID();
+      if (ID != Builtin::NotBuiltin && isPredefinedLibFunction(ID) &&
+          isInStdNamespace(ID) == InStdNamespace) {
+        Table.get(Name).setBuiltinID(Builtin::NotBuiltin);
+      }
+    }
+  }
 }
 
 unsigned Builtin::Context::getRequiredVectorWidth(unsigned ID) const {
@@ -190,8 +207,7 @@ bool Builtin::Context::performsCallback(unsigned ID,
 }
 
 bool Builtin::Context::canBeRedeclared(unsigned ID) const {
-  return ID == Builtin::NotBuiltin ||
-         ID == Builtin::BI__va_start ||
-         (!hasReferenceArgsOrResult(ID) &&
-          !hasCustomTypechecking(ID));
+  return ID == Builtin::NotBuiltin || ID == Builtin::BI__va_start ||
+         (!hasReferenceArgsOrResult(ID) && !hasCustomTypechecking(ID)) ||
+         isInStdNamespace(ID);
 }

clang/lib/CodeGen/CGBuiltin.cpp

@@ -4725,6 +4725,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     }
     break;
 
+  // C++ std:: builtins.
+  case Builtin::BImove:
+  case Builtin::BImove_if_noexcept:
+  case Builtin::BIforward:
+    return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
   case Builtin::BI__GetExceptionInfo: {
     if (llvm::GlobalVariable *GV =
             CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))

clang/lib/CodeGen/CGCall.cpp

@@ -1805,6 +1805,8 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
 
   if (AttrOnCallSite) {
     // Attributes that should go on the call site only.
+    // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
+    // the -fno-builtin-foo list.
     if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
       FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
     if (!CodeGenOpts.TrapFuncName.empty())

clang/lib/Sema/SemaChecking.cpp

@@ -2130,6 +2130,18 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
 
     TheCall->setType(Context.VoidPtrTy);
     break;
+  case Builtin::BImove:
+  case Builtin::BImove_if_noexcept:
+  case Builtin::BIforward:
+    if (checkArgCount(*this, TheCall, 1))
+      return ExprError();
+    if (!Context.hasSameUnqualifiedType(TheCall->getType(),
+                                        TheCall->getArg(0)->getType())) {
+      Diag(TheCall->getBeginLoc(), diag::err_builtin_move_forward_unsupported)
+          << (BuiltinID == Builtin::BIforward);
+      return ExprError();
+    }
+    break;
   // OpenCL v2.0, s6.13.16 - Pipe functions
   case Builtin::BIread_pipe:
   case Builtin::BIwrite_pipe: