aarch64: Add AArch64 Kernel Control Flow Integrity implementation

Implement AArch64-specific KCFI backend providing runtime validation of
indirect function calls with ARM exception handling infrastructure.

Core AArch64 KCFI Features:
* Function preamble generation using .word directives for type ID storage
  at -4 byte offset from function entry point (no prefix NOPs needed due
  to 4-byte instruction alignment)
* Enhanced debugging through ESR (Exception Syndrome Register) encoding
  in BRK instruction immediate values for precise failure analysis
* Scratch register allocation using w16/w17 (x16/x17) following
  AArch64 procedure call standard for intra-procedure-call registers
* Support for both regular calls (BLR) and sibling calls (BR) with
  appropriate register usage and jump instructions

Assembly Code Generation:
* Atomic bundled KCFI check + call/branch sequences using UNSPECV_KCFI_CHECK
  to prevent optimizer separation and maintain security properties
* Constant loading for type IDs using MOV/MOVK instruction pairs
  for values requiring 32-bit representation
* Direct comparison approach using CMP instruction for type validation
  without arithmetic operations (contrast with x86_64's additive approach)

Assembly Code Pattern for AArch64:
  ldur w16, [target, #-4]       ; Load actual type ID from preamble
  mov  w17, #type_id_low        ; Load expected type (lower 16 bits)
  movk w17, #type_id_high, lsl gcc-mirror#16  ; Load upper 16 bits if needed
  cmp  w16, w17                 ; Compare type IDs directly
  b.eq .Lpass                   ; Branch if types match
  .Ltrap: brk #esr_value        ; Enhanced trap with register info
  .Lpass: blr/br target         ; Execute validated indirect transfer

ESR (Exception Syndrome Register) Integration:
* BRK instruction immediate encoding format:
  0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31)
  * TypeIndex indicates which W register contains expected type (W17 = 17)
  * AddrIndex indicates which X register contains target address (0-30)
  * Example: brk #33313 (0x8221) = expected type in W17, target address in X1
* Enables kernel exception handlers to precisely identify KCFI violation context
* Supports advanced debugging and forensic analysis of control flow attacks

AArch64-Specific Optimizations:
* No prefix NOP calculation needed due to natural 4-byte instruction alignment
* Type ID storage using single .word directive in function preambles
* Register allocator integration via explicit w16/w17 clobber annotations
* Function label emission coordination through ASM_OUTPUT_FUNCTION_LABEL macro
  redirection to aarch64_declare_function_name() for preamble integration
* Support for large immediate values with MOV/MOVK instruction generation

Target Hook Implementation:
* aarch64_kcfi_calculate_prefix_nops(): Returns 0 (no alignment needed)
* aarch64_kcfi_gen_checked_call(): Bundled check+call RTL generation
* aarch64_kcfi_emit_type_id_instruction(): .word directive emission
* aarch64_kcfi_add_clobbers(): w16/w17 register constraint management
* Integration in aarch64_override_options() for initialization

Machine Description Integration:
* UNSPECV_KCFI_CHECK unspec for atomic check+call bundling
* Support for both regular calls and sibling calls with distinct patterns
* Runtime ESR value calculation for accurate register encoding
* Clobber specifications for w16 (loaded type) and w17 (expected type)

Security Properties:
* Direct comparison-based type validation with immediate trap on mismatch
* Enhanced exception context through ESR encoding for precise failure analysis
* Tamper-resistant type ID storage with ARM exception infrastructure integration
* Support for cross-compilation and accurate register allocation across targets

Build and run tested with Linux kernel ARCH=arm64.

Signed-off-by: Kees Cook <[email protected]>

Author: kees

gcc/config/aarch64/aarch64-protos.h

@@ -1285,4 +1285,8 @@ extern unsigned aarch64_stack_alignment (const_tree exp, unsigned align);
 extern rtx aarch64_gen_compare_zero_and_branch (rtx_code code, rtx x,
 						rtx_code_label *label);
 
+/* KCFI support.  */
+extern void aarch64_kcfi_init (void);
+extern void kcfi_emit_trap_with_section (FILE *file, rtx trap_label_rtx);
+
 #endif /* GCC_AARCH64_PROTOS_H */

gcc/config/aarch64/aarch64.cc

@@ -83,6 +83,7 @@
 #include "rtlanal.h"
 #include "tree-dfa.h"
 #include "asan.h"
+#include "kcfi.h"
 #include "aarch64-elf-metadata.h"
 #include "aarch64-feature-deps.h"
 #include "config/arm/aarch-common.h"
@@ -19521,6 +19522,9 @@ aarch64_override_options (void)
 
   aarch64_override_options_internal (&global_options);
 
+  /* Initialize KCFI target hooks for AArch64.  */
+  aarch64_kcfi_init ();
+
   /* Save these options as the default ones in case we push and pop them later
      while processing functions with potential target attributes.  */
   target_option_default_node = target_option_current_node
@@ -25578,9 +25582,13 @@ aarch64_declare_function_name (FILE *stream, const char* name,
 
   aarch64_asm_output_variant_pcs (stream, fndecl, name);
 
+  /* Emit KCFI preamble for non-patchable functions.  */
+  kcfi_emit_preamble_if_needed (stream, fndecl, false, 0, name);
+
   /* Don't forget the type directive for ELF.  */
   ASM_OUTPUT_TYPE_DIRECTIVE (stream, name, "function");
-  ASM_OUTPUT_FUNCTION_LABEL (stream, name, fndecl);
+  /* Output function label directly to avoid recursion with our ASM_OUTPUT_FUNCTION_LABEL macro.  */
+  assemble_function_label_raw (stream, name);
 
   cfun->machine->label_is_assembled = true;
 }
@@ -32811,6 +32819,111 @@ aarch64_libgcc_floating_mode_supported_p
 #undef TARGET_DOCUMENTATION_NAME
 #define TARGET_DOCUMENTATION_NAME "AArch64"
 
+
+/* AArch64 doesn't need prefix NOPs (instructions are already 4-byte aligned) */
+static int
+aarch64_kcfi_calculate_prefix_nops (HOST_WIDE_INT prefix_nops ATTRIBUTE_UNUSED)
+{
+  /* AArch64 instructions are 4-byte aligned, no prefix NOPs needed for KCFI preamble.  */
+  return 0;
+}
+
+/* Emit AArch64-specific type ID instruction.  */
+static void
+aarch64_kcfi_emit_type_id_instruction (FILE *file, uint32_t type_id)
+{
+  /* Emit type ID as a 32-bit word.  */
+  fprintf (file, "\t.word 0x%08x\n", type_id);
+}
+
+
+
+/* Generate AArch64 KCFI checked call bundle.  */
+static rtx
+aarch64_kcfi_gen_checked_call (rtx call_insn, rtx target_reg, uint32_t expected_type,
+			       HOST_WIDE_INT prefix_nops)
+{
+  /* For AArch64, we create an RTL bundle that combines the KCFI check
+     with the call instruction in an atomic sequence.  */
+
+  if (!REG_P (target_reg))
+    {
+      /* If not a register, load it into x16.  */
+      rtx temp = gen_rtx_REG (Pmode, 16);
+      emit_move_insn (temp, target_reg);
+      target_reg = temp;
+    }
+
+  /* Generate the bundled KCFI check + call pattern.  */
+  rtx pattern;
+  if (CALL_P (call_insn))
+    {
+      rtx call_pattern = PATTERN (call_insn);
+
+      /* Create labels used by both call and sibcall patterns.  */
+      rtx pass_label = gen_label_rtx ();
+      rtx trap_label = gen_label_rtx ();
+
+      /* Check if it's a sibling call.  */
+      if (find_reg_note (call_insn, REG_NORETURN, NULL_RTX)
+	  || (GET_CODE (call_pattern) == PARALLEL
+	      && GET_CODE (XVECEXP (call_pattern, 0, XVECLEN (call_pattern, 0) - 1)) == RETURN))
+	{
+	  /* Generate sibling call bundle.  */
+	  pattern = gen_aarch64_kcfi_checked_sibcall (target_reg,
+						      gen_int_mode (expected_type, SImode),
+						      gen_int_mode (prefix_nops, SImode),
+						      pass_label,
+						      trap_label);
+	}
+      else
+	{
+	  /* Generate regular call bundle.  */
+	  pattern = gen_aarch64_kcfi_checked_call (target_reg,
+						   gen_int_mode (expected_type, SImode),
+						   gen_int_mode (prefix_nops, SImode),
+						   pass_label,
+						   trap_label);
+	}
+    }
+  else
+    {
+      error ("KCFI: Expected call instruction");
+      return NULL_RTX;
+    }
+
+  return pattern;
+}
+
+/* Add AArch64-specific register clobbers for KCFI calls.  */
+static void
+aarch64_kcfi_add_clobbers (rtx_insn *call_insn)
+{
+  /* AArch64 KCFI uses w16 and w17 (x16 and x17) as scratch registers.  */
+  rtx usage = CALL_INSN_FUNCTION_USAGE (call_insn);
+
+  /* Add w16 (x16) clobber.  */
+  clobber_reg (&usage, gen_rtx_REG (SImode, 16));
+
+  /* Add w17 (x17) clobber.  */
+  clobber_reg (&usage, gen_rtx_REG (SImode, 17));
+
+  CALL_INSN_FUNCTION_USAGE (call_insn) = usage;
+}
+
+/* Initialize AArch64 KCFI target hooks.  */
+void
+aarch64_kcfi_init (void)
+{
+  if (flag_sanitize & SANITIZE_KCFI)
+    {
+      kcfi_target.gen_kcfi_checked_call = aarch64_kcfi_gen_checked_call;
+      kcfi_target.add_kcfi_clobbers = aarch64_kcfi_add_clobbers;
+      kcfi_target.calculate_prefix_nops = aarch64_kcfi_calculate_prefix_nops;
+      kcfi_target.emit_type_id_instruction = aarch64_kcfi_emit_type_id_instruction;
+    }
+}
+
 struct gcc_target targetm = TARGET_INITIALIZER;
 
 #include "gt-aarch64.h"

gcc/config/aarch64/aarch64.h

@@ -1665,6 +1665,11 @@ enum class aarch64_tristate_mode : int { NO, YES, MAYBE };
    applied.  */
 #define HARDREG_PRE_REGNOS { FPM_REGNUM, 0 }
 
+/* Intercept all function label output including cold partitions for KCFI preamble emission.  */
+#undef ASM_OUTPUT_FUNCTION_LABEL
+#define ASM_OUTPUT_FUNCTION_LABEL(FILE, NAME, DECL) \
+  aarch64_declare_function_name ((FILE), (NAME), (DECL))
+
 #endif
 
 #endif /* GCC_AARCH64_H */

gcc/config/aarch64/aarch64.md

@@ -417,6 +417,7 @@
     UNSPECV_TCANCEL		; Represent transaction cancel.
     UNSPEC_RNDR			; Represent RNDR
     UNSPEC_RNDRRS		; Represent RNDRRS
+    UNSPECV_KCFI_CHECK		; Represent KCFI check bundled with call
   ]
 )
 
@@ -1316,6 +1317,142 @@
   "brk #1000"
   [(set_attr "type" "trap")])
 
+;; KCFI bundled check and call patterns
+;; These combine the KCFI check with the call in an atomic sequence
+
+(define_insn "aarch64_kcfi_checked_call"
+  [(parallel [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
+                    (const_int 0))
+              (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
+              (unspec_volatile:DI [(match_operand:SI 1 "const_int_operand" "n")  ; type_id
+                                   (match_operand:SI 2 "const_int_operand" "n")  ; prefix_nops
+                                   (label_ref (match_operand 3))  ; pass label
+                                   (label_ref (match_operand 4))] ; trap label
+                                  UNSPECV_KCFI_CHECK)
+              (clobber (reg:DI LR_REGNUM))
+              (clobber (reg:SI 16))  ; w16 - scratch for loaded type
+              (clobber (reg:SI 17))])] ; w17 - scratch for expected type
+  "flag_sanitize & SANITIZE_KCFI"
+  "*
+  {
+    uint32_t type_id = INTVAL (operands[1]);
+    HOST_WIDE_INT prefix_nops = INTVAL (operands[2]);
+    HOST_WIDE_INT offset = -(4 + prefix_nops);
+
+    /* AArch64 KCFI check sequence:
+       1. Load actual type from function preamble
+       2. Load expected type
+       3. Compare and branch if equal
+       4. Trap if mismatch
+       5. Call target.  */
+
+    static char ldur_buffer[64];
+    sprintf (ldur_buffer, \"ldur\\tw16, [%%0, #%ld]\", offset);
+    output_asm_insn (ldur_buffer, operands);
+
+    /* Load expected type - may need multiple instructions for large constants.  */
+    if ((type_id & 0xffff0000) == 0)
+      {
+        static char mov_buffer[64];
+        sprintf (mov_buffer, \"mov\\tw17, #%u\", type_id);
+        output_asm_insn (mov_buffer, operands);
+      }
+    else
+      {
+        static char mov_buffer[64], movk_buffer[64];
+        sprintf (mov_buffer, \"mov\\tw17, #%u\", type_id & 0xffff);
+        output_asm_insn (mov_buffer, operands);
+        sprintf (movk_buffer, \"movk\\tw17, #%u, lsl #16\", (type_id >> 16) & 0xffff);
+        output_asm_insn (movk_buffer, operands);
+      }
+
+    output_asm_insn (\"cmp\\tw16, w17\", operands);
+    output_asm_insn (\"b.eq\\t%l3\", operands);
+
+    /* Generate unique trap ID and emit trap.  */
+    output_asm_insn (\"%l4:\", operands);
+
+    /* Calculate and emit BRK with ESR encoding.  */
+    unsigned type_index = 17;  /* w17 contains expected type.  */
+    unsigned addr_index = REGNO (operands[0]) - R0_REGNUM;
+    unsigned esr_value = 0x8000 | ((type_index & 31) << 5) | (addr_index & 31);
+
+    static char brk_buffer[32];
+    sprintf (brk_buffer, \"brk\\t#%u\", esr_value);
+    output_asm_insn (brk_buffer, operands);
+
+    output_asm_insn (\"%l3:\", operands);
+    output_asm_insn (\"\\tblr\\t%0\", operands);
+
+    return \"\";
+  }"
+  [(set_attr "type" "call")
+   (set_attr "length" "24")])
+
+(define_insn "aarch64_kcfi_checked_sibcall"
+  [(parallel [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
+                    (const_int 0))
+              (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
+              (unspec_volatile:DI [(match_operand:SI 1 "const_int_operand" "n")  ; type_id
+                                   (match_operand:SI 2 "const_int_operand" "n")  ; prefix_nops
+                                   (label_ref (match_operand 3))  ; pass label
+                                   (label_ref (match_operand 4))] ; trap label
+                                  UNSPECV_KCFI_CHECK)
+              (return)
+              (clobber (reg:SI 16))  ; w16 - scratch for loaded type
+              (clobber (reg:SI 17))])] ; w17 - scratch for expected type
+  "flag_sanitize & SANITIZE_KCFI"
+  "*
+  {
+    uint32_t type_id = INTVAL (operands[1]);
+    HOST_WIDE_INT prefix_nops = INTVAL (operands[2]);
+    HOST_WIDE_INT offset = -(4 + prefix_nops);
+
+    /* AArch64 KCFI check sequence for sibling calls.  */
+
+    static char ldur_buffer[64];
+    sprintf (ldur_buffer, \"ldur\\tw16, [%%0, #%ld]\", offset);
+    output_asm_insn (ldur_buffer, operands);
+
+    /* Load expected type.  */
+    if ((type_id & 0xffff0000) == 0)
+      {
+        static char mov_buffer[64];
+        sprintf (mov_buffer, \"mov\\tw17, #%u\", type_id);
+        output_asm_insn (mov_buffer, operands);
+      }
+    else
+      {
+        static char mov_buffer[64], movk_buffer[64];
+        sprintf (mov_buffer, \"mov\\tw17, #%u\", type_id & 0xffff);
+        output_asm_insn (mov_buffer, operands);
+        sprintf (movk_buffer, \"movk\\tw17, #%u, lsl #16\", (type_id >> 16) & 0xffff);
+        output_asm_insn (movk_buffer, operands);
+      }
+
+    output_asm_insn (\"cmp\\tw16, w17\", operands);
+    output_asm_insn (\"b.eq\\t%l3\", operands);
+
+    /* Generate unique trap ID and emit trap.  */
+    output_asm_insn (\"%l4:\", operands);
+
+    /* Calculate and emit BRK with ESR encoding.  */
+    unsigned type_index = 17;  /* w17 contains expected type.  */
+    unsigned addr_index = REGNO (operands[0]) - R0_REGNUM;
+    unsigned esr_value = 0x8000 | ((type_index & 31) << 5) | (addr_index & 31);
+
+    static char brk_buffer[32];
+    sprintf (brk_buffer, \"brk\\t#%u\", esr_value);
+    output_asm_insn (brk_buffer, operands);
+
+    output_asm_insn (\"%l3:\", operands);
+    output_asm_insn (\"\\tbr\\t%0\", operands);
+
+    return \"\";
+  }"
+  [(set_attr "type" "branch")
+   (set_attr "length" "24")])
+
 (define_expand "prologue"
   [(clobber (const_int 0))]
   ""