Skip to content

Commit 341d674

Browse files
paulwalker-armpaulwalker-arm
authored
[LLVM][AArch64][CodeGen] Mark FFR as a reserved register. (#83437)
This allows the removal of FFR related psuedo nodes that only existed to work round machine verifier failures.
1 parent 90e97e7 commit 341d674

28 files changed

+411
-514
lines changed

‎llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,10 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
437437
if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
438438
markSuperRegs(Reserved, AArch64::W16);
439439

440+
// FFR is modelled as global state that cannot be allocated.
441+
if (MF.getSubtarget<AArch64Subtarget>().hasSVE())
442+
Reserved.set(AArch64::FFR);
443+
440444
// SME tiles are not allocatable.
441445
if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
442446
for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA))

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -488,11 +488,11 @@ def AArch64fmin_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_fmin, AAr
488488
def AArch64fmax_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_fmax, AArch64fmax_p>;
489489

490490
let Predicates = [HasSVE] in {
491-
defm RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
492-
def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
493-
defm RDFFR_P : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>;
494-
def SETFFR : sve_int_setffr<"setffr", int_aarch64_sve_setffr>;
495-
def WRFFR : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>;
491+
def RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
492+
def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
493+
def RDFFR_P : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>;
494+
def SETFFR : sve_int_setffr<"setffr", int_aarch64_sve_setffr>;
495+
def WRFFR : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>;
496496
} // End HasSVE
497497

498498
let Predicates = [HasSVEorSME] in {

‎llvm/lib/Target/AArch64/AArch64SchedA510.td

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,24 +1198,24 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_
11981198
def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRR$")>;
11991199

12001200
// Non temporal gather load, vector + scalar 32-bit element size
1201-
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S_REAL$",
1202-
"^LDNT1S[BH]_ZZR_S_REAL$")>;
1201+
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S$",
1202+
"^LDNT1S[BH]_ZZR_S$")>;
12031203

12041204
// Non temporal gather load, vector + scalar 64-bit element size
1205-
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>;
1206-
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instrs LDNT1D_ZZR_D_REAL)>;
1205+
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D$")>;
1206+
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instrs LDNT1D_ZZR_D)>;
12071207

12081208
// Contiguous first faulting load, scalar + scalar
1209-
def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDFF1[BHWD]_REAL$",
1210-
"^LDFF1S?B_[HSD]_REAL$",
1211-
"^LDFF1S?H_[SD]_REAL$",
1212-
"^LDFF1S?W_D_REAL$")>;
1209+
def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDFF1[BHWD]$",
1210+
"^LDFF1S?B_[HSD]$",
1211+
"^LDFF1S?H_[SD]$",
1212+
"^LDFF1S?W_D$")>;
12131213

12141214
// Contiguous non faulting load, scalar + imm
1215-
def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDNF1[BHWD]_IMM_REAL$",
1216-
"^LDNF1S?B_[HSD]_IMM_REAL$",
1217-
"^LDNF1S?H_[SD]_IMM_REAL$",
1218-
"^LDNF1S?W_D_IMM_REAL$")>;
1215+
def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDNF1[BHWD]_IMM$",
1216+
"^LDNF1S?B_[HSD]_IMM$",
1217+
"^LDNF1S?H_[SD]_IMM$",
1218+
"^LDNF1S?W_D_IMM$")>;
12191219

12201220
// Contiguous Load two structures to two vectors, scalar + imm
12211221
def : InstRW<[CortexA510MCWrite<3, 1, CortexA510UnitLdSt>], (instregex "^LD2[BHWD]_IMM$")>;
@@ -1236,28 +1236,28 @@ def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHW
12361236
def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHWD]$")>;
12371237

12381238
// Gather load, vector + imm, 32-bit element size
1239-
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$",
1240-
"^GLD(FF)?1W_IMM_REAL$")>;
1239+
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM$",
1240+
"^GLD(FF)?1W_IMM$")>;
12411241

12421242
// Gather load, vector + imm, 64-bit element size
1243-
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$",
1244-
"^GLD(FF)?1D_IMM_REAL$")>;
1243+
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$",
1244+
"^GLD(FF)?1D_IMM$")>;
12451245

12461246
// Gather load, 64-bit element size
12471247
def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>],
1248-
(instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW_(SCALED_)?REAL$",
1249-
"^GLD(FF)?1S?[BHW]_D_(SCALED_)?REAL$",
1250-
"^GLD(FF)?1D_[SU]XTW_(SCALED_)?REAL$",
1251-
"^GLD(FF)?1D_(SCALED_)?REAL$")>;
1248+
(instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW(_SCALED)?$",
1249+
"^GLD(FF)?1S?[BHW]_D(_SCALED)?$",
1250+
"^GLD(FF)?1D_[SU]XTW(_SCALED)?$",
1251+
"^GLD(FF)?1D(_SCALED)?$")>;
12521252

12531253
// Gather load, 32-bit scaled offset
12541254
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>],
1255-
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$",
1256-
"^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>;
1255+
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$",
1256+
"^GLD(FF)?1W_[SU]XTW_SCALED")>;
12571257

12581258
// Gather load, 32-bit unpacked unscaled offset
1259-
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$",
1260-
"^GLD(FF)?1W_[SU]XTW_REAL$")>;
1259+
def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$",
1260+
"^GLD(FF)?1W_[SU]XTW$")>;
12611261

12621262
def : InstRW<[CortexA510Write<0, CortexA510UnitVALU>], (instregex "^PRF(B|H|W|D).*")>;
12631263
// SVE Store instructions
@@ -1357,10 +1357,10 @@ def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[BHW]_D$",
13571357
// -----------------------------------------------------------------------------
13581358

13591359
// Read first fault register, unpredicated
1360-
def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs RDFFR_P_REAL)>;
1360+
def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs RDFFR_P)>;
13611361

13621362
// Read first fault register, predicated
1363-
def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFR_PPz_REAL)>;
1363+
def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFR_PPz)>;
13641364

13651365
// Read first fault register and set flags
13661366
def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFRS_PPz)>;

‎llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2110,24 +2110,24 @@ def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNT1[BHWD]_ZRI$")>;
21102110
def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDNT1[BHWD]_ZRR$")>;
21112111

21122112
// Non temporal gather load, vector + scalar 32-bit element size
2113-
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^LDNT1[BHW]_ZZR_S_REAL$",
2114-
"^LDNT1S[BH]_ZZR_S_REAL$")>;
2113+
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^LDNT1[BHW]_ZZR_S$",
2114+
"^LDNT1S[BH]_ZZR_S$")>;
21152115

21162116
// Non temporal gather load, vector + scalar 64-bit element size
2117-
def : InstRW<[N2Write_10cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>;
2118-
def : InstRW<[N2Write_10cyc_2L_2V1], (instrs LDNT1D_ZZR_D_REAL)>;
2117+
def : InstRW<[N2Write_10cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D$")>;
2118+
def : InstRW<[N2Write_10cyc_2L_2V1], (instrs LDNT1D_ZZR_D)>;
21192119

21202120
// Contiguous first faulting load, scalar + scalar
2121-
def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]_REAL$",
2122-
"^LDFF1S?B_[HSD]_REAL$",
2123-
"^LDFF1S?H_[SD]_REAL$",
2124-
"^LDFF1S?W_D_REAL$")>;
2121+
def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]$",
2122+
"^LDFF1S?B_[HSD]$",
2123+
"^LDFF1S?H_[SD]$",
2124+
"^LDFF1S?W_D$")>;
21252125

21262126
// Contiguous non faulting load, scalar + imm
2127-
def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM_REAL$",
2128-
"^LDNF1S?B_[HSD]_IMM_REAL$",
2129-
"^LDNF1S?H_[SD]_IMM_REAL$",
2130-
"^LDNF1S?W_D_IMM_REAL$")>;
2127+
def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM$",
2128+
"^LDNF1S?B_[HSD]_IMM$",
2129+
"^LDNF1S?H_[SD]_IMM$",
2130+
"^LDNF1S?W_D_IMM$")>;
21312131

21322132
// Contiguous Load two structures to two vectors, scalar + imm
21332133
def : InstRW<[N2Write_8cyc_1L_1V], (instregex "^LD2[BHWD]_IMM$")>;
@@ -2148,28 +2148,28 @@ def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^LD4[BHWD]_IMM$")>;
21482148
def : InstRW<[N2Write_10cyc_2L_2V_2S], (instregex "^LD4[BHWD]$")>;
21492149

21502150
// Gather load, vector + imm, 32-bit element size
2151-
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$",
2152-
"^GLD(FF)?1W_IMM_REAL$")>;
2151+
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM$",
2152+
"^GLD(FF)?1W_IMM$")>;
21532153

21542154
// Gather load, vector + imm, 64-bit element size
2155-
def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$",
2156-
"^GLD(FF)?1D_IMM_REAL$")>;
2155+
def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$",
2156+
"^GLD(FF)?1D_IMM$")>;
21572157

21582158
// Gather load, 64-bit element size
21592159
def : InstRW<[N2Write_9cyc_2L_2V],
2160-
(instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW_(SCALED_)?REAL$",
2161-
"^GLD(FF)?1S?[BHW]_D_(SCALED_)?REAL$",
2162-
"^GLD(FF)?1D_[SU]XTW_(SCALED_)?REAL$",
2163-
"^GLD(FF)?1D_(SCALED_)?REAL$")>;
2160+
(instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW(_SCALED)?$",
2161+
"^GLD(FF)?1S?[BHW]_D(_SCALED)?$",
2162+
"^GLD(FF)?1D_[SU]XTW(_SCALED)?$",
2163+
"^GLD(FF)?1D(_SCALED)?$")>;
21642164

21652165
// Gather load, 32-bit scaled offset
21662166
def : InstRW<[N2Write_10cyc_2L_2V],
2167-
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$",
2168-
"^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>;
2167+
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$",
2168+
"^GLD(FF)?1W_[SU]XTW_SCALED")>;
21692169

21702170
// Gather load, 32-bit unpacked unscaled offset
2171-
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$",
2172-
"^GLD(FF)?1W_[SU]XTW_REAL$")>;
2171+
def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$",
2172+
"^GLD(FF)?1W_[SU]XTW$")>;
21732173

21742174
// SVE Store instructions
21752175
// -----------------------------------------------------------------------------
@@ -2268,10 +2268,10 @@ def : InstRW<[N2Write_2cyc_1L01_1V], (instregex "^SST1[BHW]_D$",
22682268
// -----------------------------------------------------------------------------
22692269

22702270
// Read first fault register, unpredicated
2271-
def : InstRW<[N2Write_2cyc_1M0], (instrs RDFFR_P_REAL)>;
2271+
def : InstRW<[N2Write_2cyc_1M0], (instrs RDFFR_P)>;
22722272

22732273
// Read first fault register, predicated
2274-
def : InstRW<[N2Write_3cyc_1M0_1M], (instrs RDFFR_PPz_REAL)>;
2274+
def : InstRW<[N2Write_3cyc_1M0_1M], (instrs RDFFR_PPz)>;
22752275

22762276
// Read first fault register and set flags
22772277
def : InstRW<[N2Write_4cyc_2M0_2M], (instrs RDFFRS_PPz)>;

‎llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1714,17 +1714,17 @@ def : InstRW<[V1Write_7c_1L01_1S], (instrs LDNT1H_ZRR)>;
17141714
def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDNT1[BWD]_ZRR$")>;
17151715

17161716
// Contiguous first faulting load, scalar + scalar
1717-
def : InstRW<[V1Write_7c_1L01_1S], (instregex "^LDFF1H_REAL$",
1718-
"^LDFF1S?H_[SD]_REAL$")>;
1719-
def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDFF1[BWD]_REAL$",
1720-
"^LDFF1S?B_[HSD]_REAL$",
1721-
"^LDFF1S?W_D_REAL$")>;
1717+
def : InstRW<[V1Write_7c_1L01_1S], (instregex "^LDFF1H$",
1718+
"^LDFF1S?H_[SD]$")>;
1719+
def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDFF1[BWD]$",
1720+
"^LDFF1S?B_[HSD]$",
1721+
"^LDFF1S?W_D$")>;
17221722

17231723
// Contiguous non faulting load, scalar + imm
1724-
def : InstRW<[V1Write_6c_1L01], (instregex "^LDNF1[BHWD]_IMM_REAL$",
1725-
"^LDNF1S?B_[HSD]_IMM_REAL$",
1726-
"^LDNF1S?H_[SD]_IMM_REAL$",
1727-
"^LDNF1S?W_D_IMM_REAL$")>;
1724+
def : InstRW<[V1Write_6c_1L01], (instregex "^LDNF1[BHWD]_IMM$",
1725+
"^LDNF1S?B_[HSD]_IMM$",
1726+
"^LDNF1S?H_[SD]_IMM$",
1727+
"^LDNF1S?W_D_IMM$")>;
17281728

17291729
// Contiguous Load two structures to two vectors, scalar + imm
17301730
def : InstRW<[V1Write_8c_2L01_2V01], (instregex "^LD2[BHWD]_IMM$")>;
@@ -1746,25 +1746,25 @@ def : InstRW<[V1Write_12c_4L01_4V01], (instregex "^LD4[BHWD]_IMM$")>;
17461746
def : InstRW<[V1Write_13c_4L01_2S_4V01], (instregex "^LD4[BHWD]$")>;
17471747

17481748
// Gather load, vector + imm, 32-bit element size
1749-
def : InstRW<[V1Write_11c_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$",
1750-
"^GLD(FF)?1W_IMM_REAL$")>;
1749+
def : InstRW<[V1Write_11c_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM$",
1750+
"^GLD(FF)?1W_IMM$")>;
17511751

17521752
// Gather load, vector + imm, 64-bit element size
17531753
def : InstRW<[V1Write_9c_2L_2V],
1754-
(instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$",
1755-
"^GLD(FF)?1S?[BHW]_D_([SU]XTW_)?(SCALED_)?REAL$",
1756-
"^GLD(FF)?1D_IMM_REAL$",
1757-
"^GLD(FF)?1D_([SU]XTW_)?(SCALED_)?REAL$")>;
1754+
(instregex "^GLD(FF)?1S?[BHW]_D_IMM$",
1755+
"^GLD(FF)?1S?[BHW]_D(_[SU]XTW)?(_SCALED)?$",
1756+
"^GLD(FF)?1D_IMM$",
1757+
"^GLD(FF)?1D(_[SU]XTW)?(_SCALED)?$")>;
17581758

17591759
// Gather load, 32-bit scaled offset
17601760
def : InstRW<[V1Write_11c_2L_2V],
1761-
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$",
1762-
"^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>;
1761+
(instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$",
1762+
"^GLD(FF)?1W_[SU]XTW_SCALED")>;
17631763

17641764
// Gather load, 32-bit unpacked unscaled offset
17651765
def : InstRW<[V1Write_9c_1L_1V],
1766-
(instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$",
1767-
"^GLD(FF)?1W_[SU]XTW_REAL$")>;
1766+
(instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$",
1767+
"^GLD(FF)?1W_[SU]XTW$")>;
17681768

17691769
// Prefetch
17701770
// NOTE: This is not specified in the SOG.
@@ -1848,12 +1848,12 @@ def : InstRW<[V1Write_6c_1L01_1V], (instregex "^SST1[BHW]_D_IMM$",
18481848
// Read first fault register, unpredicated
18491849
// Set first fault register
18501850
// Write to first fault register
1851-
def : InstRW<[V1Write_2c_1M0], (instrs RDFFR_P_REAL,
1851+
def : InstRW<[V1Write_2c_1M0], (instrs RDFFR_P,
18521852
SETFFR,
18531853
WRFFR)>;
18541854

18551855
// Read first fault register, predicated
1856-
def : InstRW<[V1Write_3c_2M0], (instrs RDFFR_PPz_REAL)>;
1856+
def : InstRW<[V1Write_3c_2M0], (instrs RDFFR_PPz)>;
18571857

18581858
// Read first fault register and set flags
18591859
def : InstRW<[V1Write_4c_1M], (instrs RDFFRS_PPz)>;

0 commit comments

Comments
 (0)