Skip to content

Commit 88f04bd

Browse files
kosarevkosarev
committed
[AMDGPU][GFX10] Support base+soffset+offset SMEM loads.
Also makes a step towards resolving llvm#38652 Reviewed By: foad, dp Differential Revision: https://reviews.llvm.org/D125117
1 parent ca75ac5 commit 88f04bd

File tree

7 files changed

+264
-79
lines changed

7 files changed

+264
-79
lines changed

‎llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,13 @@ void AMDGPUInstPrinter::printSMEMOffset(const MCInst *MI, unsigned OpNo,
191191
O << formatHex(MI->getOperand(OpNo).getImm());
192192
}
193193

194+
void AMDGPUInstPrinter::printSMEMOffsetMod(const MCInst *MI, unsigned OpNo,
195+
const MCSubtargetInfo &STI,
196+
raw_ostream &O) {
197+
O << " offset:";
198+
printSMEMOffset(MI, OpNo, STI, O);
199+
}
200+
194201
void AMDGPUInstPrinter::printSMRDLiteralOffset(const MCInst *MI, unsigned OpNo,
195202
const MCSubtargetInfo &STI,
196203
raw_ostream &O) {

‎llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ class AMDGPUInstPrinter : public MCInstPrinter {
6464
const MCSubtargetInfo &STI, raw_ostream &O);
6565
void printSMEMOffset(const MCInst *MI, unsigned OpNo,
6666
const MCSubtargetInfo &STI, raw_ostream &O);
67+
void printSMEMOffsetMod(const MCInst *MI, unsigned OpNo,
68+
const MCSubtargetInfo &STI, raw_ostream &O);
6769
void printSMRDLiteralOffset(const MCInst *MI, unsigned OpNo,
6870
const MCSubtargetInfo &STI, raw_ostream &O);
6971
void printGDS(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,

‎llvm/lib/Target/AMDGPU/SIInstrInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4471,7 +4471,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
44714471
if (MI.mayStore()) {
44724472
// The register offset form of scalar stores may only use m0 as the
44734473
// soffset register.
4474-
const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
4474+
const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
44754475
if (Soff && Soff->getReg() != AMDGPU::M0) {
44764476
ErrInfo = "scalar stores must use m0 as offset register";
44774477
return false;
@@ -5314,7 +5314,7 @@ void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
53145314
Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
53155315
SBase->setReg(SGPR);
53165316
}
5317-
MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
5317+
MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset);
53185318
if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
53195319
Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
53205320
SOff->setReg(SGPR);

‎llvm/lib/Target/AMDGPU/SMInstructions.td‎

Lines changed: 70 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,19 @@ def smrd_offset_8 : NamedOperandU32<"SMRDOffset8",
1111
let OperandType = "OPERAND_IMMEDIATE";
1212
}
1313

14-
def smem_offset : NamedOperandU32<"SMEMOffset",
15-
NamedMatchClass<"SMEMOffset">> {
14+
class SMEMOffset : NamedOperandU32<"SMEMOffset",
15+
NamedMatchClass<"SMEMOffset">> {
1616
let OperandType = "OPERAND_IMMEDIATE";
1717
let EncoderMethod = "getSMEMOffsetEncoding";
1818
let DecoderMethod = "decodeSMEMOffset";
1919
}
2020

21+
def smem_offset : SMEMOffset;
22+
23+
def smem_offset_mod : SMEMOffset {
24+
let PrintMethod = "printSMEMOffsetMod";
25+
}
26+
2127
//===----------------------------------------------------------------------===//
2228
// Scalar Memory classes
2329
//===----------------------------------------------------------------------===//
@@ -43,8 +49,8 @@ class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> patt
4349
bits<1> has_sdst = 1;
4450
bit has_glc = 0;
4551
bit has_dlc = 0;
46-
bits<1> has_offset = 1;
47-
bits<1> offset_is_imm = 0;
52+
bit has_offset = 0;
53+
bit has_soffset = 0;
4854
bit is_buffer = 0;
4955
}
5056

@@ -77,19 +83,21 @@ class SM_Real <SM_Pseudo ps>
7783
bits<7> sbase;
7884
bits<7> sdst;
7985
bits<32> offset;
80-
bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0);
86+
bits<8> soffset;
8187
bits<5> cpol;
8288
}
8389

8490
class SM_Probe_Pseudo <string opName, dag ins, bit isImm>
85-
: SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> {
91+
: SM_Pseudo<opName, (outs), ins,
92+
" $sdata, $sbase, " # !if(isImm, "$offset", "$soffset")> {
8693
let mayLoad = 0;
8794
let mayStore = 0;
8895
let has_glc = 0;
8996
let LGKM_CNT = 0;
9097
let ScalarStore = 0;
9198
let hasSideEffects = 1;
92-
let offset_is_imm = isImm;
99+
let has_offset = isImm;
100+
let has_soffset = !not(isImm);
93101
let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR");
94102
}
95103

@@ -114,14 +122,16 @@ class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern
114122
}
115123

116124
class SM_Discard_Pseudo <string opName, dag ins, bit isImm>
117-
: SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> {
125+
: SM_Pseudo<opName, (outs), ins,
126+
" $sbase, " # !if(isImm, "$offset", "$soffset")> {
118127
let mayLoad = 0;
119128
let mayStore = 0;
120129
let has_glc = 0;
121130
let has_sdst = 0;
122131
let ScalarStore = 0;
123132
let hasSideEffects = 1;
124-
let offset_is_imm = isImm;
133+
let has_offset = isImm;
134+
let has_soffset = !not(isImm);
125135
let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR");
126136
}
127137

@@ -132,7 +142,7 @@ multiclass SM_Pseudo_Loads<string opName,
132142
(outs dstClass:$sdst),
133143
(ins baseClass:$sbase, i32imm:$offset, CPol:$cpol),
134144
" $sdst, $sbase, $offset$cpol", []> {
135-
let offset_is_imm = 1;
145+
let has_offset = 1;
136146
let BaseClass = baseClass;
137147
let PseudoInstr = opName # "_IMM";
138148
let has_glc = 1;
@@ -141,13 +151,27 @@ multiclass SM_Pseudo_Loads<string opName,
141151

142152
def _SGPR : SM_Load_Pseudo <opName,
143153
(outs dstClass:$sdst),
144-
(ins baseClass:$sbase, SReg_32:$soff, CPol:$cpol),
145-
" $sdst, $sbase, $offset$cpol", []> {
154+
(ins baseClass:$sbase, SReg_32:$soffset, CPol:$cpol),
155+
" $sdst, $sbase, $soffset$cpol", []> {
156+
let has_soffset = 1;
146157
let BaseClass = baseClass;
147158
let PseudoInstr = opName # "_SGPR";
148159
let has_glc = 1;
149160
let has_dlc = 1;
150161
}
162+
163+
def _SGPR_IMM : SM_Load_Pseudo <opName,
164+
(outs dstClass:$sdst),
165+
(ins baseClass:$sbase, SReg_32:$soffset,
166+
i32imm:$offset, CPol:$cpol),
167+
" $sdst, $sbase, $soffset$offset$cpol", []> {
168+
let has_offset = 1;
169+
let has_soffset = 1;
170+
let BaseClass = baseClass;
171+
let PseudoInstr = opName # "_SGPR_IMM";
172+
let has_glc = 1;
173+
let has_dlc = 1;
174+
}
151175
}
152176

153177
multiclass SM_Pseudo_Stores<string opName,
@@ -156,15 +180,16 @@ multiclass SM_Pseudo_Stores<string opName,
156180
def _IMM : SM_Store_Pseudo <opName,
157181
(ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol),
158182
" $sdata, $sbase, $offset$cpol", []> {
159-
let offset_is_imm = 1;
183+
let has_offset = 1;
160184
let BaseClass = baseClass;
161185
let SrcClass = srcClass;
162186
let PseudoInstr = opName # "_IMM";
163187
}
164188

165189
def _SGPR : SM_Store_Pseudo <opName,
166-
(ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, CPol:$cpol),
167-
" $sdata, $sbase, $offset$cpol", []> {
190+
(ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPol:$cpol),
191+
" $sdata, $sbase, $soffset$cpol", []> {
192+
let has_soffset = 1;
168193
let BaseClass = baseClass;
169194
let SrcClass = srcClass;
170195
let PseudoInstr = opName # "_SGPR";
@@ -173,7 +198,7 @@ multiclass SM_Pseudo_Stores<string opName,
173198

174199
multiclass SM_Pseudo_Discards<string opName> {
175200
def _IMM : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smem_offset:$offset), 1>;
176-
def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>;
201+
def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$soffset), 0>;
177202
}
178203

179204
class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
@@ -184,7 +209,6 @@ class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pse
184209
let mayStore = 0;
185210
let mayLoad = 0;
186211
let has_sbase = 0;
187-
let has_offset = 0;
188212
}
189213

190214
class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
@@ -193,12 +217,11 @@ class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_P
193217
let mayStore = 0;
194218
let has_sdst = 0;
195219
let has_sbase = 0;
196-
let has_offset = 0;
197220
}
198221

199222
multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> {
200223
def _IMM : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smem_offset:$offset), 1>;
201-
def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>;
224+
def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$soffset), 0>;
202225
}
203226

204227
class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
@@ -208,7 +231,6 @@ class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
208231
let mayStore = 0;
209232
let mayLoad = 1;
210233
let has_sbase = 0;
211-
let has_offset = 0;
212234
}
213235

214236
//===----------------------------------------------------------------------===//
@@ -225,6 +247,7 @@ class SM_Atomic_Pseudo <string opName,
225247
let mayStore = 1;
226248
let has_glc = 1;
227249
let has_dlc = 1;
250+
let has_soffset = 1;
228251

229252
// Should these be set?
230253
let ScalarStore = 1;
@@ -250,11 +273,13 @@ class SM_Pseudo_Atomic<string opName,
250273
!if(isRet, (outs dataClass:$sdst), (outs)),
251274
!if(isImm,
252275
(ins dataClass:$sdata, baseClass:$sbase, smem_offset:$offset, CPolTy:$cpol),
253-
(ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset, CPolTy:$cpol)),
254-
!if(isRet, " $sdst", " $sdata") # ", $sbase, $offset$cpol",
276+
(ins dataClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPolTy:$cpol)),
277+
!if(isRet, " $sdst", " $sdata") # ", $sbase, " #
278+
!if(isImm, "$offset", "$soffset") # "$cpol",
255279
isRet>,
256280
AtomicNoRet <opNameWithSuffix, isRet> {
257-
let offset_is_imm = isImm;
281+
let has_offset = isImm;
282+
let has_soffset = !not(isImm);
258283
let PseudoInstr = opNameWithSuffix;
259284

260285
let Constraints = !if(isRet, "$sdst = $sdata", "");
@@ -452,8 +477,8 @@ class SMRD_Real_si <bits<5> op, SM_Pseudo ps>
452477
let AssemblerPredicate = isGFX6GFX7;
453478
let DecoderNamespace = "GFX6GFX7";
454479

455-
let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?);
456-
let Inst{8} = imm;
480+
let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
481+
let Inst{8} = ps.has_offset;
457482
let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);
458483
let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
459484
let Inst{26-22} = op;
@@ -470,10 +495,8 @@ multiclass SM_Real_Loads_si<bits<5> op, string ps,
470495
let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol);
471496
}
472497

473-
// FIXME: The operand name $offset is inconsistent with $soff used
474-
// in the pseudo
475498
def _SGPR_si : SMRD_Real_si <op, sgprPs> {
476-
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol);
499+
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
477500
}
478501

479502
}
@@ -508,14 +531,14 @@ class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
508531
let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?);
509532

510533
let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
511-
let Inst{17} = imm;
534+
let Inst{17} = ps.has_offset;
512535
let Inst{25-18} = op;
513536
let Inst{31-26} = 0x30; //encoding
514537

515538
// VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed.
516539
// Offset value is corrected accordingly when offset is encoded/decoded.
517-
let Inst{38-32} = !if(ps.has_offset, offset{6-0}, ?);
518-
let Inst{52-39} = !if(ps.has_offset, !if(imm, offset{20-7}, ?), ?);
540+
let Inst{38-32} = !if(ps.has_offset, offset{6-0}, !if(ps.has_soffset, soffset{6-0}, ?));
541+
let Inst{52-39} = !if(ps.has_offset, offset{20-7}, ?);
519542
}
520543

521544
multiclass SM_Real_Loads_vi<bits<8> op, string ps,
@@ -525,7 +548,7 @@ multiclass SM_Real_Loads_vi<bits<8> op, string ps,
525548
let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
526549
}
527550
def _SGPR_vi : SMEM_Real_vi <op, sgprPs> {
528-
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol);
551+
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
529552
}
530553
}
531554

@@ -540,14 +563,12 @@ class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
540563
multiclass SM_Real_Stores_vi<bits<8> op, string ps,
541564
SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
542565
SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
543-
// FIXME: The operand name $offset is inconsistent with $soff used
544-
// in the pseudo
545566
def _IMM_vi : SMEM_Real_Store_vi <op, immPs> {
546567
let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
547568
}
548569

549570
def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> {
550-
let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol);
571+
let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
551572
}
552573
}
553574

@@ -727,8 +748,8 @@ class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>
727748
let AssemblerPredicate = isGFX7Only;
728749
let DecoderNamespace = "GFX7";
729750

730-
let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?);
731-
let Inst{8} = imm;
751+
let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
752+
let Inst{8} = ps.has_offset;
732753
let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?);
733754
let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
734755
let Inst{26-22} = op;
@@ -887,9 +908,12 @@ class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> :
887908
let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
888909
let Inst{25-18} = op;
889910
let Inst{31-26} = 0x3d;
890-
let Inst{52-32} = !if(ps.offset_is_imm, !if(ps.has_offset, offset{20-0}, ?), ?);
891-
let Inst{63-57} = !if(ps.offset_is_imm, !cast<int>(SGPR_NULL.HWEncoding),
892-
!if(ps.has_offset, offset{6-0}, ?));
911+
912+
// There are SMEM instructions that do not employ any of the offset
913+
// fields, in which case we need them to remain undefined.
914+
let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?));
915+
let Inst{63-57} = !if(ps.has_soffset, soffset{6-0},
916+
!if(ps.has_offset, !cast<int>(SGPR_NULL.HWEncoding), ?));
893917
}
894918

895919
multiclass SM_Real_Loads_gfx10<bits<8> op, string ps,
@@ -899,7 +923,11 @@ multiclass SM_Real_Loads_gfx10<bits<8> op, string ps,
899923
let InOperandList = (ins immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
900924
}
901925
def _SGPR_gfx10 : SMEM_Real_gfx10<op, sgprPs> {
902-
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol);
926+
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
927+
}
928+
def _SGPR_IMM_gfx10 : SMEM_Real_gfx10<op, !cast<SM_Load_Pseudo>(ps#_SGPR_IMM)> {
929+
let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset,
930+
smem_offset_mod:$offset, CPol:$cpol);
903931
}
904932
}
905933

@@ -913,14 +941,12 @@ class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps>
913941
multiclass SM_Real_Stores_gfx10<bits<8> op, string ps,
914942
SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
915943
SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
916-
// FIXME: The operand name $offset is inconsistent with $soff used
917-
// in the pseudo
918944
def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> {
919945
let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
920946
}
921947

922948
def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> {
923-
let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, CPol:$cpol);
949+
let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
924950
}
925951
}
926952

0 commit comments

Comments
 (0)