AMDGPU/GlobalISel: uniformity analysis based register bank selection

Current algorithm only considers register banks for inputs but does
take control flow into account.
This is wrong in cases where inputs are uniform (in sgpr) but because
of divergent control flow instruction is divergent and should use vgpr
instead of sgpr register banks. Most notable example are phis.
Also in cases where only available machine instruction uses vgpr
registers uniform instructions end up using vgpr register banks.
Start with simple implementation for G_FADD.
Pre-select register bank for destination register using machine
uniformity analysis info. Then select register banks that would allow
selection of available machine instructions.
For G_FADD vgpr machine instruction is available on all targets but
sgpr version is not. When there is no sgpr version assign vgpr register
banks and move vgpr destination to sgpr using readfirstlane.

Author: petar-avramovic

llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp

@@ -15,7 +15,10 @@
 #include "AMDGPURegBankSelect.h"
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/MachineUniformityAnalysis.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/InitializePasses.h"
 
 #define DEBUG_TYPE "regbankselect"
@@ -68,7 +71,26 @@ bool AMDGPURegBankSelect::runOnMachineFunction(MachineFunction &MF) {
   MachineUniformityInfo Uniformity =
       computeMachineUniformityInfo(MF, CycleInfo, DomTree.getBase(),
                                    !ST.isSingleLaneExecution(F));
-  (void)Uniformity; // TODO: Use this
+
+  // Switch for uniformity info based regbank selection. Pre-selects register
+  // bank on dst registers using machine uniformity analysis.
+  // Keep in sinc with switches in getInstrMapping and applyMappingImpl.
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      switch (MI.getOpcode()) {
+      case AMDGPU::G_FADD: {
+        Register Dst = MI.getOperand(0).getReg();
+        if (Uniformity.isUniform(Dst))
+          MRI->setRegBank(Dst, RBI->getRegBank(AMDGPU::SGPRRegBankID));
+        else
+          MRI->setRegBank(Dst, RBI->getRegBank(AMDGPU::VGPRRegBankID));
+        break;
+      }
+      default:
+        break;
+      }
+    }
+  }
 
   assignRegisterBanks(MF);
 

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

@@ -718,6 +718,26 @@ Register AMDGPURegisterBankInfo::buildReadFirstLaneSrc(MachineIRBuilder &B,
   return Dst;
 }
 
+// Create new vgpr destination register for MI then move it to current
+// MI's sgpr destination using one or more V_READFIRSTLANE_B32 instructions.
+void AMDGPURegisterBankInfo::buildReadFirstLaneDst(MachineIRBuilder &B,
+                                                   MachineInstr &MI) const {
+  MachineRegisterInfo &MRI = *B.getMRI();
+  Register Dst = MI.getOperand(0).getReg();
+  const RegisterBank *DstBank = getRegBank(Dst, MRI, *TRI);
+  if (DstBank != &AMDGPU::SGPRRegBank)
+    return;
+
+  Register VgprDst = MRI.createGenericVirtualRegister(MRI.getType(Dst));
+  MRI.setRegBank(VgprDst, AMDGPU::VGPRRegBank);
+
+  MI.getOperand(0).setReg(VgprDst);
+  MachineBasicBlock *MBB = MI.getParent();
+  B.setInsertPt(*MBB, std::next(MI.getIterator()));
+  // readFirstLane VgprDst into Dst after MI.
+  return buildReadFirstLaneForType(B, Dst, VgprDst);
+}
+
 void AMDGPURegisterBankInfo::buildReadFirstLaneB32(MachineIRBuilder &B,
                                                    Register SgprDst,
                                                    Register VgprSrc) const {
@@ -741,32 +761,42 @@ void AMDGPURegisterBankInfo::buildReadFirstLaneSequenceOfB32(
   }
 
   B.buildUnmerge(VgprSrcParts, VgprSrc);
-  for (unsigned i = 0; i < NumElts; ++i) {
+  for (unsigned i = 0; i < NumElts; ++i)
     buildReadFirstLaneB32(B, SgprDstParts[i], VgprSrcParts[i]);
-  }
+
   B.buildMergeLikeInstr(SgprDst, SgprDstParts);
 }
 
 void AMDGPURegisterBankInfo::buildReadFirstLaneForType(MachineIRBuilder &B,
                                                        Register SgprDst,
                                                        Register VgprSrc) const {
   MachineRegisterInfo &MRI = *B.getMRI();
+  LLT S16 = LLT::scalar(16);
   LLT S32 = LLT::scalar(32);
   LLT S64 = LLT::scalar(64);
   LLT Ty = MRI.getType(SgprDst);
 
-  if (Ty == S32 || Ty == LLT::pointer(3, 32)) {
-    return buildReadFirstLaneB32(B, SgprDst, VgprSrc);
+  if (Ty == S16) {
+    Register VgprSrc32 = MRI.createGenericVirtualRegister(S32);
+    MRI.setRegBank(VgprSrc32, AMDGPU::VGPRRegBank);
+    Register SgprDst32 = MRI.createGenericVirtualRegister(S32);
+    MRI.setRegBank(SgprDst32, AMDGPU::SGPRRegBank);
+
+    B.buildAnyExt(VgprSrc32, VgprSrc);
+    buildReadFirstLaneB32(B, SgprDst32, VgprSrc32);
+    B.buildTrunc(SgprDst, SgprDst32);
+    return;
   }
 
-  if (Ty == S64 || Ty == LLT::pointer(0, 64) || Ty == LLT::pointer(1, 64)) {
+  if (Ty == S32 || Ty == LLT::pointer(3, 32))
+    return buildReadFirstLaneB32(B, SgprDst, VgprSrc);
+
+  if (Ty == S64 || Ty == LLT::pointer(0, 64) || Ty == LLT::pointer(1, 64))
     return buildReadFirstLaneSequenceOfB32(B, SgprDst, VgprSrc, 2);
-  }
 
-  if (Ty.isVector() && Ty.getElementType() == S32) {
+  if (Ty.isVector() && Ty.getElementType() == S32)
     return buildReadFirstLaneSequenceOfB32(B, SgprDst, VgprSrc,
                                            Ty.getNumElements());
-  }
 
   llvm_unreachable("Type not supported");
 }
@@ -1041,6 +1071,17 @@ void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
   MI.getOperand(OpIdx).setReg(Reg);
 }
 
+// MI has uniform inputs and output but only available machine instruction has
+// vgpr dest. Make it uniform by moving dst to sgpr using readfirstlane.
+void AMDGPURegisterBankInfo::constrainVgprDstOpWithReadfirstlane(
+    MachineIRBuilder &B, MachineInstr &MI,
+    const OperandsMapper &OpdMapper) const {
+  const RegisterBank *DstBank =
+      OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank;
+  if (DstBank != &AMDGPU::VGPRRegBank)
+    buildReadFirstLaneDst(B, MI);
+}
+
 /// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
 /// rest will be in the remainder.
 static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
@@ -2197,6 +2238,21 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
   B.setInstrAndDebugLoc(MI);
   unsigned Opc = MI.getOpcode();
   MachineRegisterInfo &MRI = OpdMapper.getMRI();
+
+  // Switch for uniformity info based regbank selection.
+  // Keep in sinc with switches in AMDGPURegBankSelect and getInstrMapping.
+  switch (Opc) {
+  case AMDGPU::G_FADD: {
+    applyDefaultMapping(OpdMapper);
+    unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+    if (!Subtarget.hasSALUFloatInsts() || (Size != 32 && Size != 16))
+      constrainVgprDstOpWithReadfirstlane(B, MI, OpdMapper);
+    return;
+  }
+  default:
+    break;
+  }
+
   switch (Opc) {
   case AMDGPU::G_CONSTANT:
   case AMDGPU::G_IMPLICIT_DEF: {
@@ -3571,6 +3627,28 @@ AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
                                MI.getNumOperands());
 }
 
+const RegisterBankInfo::InstructionMapping &
+AMDGPURegisterBankInfo::getDefaultMappingVOPWithPreassignedDef(
+    const MachineInstr &MI) const {
+  SmallVector<const ValueMapping *, 8> OpdsMapping(MI.getNumOperands());
+  const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+  // Dst reg bank should have been set already by uniformity info
+  OpdsMapping[0] =
+      getPreAssignedOpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+
+  for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i) {
+    const MachineOperand &Op = MI.getOperand(i);
+    if (!Op.isReg())
+      continue;
+
+    unsigned Size = getSizeInBits(Op.getReg(), MRI, *TRI);
+    unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
+    OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
+  }
+  return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
+                               MI.getNumOperands());
+}
+
 const RegisterBankInfo::InstructionMapping &
 AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
   const MachineFunction &MF = *MI.getParent()->getParent();
@@ -3723,6 +3801,20 @@ AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
   return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
 }
 
+const RegisterBankInfo::ValueMapping *
+AMDGPURegisterBankInfo::getPreAssignedOpMapping(
+    Register Reg, const MachineRegisterInfo &MRI,
+    const TargetRegisterInfo &TRI) const {
+  const RegisterBank *Bank = getRegBank(Reg, MRI, TRI);
+  assert(Bank);
+  unsigned BankId = Bank->getID();
+  unsigned Size = getSizeInBits(Reg, MRI, TRI);
+  assert(BankId == AMDGPU::SGPRRegBankID ||
+         BankId == (Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID));
+
+  return AMDGPU::getValueMapping(BankId, Size);
+}
+
 const RegisterBankInfo::ValueMapping *
 AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
                                          const MachineRegisterInfo &MRI,
@@ -3839,6 +3931,24 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
   SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
 
+  // Switch for uniformity info based regbank selection.
+  // Requires pre-selected, by AMDGPURegBankSelect, reg-banks on dst registers.
+  // Keep in sinc with switches in AMDGPURegBankSelect and applyMappingImpl.
+  switch (MI.getOpcode()) {
+  case AMDGPU::G_FADD: {
+    Register Dst = MI.getOperand(0).getReg();
+    unsigned Size = MRI.getType(Dst).getSizeInBits();
+    const RegisterBank *DstBank = getRegBank(Dst, MRI, *TRI);
+    assert(DstBank);
+    if (Subtarget.hasSALUFloatInsts() && (Size == 32 || Size == 16) &&
+        DstBank == &AMDGPU::SGPRRegBank)
+      return getDefaultMappingSOP(MI);
+    return getDefaultMappingVOPWithPreassignedDef(MI);
+  }
+  default:
+    break;
+  }
+
   switch (MI.getOpcode()) {
   default:
     return getInvalidInstructionMapping();
@@ -3936,7 +4046,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     if (isSALUMapping(MI))
       return getDefaultMappingSOP(MI);
     return getDefaultMappingVOP(MI);
-  case AMDGPU::G_FADD:
   case AMDGPU::G_FSUB:
   case AMDGPU::G_FMUL:
   case AMDGPU::G_FMA:

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h

@@ -59,6 +59,8 @@ class AMDGPURegisterBankInfo final : public AMDGPUGenRegisterBankInfo {
 
   Register buildReadFirstLaneSrc(MachineIRBuilder &B, Register Src) const;
 
+  void buildReadFirstLaneDst(MachineIRBuilder &B, MachineInstr &MI) const;
+
   void buildReadFirstLaneForType(MachineIRBuilder &B, Register SgprDst,
                                  Register VgprSrc) const;
 
@@ -74,6 +76,10 @@ class AMDGPURegisterBankInfo final : public AMDGPUGenRegisterBankInfo {
 
   void constrainOpWithReadfirstlane(MachineIRBuilder &B, MachineInstr &MI,
                                     unsigned OpIdx) const;
+  void
+  constrainVgprDstOpWithReadfirstlane(MachineIRBuilder &B, MachineInstr &MI,
+                                      const OperandsMapper &OpdMapper) const;
+
   bool applyMappingDynStackAlloc(MachineIRBuilder &B,
                                  const OperandsMapper &OpdMapper,
                                  MachineInstr &MI) const;

llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-mul.ll

@@ -49,21 +49,31 @@ define amdgpu_vs <5 x float> @test_5xf16_5xf32_add_ext_mul(<5 x half> inreg %x,
 ; GFX9-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
 ; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
 ; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v1, s1, v1
-; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
 ; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, v0
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, v1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, v2
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s6, v3
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v1, s7, v4
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v2, s8, v5
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s9, v6
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v4, s10, v7
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v4, v1
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s7, v0
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s8, v4
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s9, v1
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s6, v3
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s10, v2
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-FAST-DENORM-LABEL: test_5xf16_5xf32_add_ext_mul:
@@ -90,23 +100,35 @@ define amdgpu_vs <6 x float> @test_6xf16_6xf32_add_ext_mul_rhs(<6 x half> inreg
 ; GFX9-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:
 ; GFX9-FAST-DENORM:       ; %bb.0: ; %.entry
 ; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
-; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v0, s0, v0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v1, s1, v1
-; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
 ; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v3, v0
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, v1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v7, v2
-; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s6, v3
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v1, s7, v4
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v2, s8, v5
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s9, v6
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v4, s10, v7
-; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v5, s11, v8
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v4, v1
+; GFX9-FAST-DENORM-NEXT:    v_pk_mul_f16 v2, s2, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_e32 v5, v2
+; GFX9-FAST-DENORM-NEXT:    v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s7, v0
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s8, v4
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s9, v1
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s10, v5
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v3, s6, v3
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-FAST-DENORM-NEXT:    v_add_f32_e32 v0, s11, v2
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX9-FAST-DENORM-NEXT:    v_readfirstlane_b32 s5, v0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-FAST-DENORM-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX9-FAST-DENORM-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-FAST-DENORM-LABEL: test_6xf16_6xf32_add_ext_mul_rhs:

llvm/test/CodeGen/AMDGPU/GlobalISel/floor.f64.ll

@@ -199,6 +199,10 @@ define amdgpu_ps <2 x float> @s_floor_f64(double inreg %x) {
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX6-NEXT:    v_add_f64 v[0:1], s[2:3], -v[0:1]
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX78-LABEL: s_floor_f64:
@@ -223,6 +227,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fneg(double inreg %x) {
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX6-NEXT:    v_add_f64 v[0:1], -s[2:3], -v[0:1]
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX78-LABEL: s_floor_f64_fneg:
@@ -248,6 +256,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fabs(double inreg %x) {
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX6-NEXT:    v_add_f64 v[0:1], |s[2:3]|, -v[0:1]
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX78-LABEL: s_floor_f64_fabs:
@@ -273,6 +285,10 @@ define amdgpu_ps <2 x float> @s_floor_f64_fneg_fabs(double inreg %x) {
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GFX6-NEXT:    v_add_f64 v[0:1], -|s[2:3]|, -v[0:1]
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX78-LABEL: s_floor_f64_fneg_fabs: