diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h index 67e10c3589495..45599c940659e 100644 --- a/llvm/include/llvm/MC/MCFragment.h +++ b/llvm/include/llvm/MC/MCFragment.h @@ -23,12 +23,15 @@ namespace llvm { +class MCAssembler; class MCSection; class MCSubtargetInfo; class MCSymbol; -class MCFragment : public ilist_node_with_parent { +class MCFragment { friend class MCAsmLayout; + friend class MCAssembler; + friend class MCSection; public: enum FragmentType : uint8_t { @@ -51,6 +54,9 @@ class MCFragment : public ilist_node_with_parent { }; private: + // The next fragment within the section. + MCFragment *Next = nullptr; + /// The data for the section this fragment is in. MCSection *Parent; @@ -64,10 +70,6 @@ class MCFragment : public ilist_node_with_parent { /// The layout order of this fragment. unsigned LayoutOrder; - /// The subsection this fragment belongs to. This is 0 if the fragment is not - // in any subsection. - unsigned SubsectionNumber = 0; - FragmentType Kind; protected: @@ -88,6 +90,8 @@ class MCFragment : public ilist_node_with_parent { /// This method will dispatch to the appropriate subclass. void destroy(); + MCFragment *getNext() const { return Next; } + FragmentType getKind() const { return Kind; } MCSection *getParent() const { return Parent; } @@ -104,9 +108,6 @@ class MCFragment : public ilist_node_with_parent { bool hasInstructions() const { return HasInstructions; } void dump() const; - - void setSubsectionNumber(unsigned Value) { SubsectionNumber = Value; } - unsigned getSubsectionNumber() const { return SubsectionNumber; } }; class MCDummyFragment : public MCFragment { diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index e212d54613980..c0a337f5ea45e 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -41,7 +41,6 @@ class raw_pwrite_stream; /// implementation. class MCObjectStreamer : public MCStreamer { std::unique_ptr Assembler; - MCSection::iterator CurInsertionPoint; bool EmitEHFrame; bool EmitDebugFrame; SmallVector PendingLabels; @@ -94,7 +93,7 @@ class MCObjectStreamer : public MCStreamer { void insert(MCFragment *F) { flushPendingLabels(F); MCSection *CurSection = getCurrentSectionOnly(); - CurSection->getFragmentList().insert(CurInsertionPoint, F); + CurSection->addFragment(*F); F->setParent(CurSection); } diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 217b9b4b5bc52..e5455292d5c62 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -14,7 +14,6 @@ #define LLVM_MC_MCSECTION_H #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/ilist.h" #include "llvm/MC/MCFragment.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Alignment.h" @@ -24,20 +23,18 @@ namespace llvm { class MCAsmInfo; +class MCAssembler; class MCContext; class MCExpr; class MCSymbol; class raw_ostream; class Triple; -template <> struct ilist_alloc_traits { - static void deleteNode(MCFragment *V); -}; - /// Instances of this class represent a uniqued identifier for a section in the /// current translation unit. The MCContext class uniques and creates these. class MCSection { public: + friend MCAssembler; static constexpr unsigned NonUniqueID = ~0U; enum SectionVariant { @@ -58,12 +55,29 @@ class MCSection { BundleLockedAlignToEnd }; - using FragmentListType = iplist; + struct iterator { + MCFragment *F = nullptr; + iterator() = default; + explicit iterator(MCFragment *F) : F(F) {} + MCFragment &operator*() const { return *F; } + bool operator==(const iterator &O) const { return F == O.F; } + bool operator!=(const iterator &O) const { return F != O.F; } + iterator &operator++() { + F = F->Next; + return *this; + } + iterator operator++(int) { return iterator(F->Next); } + }; - using const_iterator = FragmentListType::const_iterator; - using iterator = FragmentListType::iterator; + struct FragList { + MCFragment *Head = nullptr; + MCFragment *Tail = nullptr; + }; private: + // At parse time, this holds the fragment list of the current subsection. At + // layout time, this holds the concatenated fragment lists of all subsections. + FragList *CurFragList; MCSymbol *Begin; MCSymbol *End = nullptr; /// The alignment requirement of this section. @@ -92,11 +106,10 @@ class MCSection { MCDummyFragment DummyFragment; - FragmentListType Fragments; - - /// Mapping from subsection number to insertion point for subsection numbers - /// below that number. - SmallVector, 1> SubsectionFragmentMap; + // Mapping from subsection number to fragment list. At layout time, the + // subsection 0 list is replaced with concatenated fragments from all + // subsections. + SmallVector, 1> Subsections; /// State for tracking labels that don't yet have Fragments struct PendingLabel { @@ -171,29 +184,27 @@ class MCSection { bool isRegistered() const { return IsRegistered; } void setIsRegistered(bool Value) { IsRegistered = Value; } - MCSection::FragmentListType &getFragmentList() { return Fragments; } - const MCSection::FragmentListType &getFragmentList() const { - return const_cast(this)->getFragmentList(); - } - - /// Support for MCFragment::getNextNode(). - static FragmentListType MCSection::*getSublistAccess(MCFragment *) { - return &MCSection::Fragments; - } - const MCDummyFragment &getDummyFragment() const { return DummyFragment; } MCDummyFragment &getDummyFragment() { return DummyFragment; } - iterator begin() { return Fragments.begin(); } - const_iterator begin() const { return Fragments.begin(); } - - iterator end() { return Fragments.end(); } - const_iterator end() const { return Fragments.end(); } - bool empty() const { return Fragments.empty(); } - - void addFragment(MCFragment &F) { Fragments.push_back(&F); } + FragList *curFragList() const { return CurFragList; } + iterator begin() const { return iterator(CurFragList->Head); } + iterator end() const { return {}; } + bool empty() const { return !CurFragList->Head; } + + void addFragment(MCFragment &F) { + // The formal layout order will be finalized in MCAssembler::layout. + if (CurFragList->Tail) { + CurFragList->Tail->Next = &F; + F.setLayoutOrder(CurFragList->Tail->getLayoutOrder() + 1); + } else { + CurFragList->Head = &F; + assert(F.getLayoutOrder() == 0); + } + CurFragList->Tail = &F; + } - MCSection::iterator getSubsectionInsertionPoint(unsigned Subsection); + void switchSubsection(unsigned Subsection); void dump() const; diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp index 8490853eda87c..4ff606d373238 100644 --- a/llvm/lib/MC/MCAssembler.cpp +++ b/llvm/lib/MC/MCAssembler.cpp @@ -831,6 +831,19 @@ void MCAssembler::layout(MCAsmLayout &Layout) { MCSection *Sec = Layout.getSectionOrder()[i]; Sec->setLayoutOrder(i); + // Chain together fragments from all subsections. + MCDummyFragment Dummy(Sec); + MCFragment *Tail = &Dummy; + for (auto &[_, List] : Sec->Subsections) { + if (!List.Head) + continue; + Tail->Next = List.Head; + Tail = List.Tail; + } + Sec->Subsections.clear(); + Sec->Subsections.push_back({0u, {Dummy.getNext(), Tail}}); + Sec->CurFragList = &Sec->Subsections[0].second; + unsigned FragmentIndex = 0; for (MCFragment &Frag : *Sec) Frag.setLayoutOrder(FragmentIndex++); @@ -1094,7 +1107,7 @@ bool MCAssembler::relaxBoundaryAlign(MCAsmLayout &Layout, uint64_t AlignedOffset = Layout.getFragmentOffset(&BF); uint64_t AlignedSize = 0; - for (const MCFragment *F = BF.getNextNode();; F = F->getNextNode()) { + for (const MCFragment *F = BF.getNext();; F = F->getNext()) { AlignedSize += computeFragmentSize(Layout, *F); if (F == BF.getLastFragment()) break; diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp index b70ac86c18ccf..2eecdb82d30bb 100644 --- a/llvm/lib/MC/MCExpr.cpp +++ b/llvm/lib/MC/MCExpr.cpp @@ -661,25 +661,16 @@ static void AttemptToFoldSymbolOffsetDifference( // this is important when the Subtarget is changed and a new MCDataFragment // is created in the case of foo: instr; .arch_extension ext; instr .if . - // foo. - if (SA.isVariable() || SB.isVariable() || - FA->getSubsectionNumber() != FB->getSubsectionNumber()) + if (SA.isVariable() || SB.isVariable()) return; // Try to find a constant displacement from FA to FB, add the displacement // between the offset in FA of SA and the offset in FB of SB. bool Reverse = false; - if (FA == FB) { + if (FA == FB) Reverse = SA.getOffset() < SB.getOffset(); - } else if (!isa(FA)) { - // Testing FA < FB is slow. Use setLayoutOrder to speed up computation. - // The formal layout order will be finalized in MCAssembler::layout. - if (FA->getLayoutOrder() == 0 || FB->getLayoutOrder()== 0) { - unsigned LayoutOrder = 0; - for (MCFragment &F : *FA->getParent()) - F.setLayoutOrder(++LayoutOrder); - } + else if (!isa(FA)) Reverse = FA->getLayoutOrder() < FB->getLayoutOrder(); - } uint64_t SAOffset = SA.getOffset(), SBOffset = SB.getOffset(); int64_t Displacement = SA.getOffset() - SB.getOffset(); @@ -695,7 +686,7 @@ static void AttemptToFoldSymbolOffsetDifference( // instruction, the difference cannot be resolved as it may be changed by // the linker. bool BBeforeRelax = false, AAfterRelax = false; - for (auto FI = FB->getIterator(), FE = SecA.end(); FI != FE; ++FI) { + for (auto FI = FB; FI; FI = FI->getNext()) { auto DF = dyn_cast(FI); if (DF && DF->isLinkerRelaxable()) { if (&*FI != FB || SBOffset != DF->getContents().size()) @@ -726,12 +717,14 @@ static void AttemptToFoldSymbolOffsetDifference( return; } } - // If the previous loop does not find FA, FA must be a dummy fragment not in - // the fragment list (which means SA is a pending label (see - // flushPendingLabels)). In either case, we can resolve the difference. - assert(Found || isa(FA)); - Addend += Reverse ? -Displacement : Displacement; - FinalizeFolding(); + // If FA and FB belong to the same subsection, either the previous loop + // found FA, or FA is a dummy fragment not in the fragment list (which means + // SA is a pending label (see flushPendingLabels)) or FA and FB belong to + // different subsections. In either case, we can resolve the difference. + if (Found || isa(FA)) { + Addend += Reverse ? -Displacement : Displacement; + FinalizeFolding(); + } } } diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp index 84a587164c788..6d97e8ce552ba 100644 --- a/llvm/lib/MC/MCFragment.cpp +++ b/llvm/lib/MC/MCFragment.cpp @@ -141,7 +141,7 @@ const MCSymbol *MCAsmLayout::getBaseSymbol(const MCSymbol &Symbol) const { uint64_t MCAsmLayout::getSectionAddressSize(const MCSection *Sec) const { // The size is the last fragment's end offset. - const MCFragment &F = Sec->getFragmentList().back(); + const MCFragment &F = *Sec->curFragList()->Tail; return getFragmentOffset(&F) + getAssembler().computeFragmentSize(*this, F); } @@ -197,8 +197,6 @@ uint64_t llvm::computeBundlePadding(const MCAssembler &Assembler, /* *** */ -void ilist_alloc_traits::deleteNode(MCFragment *V) { V->destroy(); } - MCFragment::MCFragment(FragmentType Kind, bool HasInstructions, MCSection *Parent) : Parent(Parent), Atom(nullptr), Offset(~UINT64_C(0)), LayoutOrder(0), diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index ae4e6915fa294..bf1ce76cdc14b 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -180,7 +180,6 @@ void MCObjectStreamer::reset() { if (getContext().getTargetOptions()) Assembler->setRelaxAll(getContext().getTargetOptions()->MCRelaxAll); } - CurInsertionPoint = MCSection::iterator(); EmitEHFrame = true; EmitDebugFrame = false; PendingLabels.clear(); @@ -200,12 +199,7 @@ void MCObjectStreamer::emitFrames(MCAsmBackend *MAB) { } MCFragment *MCObjectStreamer::getCurrentFragment() const { - assert(getCurrentSectionOnly() && "No current section!"); - - if (CurInsertionPoint != getCurrentSectionOnly()->begin()) - return &*std::prev(CurInsertionPoint); - - return nullptr; + return getCurrentSectionOnly()->curFragList()->Tail; } static bool canReuseDataFragment(const MCDataFragment &F, @@ -391,8 +385,7 @@ bool MCObjectStreamer::changeSectionImpl(MCSection *Section, } CurSubsectionIdx = unsigned(IntSubsection); - CurInsertionPoint = - Section->getSubsectionInsertionPoint(CurSubsectionIdx); + Section->switchSubsection(CurSubsectionIdx); return Created; } diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp index 9848d7fafe764..1d9fe2cafd617 100644 --- a/llvm/lib/MC/MCSection.cpp +++ b/llvm/lib/MC/MCSection.cpp @@ -24,7 +24,10 @@ MCSection::MCSection(SectionVariant V, StringRef Name, SectionKind K, MCSymbol *Begin) : Begin(Begin), BundleGroupBeforeFirstInst(false), HasInstructions(false), HasLayout(false), IsRegistered(false), DummyFragment(this), Name(Name), - Variant(V), Kind(K) {} + Variant(V), Kind(K) { + // The initial subsection number is 0. Create a fragment list. + CurFragList = &Subsections.emplace_back(0u, FragList{}).second; +} MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) { if (!End) @@ -34,7 +37,14 @@ MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) { bool MCSection::hasEnded() const { return End && End->isInSection(); } -MCSection::~MCSection() = default; +MCSection::~MCSection() { + for (auto &[_, Chain] : Subsections) { + for (MCFragment *X = Chain.Head, *Y; X; X = Y) { + Y = X->Next; + X->destroy(); + } + } +} void MCSection::setBundleLockState(BundleLockStateType NewState) { if (NewState == NotBundleLocked) { @@ -55,35 +65,15 @@ void MCSection::setBundleLockState(BundleLockStateType NewState) { ++BundleLockNestingDepth; } -MCSection::iterator -MCSection::getSubsectionInsertionPoint(unsigned Subsection) { - if (Subsection == 0 && SubsectionFragmentMap.empty()) - return end(); - - SmallVectorImpl>::iterator MI = lower_bound( - SubsectionFragmentMap, std::make_pair(Subsection, (MCFragment *)nullptr)); - bool ExactMatch = false; - if (MI != SubsectionFragmentMap.end()) { - ExactMatch = MI->first == Subsection; - if (ExactMatch) - ++MI; - } - iterator IP; - if (MI == SubsectionFragmentMap.end()) - IP = end(); - else - IP = MI->second->getIterator(); - if (!ExactMatch && Subsection != 0) { - // The GNU as documentation claims that subsections have an alignment of 4, - // although this appears not to be the case. - MCFragment *F = new MCDataFragment(); - SubsectionFragmentMap.insert(MI, std::make_pair(Subsection, F)); - getFragmentList().insert(IP, F); - F->setParent(this); - F->setSubsectionNumber(Subsection); - } - - return IP; +void MCSection::switchSubsection(unsigned Subsection) { + size_t I = 0, E = Subsections.size(); + while (I != E && Subsections[I].first < Subsection) + ++I; + // If the subsection number is not in the sorted Subsections list, create a + // new fragment list. + if (I == E || Subsections[I].first != Subsection) + Subsections.insert(Subsections.begin() + I, {Subsection, FragList{}}); + CurFragList = &Subsections[I].second; } StringRef MCSection::getVirtualSectionKind() const { return "virtual"; } @@ -111,13 +101,11 @@ void MCSection::flushPendingLabels() { // creating new empty data fragments for each Subsection with labels pending. while (!PendingLabels.empty()) { PendingLabel& Label = PendingLabels[0]; - iterator CurInsertionPoint = - this->getSubsectionInsertionPoint(Label.Subsection); - const MCSymbol *Atom = nullptr; - if (CurInsertionPoint != begin()) - Atom = std::prev(CurInsertionPoint)->getAtom(); + switchSubsection(Label.Subsection); + const MCSymbol *Atom = + CurFragList->Tail ? CurFragList->Tail->getAtom() : nullptr; MCFragment *F = new MCDataFragment(); - getFragmentList().insert(CurInsertionPoint, F); + addFragment(*F); F->setParent(this); F->setAtom(Atom); flushPendingLabels(F, 0, Label.Subsection); diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp index 451269608f179..522e268156aa3 100644 --- a/llvm/lib/MC/WasmObjectWriter.cpp +++ b/llvm/lib/MC/WasmObjectWriter.cpp @@ -1864,15 +1864,14 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm, if (EmptyFrag.getKind() != MCFragment::FT_Data) report_fatal_error(".init_array section should be aligned"); - IT = std::next(IT); - const MCFragment &AlignFrag = *IT; + const MCFragment &AlignFrag = *EmptyFrag.getNext(); if (AlignFrag.getKind() != MCFragment::FT_Align) report_fatal_error(".init_array section should be aligned"); if (cast(AlignFrag).getAlignment() != Align(is64Bit() ? 8 : 4)) report_fatal_error(".init_array section should be aligned for pointers"); - const MCFragment &Frag = *std::next(IT); + const MCFragment &Frag = *AlignFrag.getNext(); if (Frag.hasInstructions() || Frag.getKind() != MCFragment::FT_Data) report_fatal_error("only data supported in .init_array section"); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp index 3b6ea81cdf10e..54efe4bc25efe 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp @@ -712,17 +712,20 @@ class HexagonAsmBackend : public MCAsmBackend { void finishLayout(MCAssembler const &Asm, MCAsmLayout &Layout) const override { + SmallVector Frags; for (auto *I : Layout.getSectionOrder()) { - for (auto &J : *I) { - switch (J.getKind()) { + Frags.clear(); + for (MCFragment &F : *I) + Frags.push_back(&F); + for (size_t J = 0, E = Frags.size(); J != E; ++J) { + switch (Frags[J]->getKind()) { default: break; case MCFragment::FT_Align: { - auto Size = Asm.computeFragmentSize(Layout, J); - for (auto K = J.getIterator(); - K != I->begin() && Size >= HEXAGON_PACKET_SIZE;) { + auto Size = Asm.computeFragmentSize(Layout, *Frags[J]); + for (auto K = J; K != 0 && Size >= HEXAGON_PACKET_SIZE;) { --K; - switch (K->getKind()) { + switch (Frags[K]->getKind()) { default: break; case MCFragment::FT_Align: { @@ -732,7 +735,7 @@ class HexagonAsmBackend : public MCAsmBackend { } case MCFragment::FT_Relaxable: { MCContext &Context = Asm.getContext(); - auto &RF = cast(*K); + auto &RF = cast(*Frags[K]); auto &Inst = const_cast(RF.getInst()); while (Size > 0 && HexagonMCInstrInfo::bundleSize(Inst) < MaxPacketSize) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp index b8e0f3a867f40..d83dadd301619 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp @@ -62,7 +62,7 @@ const MCFixup *RISCVMCExpr::getPCRelHiFixup(const MCFragment **DFOut) const { uint64_t Offset = AUIPCSymbol->getOffset(); if (DF->getContents().size() == Offset) { - DF = dyn_cast_or_null(DF->getNextNode()); + DF = dyn_cast_or_null(DF->getNext()); if (!DF) return nullptr; Offset = 0; diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index bc2eb6dcd541c..1b8462f2d258c 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -530,7 +530,7 @@ void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS, if (!canPadInst(Inst, OS)) return; - if (PendingBA && PendingBA->getNextNode() == OS.getCurrentFragment()) { + if (PendingBA && PendingBA->getNext() == OS.getCurrentFragment()) { // Macro fusion actually happens and there is no other fragment inserted // after the previous instruction. // @@ -978,8 +978,8 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm, // The layout is done. Mark every fragment as valid. for (unsigned int i = 0, n = Layout.getSectionOrder().size(); i != n; ++i) { MCSection &Section = *Layout.getSectionOrder()[i]; - Layout.getFragmentOffset(&*Section.getFragmentList().rbegin()); - Asm.computeFragmentSize(Layout, *Section.getFragmentList().rbegin()); + Layout.getFragmentOffset(&*Section.curFragList()->Tail); + Asm.computeFragmentSize(Layout, *Section.curFragList()->Tail); } }