diff --git a/llvm/include/llvm/IR/ConstantRangeList.h b/llvm/include/llvm/IR/ConstantRangeList.h index 44d1daebe49e4..b12c913103df5 100644 --- a/llvm/include/llvm/IR/ConstantRangeList.h +++ b/llvm/include/llvm/IR/ConstantRangeList.h @@ -35,7 +35,7 @@ class [[nodiscard]] ConstantRangeList { ConstantRangeList(ArrayRef RangesRef) { assert(isOrderedRanges(RangesRef)); for (const ConstantRange &R : RangesRef) { - assert(R.getBitWidth() == getBitWidth()); + assert(empty() || R.getBitWidth() == getBitWidth()); Ranges.push_back(R); } } @@ -59,8 +59,9 @@ class [[nodiscard]] ConstantRangeList { /// Return true if this list contains no members. bool empty() const { return Ranges.empty(); } - /// Get the bit width of this ConstantRangeList. - uint32_t getBitWidth() const { return 64; } + /// Get the bit width of this ConstantRangeList. It is invalid to call this + /// with an empty range. + uint32_t getBitWidth() const { return Ranges.front().getBitWidth(); } /// Return the number of ranges in this ConstantRangeList. size_t size() const { return Ranges.size(); } diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index 7088276754516..35580f3f38c61 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -1456,6 +1456,7 @@ class MDNode : public Metadata { static MDNode *getMostGenericTBAA(MDNode *A, MDNode *B); static MDNode *getMostGenericFPMath(MDNode *A, MDNode *B); static MDNode *getMostGenericRange(MDNode *A, MDNode *B); + static MDNode *getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B); static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B); static MDNode *getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B); /// Merge !prof metadata from two instructions. diff --git a/llvm/lib/IR/ConstantRangeList.cpp b/llvm/lib/IR/ConstantRangeList.cpp index 0856f79bb9191..3ee8d6f22b748 100644 --- a/llvm/lib/IR/ConstantRangeList.cpp +++ b/llvm/lib/IR/ConstantRangeList.cpp @@ -39,12 +39,14 @@ void ConstantRangeList::insert(const ConstantRange &NewRange) { return; assert(!NewRange.isFullSet() && "Do not support full set"); assert(NewRange.getLower().slt(NewRange.getUpper())); - assert(getBitWidth() == NewRange.getBitWidth()); // Handle common cases. if (empty() || Ranges.back().getUpper().slt(NewRange.getLower())) { Ranges.push_back(NewRange); return; } + + assert(getBitWidth() == NewRange.getBitWidth()); + if (NewRange.getUpper().slt(Ranges.front().getLower())) { Ranges.insert(Ranges.begin(), NewRange); return; @@ -142,14 +144,15 @@ void ConstantRangeList::subtract(const ConstantRange &SubRange) { ConstantRangeList ConstantRangeList::unionWith(const ConstantRangeList &CRL) const { - assert(getBitWidth() == CRL.getBitWidth() && - "ConstantRangeList bitwidths don't agree!"); // Handle common cases. if (empty()) return CRL; if (CRL.empty()) return *this; + assert(getBitWidth() == CRL.getBitWidth() && + "ConstantRangeList bitwidths don't agree!"); + ConstantRangeList Result; size_t i = 0, j = 0; // "PreviousRange" tracks the lowest unioned range that is being processed. @@ -192,15 +195,15 @@ ConstantRangeList::unionWith(const ConstantRangeList &CRL) const { ConstantRangeList ConstantRangeList::intersectWith(const ConstantRangeList &CRL) const { - assert(getBitWidth() == CRL.getBitWidth() && - "ConstantRangeList bitwidths don't agree!"); - // Handle common cases. if (empty()) return *this; if (CRL.empty()) return CRL; + assert(getBitWidth() == CRL.getBitWidth() && + "ConstantRangeList bitwidths don't agree!"); + ConstantRangeList Result; size_t i = 0, j = 0; while (i < size() && j < CRL.size()) { diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index 28f2ca550f5ec..94faad1a52243 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -29,6 +29,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" +#include "llvm/IR/ConstantRangeList.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" @@ -1353,6 +1354,43 @@ MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) { return MDNode::get(A->getContext(), MDs); } +MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) { + if (!A || !B) + return nullptr; + + if (A == B) + return A; + + SmallVector RangeListA, RangeListB; + for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) { + auto *LowA = mdconst::extract(A->getOperand(2 * I + 0)); + auto *HighA = mdconst::extract(A->getOperand(2 * I + 1)); + RangeListA.push_back(ConstantRange(LowA->getValue(), HighA->getValue())); + } + + for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) { + auto *LowB = mdconst::extract(B->getOperand(2 * I + 0)); + auto *HighB = mdconst::extract(B->getOperand(2 * I + 1)); + RangeListB.push_back(ConstantRange(LowB->getValue(), HighB->getValue())); + } + + ConstantRangeList CRLA(RangeListA); + ConstantRangeList CRLB(RangeListB); + ConstantRangeList Result = CRLA.intersectWith(CRLB); + if (Result.empty()) + return nullptr; + + SmallVector MDs; + for (const ConstantRange &CR : Result) { + MDs.push_back(ConstantAsMetadata::get( + ConstantInt::get(A->getContext(), CR.getLower()))); + MDs.push_back(ConstantAsMetadata::get( + ConstantInt::get(A->getContext(), CR.getUpper()))); + } + + return MDNode::get(A->getContext(), MDs); +} + MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { if (!A || !B) return nullptr; diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 47a7049255961..a8f66a320af67 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -3391,6 +3391,11 @@ void llvm::combineMetadata(Instruction *K, const Instruction *J, if (DoesKMove) K->setMetadata(Kind, MDNode::getMergedProfMetadata(KMD, JMD, K, J)); break; + case LLVMContext::MD_noalias_addrspace: + if (DoesKMove) + K->setMetadata(Kind, + MDNode::getMostGenericNoaliasAddrspace(JMD, KMD)); + break; } } // Set !invariant.group from J if J has it. If both instructions have it @@ -3432,7 +3437,8 @@ void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J, LLVMContext::MD_prof, LLVMContext::MD_nontemporal, LLVMContext::MD_noundef, - LLVMContext::MD_mmra}; + LLVMContext::MD_mmra, + LLVMContext::MD_noalias_addrspace}; combineMetadata(K, J, KnownIDs, KDominatesJ); } diff --git a/llvm/test/Transforms/EarlyCSE/noalias-addrspace.ll b/llvm/test/Transforms/EarlyCSE/noalias-addrspace.ll new file mode 100644 index 0000000000000..0a001b55f684c --- /dev/null +++ b/llvm/test/Transforms/EarlyCSE/noalias-addrspace.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes='early-cse' -S < %s | FileCheck %s + +declare void @use(i1) +declare void @use.ptr(ptr) memory(read) + +define void @load_first_noalias_addrspace(ptr %p) { +; CHECK-LABEL: define void @load_first_noalias_addrspace( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = load ptr, ptr [[P]], align 8, !nonnull [[META0:![0-9]+]], !noundef [[META0]], !noalias.addrspace [[META1:![0-9]+]] +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: ret void +; + %v1 = load ptr, ptr %p, !nonnull !{}, !noundef !{}, !noalias.addrspace !0 + call void @use.ptr(ptr %v1) + %v2 = load ptr, ptr %p + call void @use.ptr(ptr %v2) + ret void +} + +define void @load_both_same_noalias_addrspace(ptr %p) { +; CHECK-LABEL: define void @load_both_same_noalias_addrspace( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = load ptr, ptr [[P]], align 8, !nonnull [[META0]], !noundef [[META0]], !noalias.addrspace [[META1]] +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: ret void +; + %v1 = load ptr, ptr %p, !nonnull !{}, !noundef !{}, !noalias.addrspace !0 + call void @use.ptr(ptr %v1) + %v2 = load ptr, ptr %p, !noalias.addrspace !0 + call void @use.ptr(ptr %v2) + ret void +} + +define void @load_both_disjoint_noalias_addrspace(ptr %p) { +; CHECK-LABEL: define void @load_both_disjoint_noalias_addrspace( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = load ptr, ptr [[P]], align 8, !nonnull [[META0]], !noundef [[META0]], !noalias.addrspace [[META1]] +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: ret void +; + %v1 = load ptr, ptr %p, !nonnull !{}, !noundef !{}, !noalias.addrspace !0 + call void @use.ptr(ptr %v1) + %v2 = load ptr, ptr %p, !noalias.addrspace !1 + call void @use.ptr(ptr %v2) + ret void +} + +define void @load_both_overlap_noalias_addrspace(ptr %p) { +; CHECK-LABEL: define void @load_both_overlap_noalias_addrspace( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = load ptr, ptr [[P]], align 8, !nonnull [[META0]], !noundef [[META0]], !noalias.addrspace [[META1]] +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: call void @use.ptr(ptr [[V1]]) +; CHECK-NEXT: ret void +; + %v1 = load ptr, ptr %p, !nonnull !{}, !noundef !{}, !noalias.addrspace !0 + call void @use.ptr(ptr %v1) + %v2 = load ptr, ptr %p, !noalias.addrspace !2 + call void @use.ptr(ptr %v2) + ret void +} + +!0 = !{i32 5, i32 6} +!1 = !{i32 7, i32 8} +!2 = !{i32 5, i32 7} +;. +; CHECK: [[META0]] = !{} +; CHECK: [[META1]] = !{i32 5, i32 6} +;. diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll index 18aa5c9e044a9..d34ac2bb30040 100644 --- a/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll +++ b/llvm/test/Transforms/SimplifyCFG/hoist-with-metadata.ll @@ -319,7 +319,7 @@ out: define void @hoist_noalias_addrspace_both(i1 %c, ptr %p, i64 %val) { ; CHECK-LABEL: @hoist_noalias_addrspace_both( ; CHECK-NEXT: if: -; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8 +; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8, !noalias.addrspace [[META7:![0-9]+]] ; CHECK-NEXT: ret void ; if: @@ -361,7 +361,7 @@ out: define void @hoist_noalias_addrspace_switch(i64 %i, ptr %p, i64 %val) { ; CHECK-LABEL: @hoist_noalias_addrspace_switch( ; CHECK-NEXT: out: -; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8 +; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8, !noalias.addrspace [[META7]] ; CHECK-NEXT: ret void ; switch i64 %i, label %bb0 [ @@ -381,6 +381,48 @@ out: ret void } +define void @hoist_noalias_addrspace_switch_multiple(i64 %i, ptr %p, i64 %val) { +; CHECK-LABEL: @hoist_noalias_addrspace_switch_multiple( +; CHECK-NEXT: out: +; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8, !noalias.addrspace [[META8:![0-9]+]] +; CHECK-NEXT: ret void +; + switch i64 %i, label %bb0 [ + i64 1, label %bb1 + i64 2, label %bb2 + ] +bb0: + %t = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !7 + br label %out +bb1: + %e = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !8 + br label %out +bb2: + %f = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !9 + br label %out +out: + ret void +} + +; !noalias_addrspace is not safe to speculate as it causes immediate undefined behavior. +define ptr @speculate_noalias_addrspace(i1 %c, ptr dereferenceable(8) align 8 %p) { +; CHECK-LABEL: @speculate_noalias_addrspace( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P:%.*]], align 8, !nonnull [[META2]] +; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C:%.*]], ptr [[V]], ptr null +; CHECK-NEXT: ret ptr [[SPEC_SELECT]] +; +entry: + br i1 %c, label %if, label %join + +if: + %v = load ptr, ptr %p, !nonnull !{}, !noundef !{}, !noalias.addrspace !4 + br label %join + +join: + %phi = phi ptr [ %v, %if ], [ null, %entry ] + ret ptr %phi +} !0 = !{ i8 0, i8 1 } !1 = !{ i8 3, i8 5 } @@ -389,6 +431,9 @@ out: !4 = !{i32 5, i32 6} !5 = !{i32 5, i32 7} !6 = !{i32 4, i32 8} +!7 = !{i32 4, i32 8, i32 20, i32 31} +!8 = !{i32 2, i32 5} +!9 = !{i32 2, i32 5, i32 22, i32 42, i32 45, i32 50} ;. ; CHECK: [[RNG0]] = !{i8 0, i8 1, i8 3, i8 5} @@ -398,4 +443,6 @@ out: ; CHECK: [[RNG4]] = !{i32 0, i32 10} ; CHECK: [[META5]] = !{i64 4} ; CHECK: [[META6]] = !{float 2.500000e+00} +; CHECK: [[META7]] = !{i32 5, i32 6} +; CHECK: [[META8]] = !{i32 4, i32 5} ;.