From 79f090f9c971a9a2405b2ea952230402d5489773 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Mon, 29 Aug 2016 17:16:54 -0700 Subject: [PATCH 01/38] WIP: New refcount representation. --- include/swift/Runtime/HeapObject.h | 4 +- stdlib/public/SwiftShims/HeapObject.h | 6 +- stdlib/public/SwiftShims/RefCount.h | 724 +++++++++++++++++++------- stdlib/public/runtime/HeapObject.cpp | 74 ++- stdlib/public/runtime/SwiftObject.mm | 8 +- 5 files changed, 568 insertions(+), 248 deletions(-) diff --git a/include/swift/Runtime/HeapObject.h b/include/swift/Runtime/HeapObject.h index e94a67a6b9111..30451881d11bb 100644 --- a/include/swift/Runtime/HeapObject.h +++ b/include/swift/Runtime/HeapObject.h @@ -235,13 +235,13 @@ void (*SWIFT_CC(RegisterPreservingCC) _swift_nonatomic_retain_n)(HeapObject *obj static inline void _swift_retain_inlined(HeapObject *object) { if (object) { - object->refCount.increment(); + object->refCounts.increment(); } } static inline void _swift_nonatomic_retain_inlined(HeapObject *object) { if (object) { - object->refCount.incrementNonAtomic(); + object->refCounts.incrementNonAtomic(); } } diff --git a/stdlib/public/SwiftShims/HeapObject.h b/stdlib/public/SwiftShims/HeapObject.h index 585095f249214..3c2f9175c1ce5 100644 --- a/stdlib/public/SwiftShims/HeapObject.h +++ b/stdlib/public/SwiftShims/HeapObject.h @@ -31,8 +31,7 @@ typedef struct HeapMetadata HeapMetadata; // The members of the HeapObject header that are not shared by a // standard Objective-C instance #define SWIFT_HEAPOBJECT_NON_OBJC_MEMBERS \ - StrongRefCount refCount; \ - WeakRefCount weakRefCount + InlineRefCounts refCounts; /// The Swift heap-object header. struct HeapObject { @@ -48,8 +47,7 @@ struct HeapObject { // Initialize a HeapObject header as appropriate for a newly-allocated object. constexpr HeapObject(HeapMetadata const *newMetadata) : metadata(newMetadata) - , refCount(StrongRefCount::Initialized) - , weakRefCount(WeakRefCount::Initialized) + , refCounts(InlineRefCounts::Initialized) { } #endif }; diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 47b77a14495fb..33b4fa9b202fb 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -20,24 +20,209 @@ #include "SwiftStdint.h" typedef struct { - __swift_uint32_t refCount __attribute__((__unavailable__)); -} StrongRefCount; - -typedef struct { - __swift_uint32_t weakRefCount __attribute__((__unavailable__)); -} WeakRefCount; + __swift_uint64_t refCounts __attribute__((__unavailable__)); +} InlineRefCounts; // not __cplusplus #else // __cplusplus #include +#include #include #include +#include "llvm/Support/Compiler.h" #include "swift/Basic/type_traits.h" -// Strong reference count. +#define relaxed std::memory_order_relaxed +#define acquire std::memory_order_acquire +#define release std::memory_order_release + +// Basic encoding of refcount and flag data into the object's header. +// FIXME: Specialize this for a 32-bit field on 32-bit architectures. +class InlineRefCountBits { + uint64_t bits; + + // Layout of bits. + // field value = (bits & mask) >> shift + +# define MaskForField(name) (((1UL<> name##Shift) +# define SetField(name, val) \ + bits = (bits & ~name##Mask) | (((uint64_t(val) << name##Shift) & name##Mask)) + + enum class DiscriminatorValue { + /* 00 */ Normal = 0, + /* 01 */ Unused = 1, + /* 10 */ HasSideTable = 2, + /* 11 */ Unusual = 3, + }; + + // InlineRefCountBits uses always_inline everywhere + // to improve performance of debug builds. + + private: + LLVM_ATTRIBUTE_ALWAYS_INLINE + DiscriminatorValue getDiscriminator() const { + return DiscriminatorValue(GetField(Discriminator)); + } + + public: + + LLVM_ATTRIBUTE_ALWAYS_INLINE + InlineRefCountBits() = default; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + constexpr InlineRefCountBits(uint32_t strongCount, uint32_t unownedCount) + : bits((uint64_t(strongCount) << StrongRefCountShift) | + (uint64_t(unownedCount) << UnownedRefCountShift)) + { } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool hasSideTable() const { + return getDiscriminator() == DiscriminatorValue::HasSideTable; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + uintptr_t getSideTable() const { + assert(hasSideTable()); + // Stored value is a shifted pointer. + return uintptr_t(GetField(SideTable)) << DiscriminatorBitCount; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + uint32_t getUnownedRefCount() const { + assert(!hasSideTable()); + return uint32_t(GetField(UnownedRefCount)); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool getIsPinned() const { + assert(!hasSideTable()); + return bool(GetField(IsPinned)); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool getIsDeiniting() const { + assert(!hasSideTable()); + return bool(GetField(IsDeiniting)); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + uint32_t getStrongRefCount() const { + assert(!hasSideTable()); + return uint32_t(GetField(StrongRefCount)); + } + + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setHasSideTable(bool value) { + bits = 0; + SetField(Discriminator, + value ? uint32_t(DiscriminatorValue::HasSideTable) + : uint32_t(DiscriminatorValue::Normal)); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setSideTable(uintptr_t value) { + assert(hasSideTable()); + // Stored value is a shifted pointer + uintptr_t storedValue = value >> DiscriminatorShift; + assert(storedValue << DiscriminatorShift == value); + SetField(SideTable, storedValue); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setUnownedRefCount(uint32_t value) { + assert(!hasSideTable()); + SetField(UnownedRefCount, value); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setIsPinned(bool value) { + assert(!hasSideTable()); + SetField(IsPinned, value); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setIsDeiniting(bool value) { + assert(!hasSideTable()); + SetField(IsDeiniting, value); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setStrongRefCount(uint32_t value) { + assert(!hasSideTable()); + SetField(StrongRefCount, value); + } + + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void incrementStrongRefCount(uint32_t inc = 1) { + setStrongRefCount(getStrongRefCount() + inc); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void incrementUnownedRefCount(uint32_t inc = 1) { + setUnownedRefCount(getUnownedRefCount() + inc); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementStrongRefCount(uint32_t dec = 1) { + setStrongRefCount(getStrongRefCount() - dec); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementUnownedRefCount(uint32_t dec = 1) { + setUnownedRefCount(getUnownedRefCount() - dec); + } + +# undef GetField +# undef SetField +}; // Barriers // @@ -46,70 +231,65 @@ typedef struct { // Strong refcount decrement is a release operation with respect to other // memory locations. When an object's reference count becomes zero, // an acquire fence is performed before beginning Swift deinit or ObjC -// dealloc code. This ensures that the deinit code sees all modifications +// -dealloc code. This ensures that the deinit code sees all modifications // of the object's contents that were made before the object was released. -class StrongRefCount { - uint32_t refCount; - - // The low bit is the pinned marker. - // The next bit is the deallocating marker. - // The remaining bits are the reference count. - // refCount == RC_ONE means reference count == 1. - enum : uint32_t { - RC_PINNED_FLAG = 0x1, - RC_DEALLOCATING_FLAG = 0x2, - - RC_FLAGS_COUNT = 2, - RC_FLAGS_MASK = 3, - RC_COUNT_MASK = ~RC_FLAGS_MASK, - - RC_ONE = RC_FLAGS_MASK + 1 - }; - - static_assert(RC_ONE == RC_DEALLOCATING_FLAG << 1, - "deallocating bit must be adjacent to refcount bits"); - static_assert(RC_ONE == 1 << RC_FLAGS_COUNT, - "inconsistent refcount flags"); - static_assert(RC_ONE == 1 + RC_FLAGS_MASK, - "inconsistent refcount flags"); +class InlineRefCounts { + std::atomic refCounts; public: enum Initialized_t { Initialized }; - // StrongRefCount must be trivially constructible to avoid ObjC++ - // destruction overhead at runtime. Use StrongRefCount(Initialized) to produce - // an initialized instance. - StrongRefCount() = default; + // InlineRefCounts must be trivially constructible to avoid ObjC++ + // destruction overhead at runtime. Use InlineRefCounts(Initialized) + // to produce an initialized instance. + InlineRefCounts() = default; // Refcount of a new object is 1. - constexpr StrongRefCount(Initialized_t) - : refCount(RC_ONE) { } + constexpr InlineRefCounts(Initialized_t) + : refCounts(InlineRefCountBits(1, 1)) { } void init() { - refCount = RC_ONE; + refCounts.store(InlineRefCountBits(1, 1), relaxed); } + /// Initialize for a stack promoted object. This prevents that the final + /// release frees the memory of the object. + void initForNotFreeing() { + refCounts.store(InlineRefCountBits(1, 2), relaxed); + } + + // Increment the reference count. void increment() { - __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { + newbits = oldbits; + newbits.incrementStrongRefCount(); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } void incrementNonAtomic() { - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val += RC_ONE; - __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); + auto bits = refCounts.load(relaxed); + bits.incrementStrongRefCount(); + refCounts.store(bits, relaxed); } // Increment the reference count by n. void increment(uint32_t n) { - __atomic_fetch_add(&refCount, n << RC_FLAGS_COUNT, __ATOMIC_RELAXED); + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { + newbits = oldbits; + newbits.incrementStrongRefCount(n); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } void incrementNonAtomic(uint32_t n) { - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val += n << RC_FLAGS_COUNT; - __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); + auto bits = refCounts.load(relaxed); + bits.incrementStrongRefCount(n); + refCounts.store(bits, relaxed); } // Try to simultaneously set the pinned flag and increment the @@ -122,103 +302,115 @@ class StrongRefCount { // // Postcondition: the flag is set. bool tryIncrementAndPin() { - uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - while (true) { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { // If the flag is already set, just fail. - if (oldval & RC_PINNED_FLAG) { + if (oldbits.getIsPinned()) { return false; } // Try to simultaneously set the flag and increment the reference count. - uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); - if (__atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - return true; - } - - // Try again; oldval has been updated with the value we saw. - } + newbits = oldbits; + newbits.setIsPinned(true); + newbits.incrementStrongRefCount(); + } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + return true; } bool tryIncrementAndPinNonAtomic() { - uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + auto bits = refCounts.load(relaxed); + // If the flag is already set, just fail. - if (oldval & RC_PINNED_FLAG) { + if (bits.getIsPinned()) { return false; } // Try to simultaneously set the flag and increment the reference count. - uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); - __atomic_store_n(&refCount, newval, __ATOMIC_RELAXED); + bits.setIsPinned(true); + bits.incrementStrongRefCount(); + refCounts.store(bits, relaxed); return true; } - // Increment the reference count, unless the object is deallocating. + // Increment the reference count, unless the object is deiniting. bool tryIncrement() { - // FIXME: this could be better on LL/SC architectures like arm64 - uint32_t oldval = __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); - if (oldval & RC_DEALLOCATING_FLAG) { - __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); - return false; - } else { - return true; - } + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { + if (oldbits.getIsDeiniting()) { + return false; + } + + newbits = oldbits; + newbits.incrementStrongRefCount(); + } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + return true; } // Simultaneously clear the pinned flag and decrement the reference // count. // // Precondition: the pinned flag is set. - bool decrementAndUnpinShouldDeallocate() { - return doDecrementShouldDeallocate(); + bool decrementAndUnpinShouldDeinit() { + return doDecrementShouldDeinit(); } - bool decrementAndUnpinShouldDeallocateNonAtomic() { - return doDecrementShouldDeallocateNonAtomic(); + bool decrementAndUnpinShouldDeinitNonAtomic() { + return doDecrementShouldDeinitNonAtomic(); } // Decrement the reference count. - // Return true if the caller should now deallocate the object. - bool decrementShouldDeallocate() { - return doDecrementShouldDeallocate(); + // Return true if the caller should now deinit the object. + bool decrementShouldDeinit() { + return doDecrementShouldDeinit(); } - bool decrementShouldDeallocateNonAtomic() { - return doDecrementShouldDeallocateNonAtomic(); + bool decrementShouldDeinitNonAtomic() { + return doDecrementShouldDeinitNonAtomic(); } - bool decrementShouldDeallocateN(uint32_t n) { - return doDecrementShouldDeallocateN(n); + bool decrementShouldDeinitN(uint32_t n) { + return doDecrementShouldDeinitN(n); } - // Set the RC_DEALLOCATING_FLAG flag non-atomically. + // Non-atomically release the last strong reference and mark the + // object as deiniting. // // Precondition: the reference count must be 1 - void decrementFromOneAndDeallocateNonAtomic() { - assert(refCount == RC_ONE && "Expect a count of 1"); - __atomic_store_n(&refCount, RC_DEALLOCATING_FLAG, __ATOMIC_RELAXED); + void decrementFromOneAndDeinitNonAtomic() { + auto bits = refCounts.load(relaxed); + assert(bits.getStrongRefCount() == 1 && "Expect a count of 1"); + + bits.setStrongRefCount(0); + bits.setIsDeiniting(true); + refCounts.store(bits, relaxed); } - bool decrementShouldDeallocateNNonAtomic(uint32_t n) { - return doDecrementShouldDeallocateNNonAtomic(n); + bool decrementShouldDeinitNNonAtomic(uint32_t n) { + return doDecrementShouldDeinitNNonAtomic(n); } // Return the reference count. - // During deallocation the reference count is undefined. + // Once deinit begins the reference count is undefined. uint32_t getCount() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; + auto bits = refCounts.load(relaxed); + return bits.getStrongRefCount(); } // Return whether the reference count is exactly 1. - // During deallocation the reference count is undefined. + // Once deinit begins the reference count is undefined. bool isUniquelyReferenced() const { return getCount() == 1; } // Return whether the reference count is exactly 1 or the pin flag - // is set. During deallocation the reference count is undefined. + // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { - auto value = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + auto bits = refCounts.load(relaxed); + return (bits.getStrongRefCount() == 1 || bits.getIsPinned()); + // FIXME: rewrite with rotate if compiler doesn't do it. + // Rotating right by one sets the sign bit to the pinned bit. After // rotation, the dealloc flag is the least significant bit followed by the // reference count. A reference count of two or higher means that our value @@ -226,25 +418,58 @@ class StrongRefCount { // the value is negative. // Note: Because we are using the sign bit for testing pinnedness it // is important to do a signed comparison below. - static_assert(RC_PINNED_FLAG == 1, - "The pinned flag must be the lowest bit"); - auto rotateRightByOne = ((value >> 1) | (value << 31)); - return (int32_t)rotateRightByOne < (int32_t)RC_ONE; + // static_assert(RC_PINNED_FLAG == 1, + // "The pinned flag must be the lowest bit"); + // auto rotateRightByOne = ((value >> 1) | (value << 31)); + // return (int32_t)rotateRightByOne < (int32_t)RC_ONE; } - // Return true if the object is inside deallocation. - bool isDeallocating() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) & RC_DEALLOCATING_FLAG; + // Return true if the object has started deiniting. + bool isDeiniting() const { + auto bits = refCounts.load(relaxed); + return bits.getIsDeiniting(); } private: template - bool doDecrementShouldDeallocate() { + bool doDecrementShouldDeinit() { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + bool performDeinit; + do { + newbits = oldbits; + + // ClearPinnedFlag assumes the pinned flag is already set. + assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && + "unpinning reference that was not pinned"); + assert(oldbits.getStrongRefCount() >= 1 && + "releasing reference with a refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + if (ClearPinnedFlag) { + newbits.setIsPinned(false); + } + newbits.decrementStrongRefCount(); + performDeinit = (newbits.getStrongRefCount() == 0 && + !newbits.getIsDeiniting()); + if (performDeinit) { + newbits.setIsDeiniting(true); + } + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + if (performDeinit) { + std::atomic_thread_fence(acquire); + } + return performDeinit; + + /* Old implementation for optimization reference: + // If we're being asked to clear the pinned flag, we can assume // it's already set. constexpr uint32_t quantum = (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t newval = __atomic_sub_fetch(&refCount, quantum, __ATOMIC_RELEASE); + uint32_t newval = __atomic_sub_fetch(&strongRefCount, quantum, __ATOMIC_RELEASE); assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && "unpinning reference that was not pinned"); @@ -268,23 +493,62 @@ class StrongRefCount { // // This also performs the before-deinit acquire barrier if we set the flag. static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); + "fix decrementShouldDeinit() if you add more flags"); uint32_t oldval = 0; newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + */ } template - bool doDecrementShouldDeallocateNonAtomic() { + bool doDecrementShouldDeinitNonAtomic() { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + // FIXME: can probably do this without CAS once zeroing weak references + // are pushed to the side table. Then the presence of inline RC will + // prove that the object is not already weakly-referenced so there can't + // be a race vs a weak load; and presumably no other thread can see + // the object so there can't be a race vs a weak store. + + bool performDeinit; + do { + newbits = oldbits; + + // ClearPinnedFlag assumes the pinned flag is already set. + assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && + "unpinning reference that was not pinned"); + assert(oldbits.getStrongRefCount() >= 1 && + "releasing reference with a refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + if (ClearPinnedFlag) { + newbits.setIsPinned(false); + } + newbits.decrementStrongRefCount(); + performDeinit = (newbits.getStrongRefCount() == 0 && + !newbits.getIsDeiniting()); + if (performDeinit) { + newbits.setIsDeiniting(true); + } + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + if (performDeinit) { + std::atomic_thread_fence(acquire); + } + return performDeinit; + + /* Old implementation for optimization reference: + // If we're being asked to clear the pinned flag, we can assume // it's already set. constexpr uint32_t quantum = (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + uint32_t val = __atomic_load_n(&strongRefCount, __ATOMIC_RELAXED); val -= quantum; - __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); - uint32_t newval = refCount; + __atomic_store_n(&strongRefCount, val, __ATOMIC_RELEASE); + uint32_t newval = strongRefCount; assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && "unpinning reference that was not pinned"); @@ -308,19 +572,53 @@ class StrongRefCount { // // This also performs the before-deinit acquire barrier if we set the flag. static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); + "fix decrementShouldDeinit() if you add more flags"); uint32_t oldval = 0; newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + */ } template - bool doDecrementShouldDeallocateN(uint32_t n) { + bool doDecrementShouldDeinitN(uint32_t n) { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + bool performDeinit; + do { + newbits = oldbits; + + // ClearPinnedFlag assumes the pinned flag is already set. + assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && + "unpinning reference that was not pinned"); + assert(oldbits.getStrongRefCount() >= n && + "releasing reference with a refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + if (ClearPinnedFlag) { + newbits.setIsPinned(false); + } + newbits.decrementStrongRefCount(n); + performDeinit = (newbits.getStrongRefCount() == 0 && + !newbits.getIsDeiniting()); + if (performDeinit) { + newbits.setIsDeiniting(true); + } + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + if (performDeinit) { + std::atomic_thread_fence(acquire); + } + return performDeinit; + + + /* Old implementation for optimization reference: + // If we're being asked to clear the pinned flag, we can assume // it's already set. uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t newval = __atomic_sub_fetch(&refCount, delta, __ATOMIC_RELEASE); + uint32_t newval = __atomic_sub_fetch(&strongRefCount, delta, __ATOMIC_RELEASE); assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && "unpinning reference that was not pinned"); @@ -344,21 +642,60 @@ class StrongRefCount { // // This also performs the before-deinit acquire barrier if we set the flag. static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); + "fix decrementShouldDeinit() if you add more flags"); uint32_t oldval = 0; newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + */ } template - bool doDecrementShouldDeallocateNNonAtomic(uint32_t n) { + bool doDecrementShouldDeinitNNonAtomic(uint32_t n) { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + // FIXME: can probably do this without CAS once zeroing weak references + // are pushed to the side table. Then the presence of inline RC will + // prove that the object is not already weakly-referenced so there can't + // be a race vs a weak load; and presumably no other thread can see + // the object so there can't be a race vs a weak store. + + bool performDeinit; + do { + newbits = oldbits; + + // ClearPinnedFlag assumes the pinned flag is already set. + assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && + "unpinning reference that was not pinned"); + assert(oldbits.getStrongRefCount() >= n && + "releasing reference with a refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + if (ClearPinnedFlag) { + newbits.setIsPinned(false); + } + newbits.decrementStrongRefCount(n); + performDeinit = (newbits.getStrongRefCount() == 0 && + !newbits.getIsDeiniting()); + if (performDeinit) { + newbits.setIsDeiniting(true); + } + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + if (performDeinit) { + std::atomic_thread_fence(acquire); + } + return performDeinit; + + /* Old implementation for optimization reference: + // If we're being asked to clear the pinned flag, we can assume // it's already set. uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + uint32_t val = __atomic_load_n(&strongRefCount, __ATOMIC_RELAXED); val -= delta; - __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); + __atomic_store_n(&strongRefCount, val, __ATOMIC_RELEASE); uint32_t newval = val; assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && @@ -383,109 +720,96 @@ class StrongRefCount { // // This also performs the before-deinit acquire barrier if we set the flag. static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); + "fix decrementShouldDeinit() if you add more flags"); uint32_t oldval = 0; newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + */ } -}; - - -// Weak reference count. -class WeakRefCount { - uint32_t refCount; - - enum : uint32_t { - // There isn't really a flag here. - RC_UNUSED_FLAG = 1, - - RC_FLAGS_COUNT = 1, - RC_FLAGS_MASK = 1, - RC_COUNT_MASK = ~RC_FLAGS_MASK, - - RC_ONE = RC_FLAGS_MASK + 1 - }; - - static_assert(RC_ONE == 1 << RC_FLAGS_COUNT, - "inconsistent refcount flags"); - static_assert(RC_ONE == 1 + RC_FLAGS_MASK, - "inconsistent refcount flags"); - - public: - enum Initialized_t { Initialized }; - // WeakRefCount must be trivially constructible to avoid ObjC++ - // destruction overhead at runtime. Use WeakRefCount(Initialized) to produce - // an initialized instance. - WeakRefCount() = default; + // UNOWNED - // Weak refcount of a new object is 1. - constexpr WeakRefCount(Initialized_t) - : refCount(RC_ONE) { } - - void init() { - refCount = RC_ONE; - } - - /// Initialize for a stack promoted object. This prevents that the final - /// release frees the memory of the object. - void initForNotDeallocating() { - refCount = RC_ONE + RC_ONE; - } - - // Increment the weak reference count. - void increment() { - uint32_t newval = __atomic_add_fetch(&refCount, RC_ONE, __ATOMIC_RELAXED); - assert(newval >= RC_ONE && "weak refcount overflow"); - (void)newval; + public: + // Increment the unowned reference count. + void incrementUnowned() { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { + newbits = oldbits; + newbits.incrementUnownedRefCount(); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } - /// Increment the weak reference count by n. - void increment(uint32_t n) { - uint32_t addval = (n << RC_FLAGS_COUNT); - uint32_t newval = __atomic_add_fetch(&refCount, addval, __ATOMIC_RELAXED); - assert(newval >= addval && "weak refcount overflow"); - (void)newval; + // Increment the unowned reference count by n. + void incrementUnowned(uint32_t n) { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + do { + newbits = oldbits; + newbits.incrementUnownedRefCount(n); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } - // Decrement the weak reference count. - // Return true if the caller should deallocate the object. - bool decrementShouldDeallocate() { - uint32_t oldval = __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); - assert(oldval >= RC_ONE && "weak refcount underflow"); - - // Should dealloc if count was 1 before decrementing (i.e. it is zero now) - return (oldval & RC_COUNT_MASK) == RC_ONE; + // Decrement the unowned reference count. + // Return true if the caller should free the object. + bool decrementUnownedShouldFree() { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + bool performFree; + do { + newbits = oldbits; + + assert(oldbits.getUnownedRefCount() >= 1 && + "releasing reference with an unowned refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + newbits.decrementUnownedRefCount(); + performFree = (newbits.getUnownedRefCount() == 0); + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + return performFree; } - /// Decrement the weak reference count. - /// Return true if the caller should deallocate the object. - bool decrementShouldDeallocateN(uint32_t n) { - uint32_t subval = (n << RC_FLAGS_COUNT); - uint32_t oldval = __atomic_fetch_sub(&refCount, subval, __ATOMIC_RELAXED); - assert(oldval >= subval && "weak refcount underflow"); - - // Should dealloc if count was subval before decrementing (i.e. it is zero now) - return (oldval & RC_COUNT_MASK) == subval; + // Decrement the unowned reference count. + // Return true if the caller should free the object. + bool decrementUnownedShouldFreeN(uint32_t n) { + auto oldbits = refCounts.load(relaxed); + InlineRefCountBits newbits; + + bool performFree; + do { + newbits = oldbits; + + assert(oldbits.getUnownedRefCount() >= n && + "releasing reference with an unowned refcount of zero"); + + // FIXME: hand optimize these bit operations if necessary + newbits.decrementUnownedRefCount(n); + performFree = (newbits.getUnownedRefCount() == 0); + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + return performFree; } // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. - uint32_t getCount() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; + uint32_t getUnownedCount() const { + auto bits = refCounts.load(relaxed); + return bits.getUnownedRefCount(); } }; -static_assert(swift::IsTriviallyConstructible::value, - "StrongRefCount must be trivially initializable"); -static_assert(swift::IsTriviallyConstructible::value, - "WeakRefCount must be trivially initializable"); -static_assert(std::is_trivially_destructible::value, - "StrongRefCount must be trivially destructible"); -static_assert(std::is_trivially_destructible::value, - "WeakRefCount must be trivially destructible"); +static_assert(swift::IsTriviallyConstructible::value, + "InlineRefCounts must be trivially initializable"); +static_assert(std::is_trivially_destructible::value, + "InlineRefCounts must be trivially destructible"); + +#undef relaxed +#undef acquire +#undef release // __cplusplus #endif diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index ea217b3d46feb..a96cd48fb91c6 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -65,8 +65,7 @@ SWIFT_RT_ENTRY_IMPL(swift_allocObject)(HeapMetadata const *metadata, requiredAlignmentMask)); // FIXME: this should be a placement new but that adds a null check object->metadata = metadata; - object->refCount.init(); - object->weakRefCount.init(); + object->refCounts.init(); // If leak tracking is enabled, start tracking this object. SWIFT_LEAKS_START_TRACKING_OBJECT(object); @@ -78,8 +77,7 @@ HeapObject * swift::swift_initStackObject(HeapMetadata const *metadata, HeapObject *object) { object->metadata = metadata; - object->refCount.init(); - object->weakRefCount.initForNotDeallocating(); + object->refCounts.initForNotFreeing(); return object; @@ -87,11 +85,11 @@ swift::swift_initStackObject(HeapMetadata const *metadata, void swift::swift_verifyEndOfLifetime(HeapObject *object) { - if (object->refCount.getCount() != 0) + if (object->refCounts.getCount() != 0) swift::fatalError(/* flags = */ 0, "fatal error: stack object escaped\n"); - if (object->weakRefCount.getCount() != 1) + if (object->refCounts.getUnownedCount() != 1) swift::fatalError(/* flags = */ 0, "fatal error: weak/unowned reference to stack object\n"); } @@ -245,7 +243,7 @@ void swift::swift_nonatomic_release(HeapObject *object) { SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_release)(HeapObject *object) { - if (object && object->refCount.decrementShouldDeallocateNonAtomic()) { + if (object && object->refCounts.decrementShouldDeinitNonAtomic()) { // TODO: Use non-atomic _swift_release_dealloc? _swift_release_dealloc(object); } @@ -270,7 +268,7 @@ extern "C" void SWIFT_RT_ENTRY_IMPL(swift_retain_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { if (object) { - object->refCount.increment(n); + object->refCounts.increment(n); } } @@ -286,7 +284,7 @@ extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_retain_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { if (object) { - object->refCount.incrementNonAtomic(n); + object->refCounts.incrementNonAtomic(n); } } @@ -301,7 +299,7 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_release)(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCount.decrementShouldDeallocate()) { + if (object && object->refCounts.decrementShouldDeinit()) { _swift_release_dealloc(object); } } @@ -316,13 +314,13 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_release_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCount.decrementShouldDeallocateN(n)) { + if (object && object->refCounts.decrementShouldDeinitN(n)) { _swift_release_dealloc(object); } } void swift::swift_setDeallocating(HeapObject *object) { - object->refCount.decrementFromOneAndDeallocateNonAtomic(); + object->refCounts.decrementFromOneAndDeinitNonAtomic(); } SWIFT_RT_ENTRY_VISIBILITY @@ -335,17 +333,17 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_release_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCount.decrementShouldDeallocateNNonAtomic(n)) { + if (object && object->refCounts.decrementShouldDeinitNNonAtomic(n)) { _swift_release_dealloc(object); } } size_t swift::swift_retainCount(HeapObject *object) { - return object->refCount.getCount(); + return object->refCounts.getCount(); } size_t swift::swift_unownedRetainCount(HeapObject *object) { - return object->weakRefCount.getCount(); + return object->refCounts.getUnownedCount(); } SWIFT_RT_ENTRY_VISIBILITY @@ -354,7 +352,7 @@ void swift::swift_unownedRetain(HeapObject *object) if (!object) return; - object->weakRefCount.increment(); + object->refCounts.incrementUnowned(); } SWIFT_RT_ENTRY_VISIBILITY @@ -363,7 +361,7 @@ void swift::swift_unownedRelease(HeapObject *object) if (!object) return; - if (object->weakRefCount.decrementShouldDeallocate()) { + if (object->refCounts.decrementUnownedShouldFree()) { // Only class objects can be weak-retained and weak-released. auto metadata = object->metadata; assert(metadata->isClassObject()); @@ -382,7 +380,7 @@ void swift::swift_unownedRetain_n(HeapObject *object, int n) if (!object) return; - object->weakRefCount.increment(n); + object->refCounts.incrementUnowned(n); } SWIFT_RT_ENTRY_VISIBILITY @@ -392,7 +390,7 @@ void swift::swift_unownedRelease_n(HeapObject *object, int n) if (!object) return; - if (object->weakRefCount.decrementShouldDeallocateN(n)) { + if (object->refCounts.decrementUnownedShouldFreeN(n)) { // Only class objects can be weak-retained and weak-released. auto metadata = object->metadata; assert(metadata->isClassObject()); @@ -411,7 +409,7 @@ HeapObject *swift::swift_tryPin(HeapObject *object) // Try to set the flag. If this succeeds, the caller will be // responsible for clearing it. - if (object->refCount.tryIncrementAndPin()) { + if (object->refCounts.tryIncrementAndPin()) { return object; } @@ -423,7 +421,7 @@ HeapObject *swift::swift_tryPin(HeapObject *object) SWIFT_RT_ENTRY_VISIBILITY void swift::swift_unpin(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCount.decrementAndUnpinShouldDeallocate()) { + if (object && object->refCounts.decrementAndUnpinShouldDeinit()) { _swift_release_dealloc(object); } } @@ -441,7 +439,7 @@ HeapObject *swift::swift_nonatomic_tryPin(HeapObject *object) // Try to set the flag. If this succeeds, the caller will be // responsible for clearing it. - if (object->refCount.tryIncrementAndPinNonAtomic()) { + if (object->refCounts.tryIncrementAndPinNonAtomic()) { return object; } @@ -453,7 +451,7 @@ HeapObject *swift::swift_nonatomic_tryPin(HeapObject *object) SWIFT_RT_ENTRY_VISIBILITY void swift::swift_nonatomic_unpin(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCount.decrementAndUnpinShouldDeallocateNonAtomic()) { + if (object && object->refCounts.decrementAndUnpinShouldDeinitNonAtomic()) { _swift_release_dealloc(object); } } @@ -465,7 +463,7 @@ HeapObject *SWIFT_RT_ENTRY_IMPL(swift_tryRetain)(HeapObject *object) if (!object) return nullptr; - if (object->refCount.tryIncrement()) return object; + if (object->refCounts.tryIncrement()) return object; else return nullptr; } @@ -479,7 +477,7 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" bool SWIFT_RT_ENTRY_IMPL(swift_isDeallocating)(HeapObject *object) { if (!object) return false; - return object->refCount.isDeallocating(); + return object->refCounts.isDeiniting(); } SWIFT_RT_ENTRY_VISIBILITY @@ -487,10 +485,10 @@ void swift::swift_unownedRetainStrong(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { if (!object) return; - assert(object->weakRefCount.getCount() && + assert(object->refCounts.getUnownedCount() && "object is not currently weakly retained"); - if (! object->refCount.tryIncrement()) + if (! object->refCounts.tryIncrement()) _swift_abortRetainUnowned(object); } @@ -500,24 +498,24 @@ swift::swift_unownedRetainStrongAndRelease(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { if (!object) return; - assert(object->weakRefCount.getCount() && + assert(object->refCounts.getUnownedCount() && "object is not currently weakly retained"); - if (! object->refCount.tryIncrement()) + if (! object->refCounts.tryIncrement()) _swift_abortRetainUnowned(object); // This should never cause a deallocation. - bool dealloc = object->weakRefCount.decrementShouldDeallocate(); + bool dealloc = object->refCounts.decrementUnownedShouldFree(); assert(!dealloc && "retain-strong-and-release caused dealloc?"); (void) dealloc; } void swift::swift_unownedCheck(HeapObject *object) { if (!object) return; - assert(object->weakRefCount.getCount() && + assert(object->refCounts.getUnownedCount() && "object is not currently weakly retained"); - if (object->refCount.isDeallocating()) + if (object->refCounts.isDeiniting()) _swift_abortRetainUnowned(object); } @@ -605,7 +603,7 @@ extern "C" void swift_deallocPartialClassInstance(HeapObject *object, #endif // The strong reference count should be +1 -- tear down the object - bool shouldDeallocate = object->refCount.decrementShouldDeallocate(); + bool shouldDeallocate = object->refCounts.decrementShouldDeinit(); assert(shouldDeallocate); (void) shouldDeallocate; swift_deallocClassInstance(object, allocatedSize, allocatedAlignMask); @@ -629,7 +627,7 @@ void swift::swift_deallocObject(HeapObject *object, size_t allocatedAlignMask) SWIFT_CC(RegisterPreservingCC_IMPL) { assert(isAlignmentMask(allocatedAlignMask)); - assert(object->refCount.isDeallocating()); + assert(object->refCounts.isDeiniting()); #ifdef SWIFT_RUNTIME_CLOBBER_FREED_OBJECTS memset_pattern8((uint8_t *)object + sizeof(HeapObject), "\xAB\xAD\x1D\xEA\xF4\xEE\xD0\bB9", @@ -704,7 +702,7 @@ void swift::swift_deallocObject(HeapObject *object, // release, we will fall back on swift_unownedRelease, which does an // atomic decrement (and has the ability to reconstruct // allocatedSize and allocatedAlignMask). - if (object->weakRefCount.getCount() == 1) { + if (object->refCounts.getUnownedCount() == 1) { SWIFT_RT_ENTRY_CALL(swift_slowDealloc) (object, allocatedSize, allocatedAlignMask); @@ -766,7 +764,7 @@ HeapObject *swift::swift_weakLoadStrong(WeakReference *ref) { __atomic_store_n(&ref->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); return nullptr; } - if (object->refCount.isDeallocating()) { + if (object->refCounts.isDeiniting()) { __atomic_store_n(&ref->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); return nullptr; @@ -814,7 +812,7 @@ void swift::swift_weakCopyInit(WeakReference *dest, WeakReference *src) { if (object == nullptr) { __atomic_store_n(&src->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); dest->Value = (uintptr_t)nullptr; - } else if (object->refCount.isDeallocating()) { + } else if (object->refCounts.isDeiniting()) { __atomic_store_n(&src->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); dest->Value = (uintptr_t)nullptr; @@ -829,7 +827,7 @@ void swift::swift_weakTakeInit(WeakReference *dest, WeakReference *src) { auto object = (HeapObject*) (src->Value & ~WR_NATIVE); if (object == nullptr) { dest->Value = (uintptr_t)nullptr; - } else if (object->refCount.isDeallocating()) { + } else if (object->refCounts.isDeiniting()) { dest->Value = (uintptr_t)nullptr; SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); } else { diff --git a/stdlib/public/runtime/SwiftObject.mm b/stdlib/public/runtime/SwiftObject.mm index 523cd9ccaf5d5..aa7c454dbc544 100644 --- a/stdlib/public/runtime/SwiftObject.mm +++ b/stdlib/public/runtime/SwiftObject.mm @@ -1345,8 +1345,8 @@ static bool usesNativeSwiftReferenceCounting_nonNull( const HeapObject* object ) SWIFT_CC(RegisterPreservingCC_IMPL) { assert(object != nullptr); - assert(!object->refCount.isDeallocating()); - return object->refCount.isUniquelyReferenced(); + assert(!object->refCounts.isDeiniting()); + return object->refCounts.isUniquelyReferenced(); } bool swift::swift_isUniquelyReferenced_native(const HeapObject* object) { @@ -1455,8 +1455,8 @@ static bool usesNativeSwiftReferenceCounting_nonNull( const HeapObject* object) SWIFT_CC(RegisterPreservingCC_IMPL) { assert(object != nullptr); - assert(!object->refCount.isDeallocating()); - return object->refCount.isUniquelyReferencedOrPinned(); + assert(!object->refCounts.isDeiniting()); + return object->refCounts.isUniquelyReferencedOrPinned(); } using ClassExtents = TwoWordPair; From fcdf1216229e212003a6d740ffd7ae2306fad1bd Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Mon, 29 Aug 2016 17:16:54 -0700 Subject: [PATCH 02/38] WIP: New refcount representation. From dcd0f519eae6a04f9ed35e6e2190976e1fec9b65 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 31 Aug 2016 22:02:58 -0700 Subject: [PATCH 03/38] WIP: Bias strong refcount. Add strong refcount overflow checking. --- stdlib/public/SwiftShims/RefCount.h | 558 +++++++++++----------------- stdlib/public/runtime/Errors.cpp | 8 + unittests/runtime/Refcounting.cpp | 75 ++++ 3 files changed, 304 insertions(+), 337 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 33b4fa9b202fb..d102b1404887c 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -39,6 +39,12 @@ typedef struct { #define acquire std::memory_order_acquire #define release std::memory_order_release +// Error reporting functions +namespace swift { + LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE + void swift_abortRetainOverflow(); +}; + // Basic encoding of refcount and flag data into the object's header. // FIXME: Specialize this for a 32-bit field on 32-bit architectures. class InlineRefCountBits { @@ -63,28 +69,29 @@ class InlineRefCountBits { IsDeinitingBitCount = 1, IsDeinitingMask = MaskForField(IsDeiniting), - StrongRefCountShift = ShiftAfterField(IsDeiniting), - StrongRefCountBitCount = 28, - StrongRefCountMask = MaskForField(StrongRefCount), + StrongExtraRefCountShift = ShiftAfterField(IsDeiniting), + StrongExtraRefCountBitCount = 29, + StrongExtraRefCountMask = MaskForField(StrongExtraRefCount), - DiscriminatorShift = ShiftAfterField(StrongRefCount), - DiscriminatorBitCount = 2, - DiscriminatorMask = MaskForField(Discriminator), + UseSlowRCShift = ShiftAfterField(StrongExtraRefCount), + UseSlowRCBitCount = 1, + UseSlowRCMask = MaskForField(UseSlowRC), + // FIXME: 63 bits but MSB must always be zero. Handle that differently. SideTableShift = 0, - SideTableBitCount = 62, + SideTableBitCount = 63, SideTableMask = MaskForField(SideTable) }; - static_assert(StrongRefCountShift == IsDeinitingShift + 1, - "IsDeiniting must be LSB-wards of StrongRefCount"); - static_assert(DiscriminatorShift + DiscriminatorBitCount == sizeof(bits)*8, - "Discriminator must be MSB"); - static_assert(SideTableBitCount + DiscriminatorBitCount == sizeof(bits)*8, + static_assert(StrongExtraRefCountShift == IsDeinitingShift + 1, + "IsDeiniting must be LSB-wards of StrongExtraRefCount"); + static_assert(UseSlowRCShift + UseSlowRCBitCount == sizeof(bits)*8, + "UseSlowRC must be MSB"); + static_assert(SideTableBitCount + UseSlowRCBitCount == sizeof(bits)*8, "wrong bit count for InlineRefCountBits side table version"); static_assert(UnownedRefCountBitCount + IsPinnedBitCount + - IsDeinitingBitCount + StrongRefCountBitCount + - DiscriminatorBitCount == sizeof(bits)*8, + IsDeinitingBitCount + StrongExtraRefCountBitCount + + UseSlowRCBitCount == sizeof(bits)*8, "wrong bit count for InlineRefCountBits refcount version"); # undef MaskForField # undef ShiftAfterField @@ -94,20 +101,18 @@ class InlineRefCountBits { # define SetField(name, val) \ bits = (bits & ~name##Mask) | (((uint64_t(val) << name##Shift) & name##Mask)) - enum class DiscriminatorValue { - /* 00 */ Normal = 0, - /* 01 */ Unused = 1, - /* 10 */ HasSideTable = 2, - /* 11 */ Unusual = 3, - }; - // InlineRefCountBits uses always_inline everywhere // to improve performance of debug builds. private: LLVM_ATTRIBUTE_ALWAYS_INLINE - DiscriminatorValue getDiscriminator() const { - return DiscriminatorValue(GetField(Discriminator)); + bool getUseSlowRC() const { + return bool(GetField(UseSlowRC)); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setUseSlowRC(bool value) { + SetField(UseSlowRC, value); } public: @@ -116,21 +121,23 @@ class InlineRefCountBits { InlineRefCountBits() = default; LLVM_ATTRIBUTE_ALWAYS_INLINE - constexpr InlineRefCountBits(uint32_t strongCount, uint32_t unownedCount) - : bits((uint64_t(strongCount) << StrongRefCountShift) | - (uint64_t(unownedCount) << UnownedRefCountShift)) + constexpr InlineRefCountBits(uint32_t strongExtraCount, uint32_t unownedCount) + : bits((uint64_t(strongExtraCount) << StrongExtraRefCountShift) | + (uint64_t(unownedCount) << UnownedRefCountShift)) { } LLVM_ATTRIBUTE_ALWAYS_INLINE bool hasSideTable() const { - return getDiscriminator() == DiscriminatorValue::HasSideTable; + // FIXME: change this when introducing immutable RC objects + return getUseSlowRC(); } LLVM_ATTRIBUTE_ALWAYS_INLINE uintptr_t getSideTable() const { assert(hasSideTable()); // Stored value is a shifted pointer. - return uintptr_t(GetField(SideTable)) << DiscriminatorBitCount; + // FIXME: Don't hard-code this shift amount? + return uintptr_t(GetField(SideTable)) << 2; } LLVM_ATTRIBUTE_ALWAYS_INLINE @@ -152,26 +159,25 @@ class InlineRefCountBits { } LLVM_ATTRIBUTE_ALWAYS_INLINE - uint32_t getStrongRefCount() const { + uint32_t getStrongExtraRefCount() const { assert(!hasSideTable()); - return uint32_t(GetField(StrongRefCount)); + return uint32_t(GetField(StrongExtraRefCount)); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setHasSideTable(bool value) { bits = 0; - SetField(Discriminator, - value ? uint32_t(DiscriminatorValue::HasSideTable) - : uint32_t(DiscriminatorValue::Normal)); + setUseSlowRC(value); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setSideTable(uintptr_t value) { assert(hasSideTable()); - // Stored value is a shifted pointer - uintptr_t storedValue = value >> DiscriminatorShift; - assert(storedValue << DiscriminatorShift == value); + // Stored value is a shifted pointer. + // FIXME: Don't hard-code this shift amount? + uintptr_t storedValue = value >> 2; + assert(storedValue << 2 == value); SetField(SideTable, storedValue); } @@ -194,25 +200,49 @@ class InlineRefCountBits { } LLVM_ATTRIBUTE_ALWAYS_INLINE - void setStrongRefCount(uint32_t value) { + void setStrongExtraRefCount(uint32_t value) { assert(!hasSideTable()); - SetField(StrongRefCount, value); + SetField(StrongExtraRefCount, value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE - void incrementStrongRefCount(uint32_t inc = 1) { - setStrongRefCount(getStrongRefCount() + inc); + // Returns true if the increment is a fast-path result. + // Returns false if the increment should fall back to some slow path + // (for example, because UseSlowRC is set or because the refcount overflowed). + LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + bool incrementStrongExtraRefCount(uint32_t inc = 1) { + // This deliberately overflows into the UseSlowRC field. + bits += uint64_t(inc) << StrongExtraRefCountShift; + return (int64_t(bits) >= 0); } LLVM_ATTRIBUTE_ALWAYS_INLINE void incrementUnownedRefCount(uint32_t inc = 1) { setUnownedRefCount(getUnownedRefCount() + inc); } + + // Returns true if the decrement is a fast-path result. + // Returns false if the decrement should fall back to some slow path + // (for example, because UseSlowRC is set + // or because the refcount is now zero and should deinit). + template + LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + bool decrementStrongExtraRefCount(uint32_t dec = 1) { + // ClearPinnedFlag assumes the flag is already set. + if (ClearPinnedFlag) + assert(getIsPinned() && "unpinning reference that was not pinned"); + + if (getIsDeiniting()) + assert(getStrongExtraRefCount() >= dec && + "releasing reference whose refcount is already zero"); + else + assert(getStrongExtraRefCount() + 1 >= dec && + "releasing reference whose refcount is already zero"); - LLVM_ATTRIBUTE_ALWAYS_INLINE - void decrementStrongRefCount(uint32_t dec = 1) { - setStrongRefCount(getStrongRefCount() - dec); + uint64_t unpin = ClearPinnedFlag ? (uint64_t(1) << IsPinnedShift) : 0; + // This deliberately underflows by borrowing from the UseSlowRC field. + bits -= unpin + (uint64_t(dec) << StrongExtraRefCountShift); + return (int64_t(bits) >= 0); } LLVM_ATTRIBUTE_ALWAYS_INLINE @@ -237,6 +267,48 @@ class InlineRefCountBits { class InlineRefCounts { std::atomic refCounts; + // Template parameters. + enum ClearPinnedFlag { DontClearPinnedFlag = false, + DoClearPinnedFlag = true }; + enum Atomicity { NonAtomic = false, Atomic = true }; + + // Out-of-line slow paths. + + LLVM_ATTRIBUTE_NOINLINE + void incrementSlow(InlineRefCountBits oldbits, uint32_t n = 1) { + if (oldbits.hasSideTable()) { + // Out-of-line slow path. + abort(); + } else { + swift::swift_abortRetainOverflow(); + } + } + + LLVM_ATTRIBUTE_NOINLINE + void incrementNonAtomicSlow(InlineRefCountBits oldbits, uint32_t n = 1) { + if (oldbits.hasSideTable()) { + // Out-of-line slow path. + abort(); + } else { + swift::swift_abortRetainOverflow(); + } + } + + LLVM_ATTRIBUTE_NOINLINE + bool tryIncrementAndPinSlow() { + abort(); + } + + LLVM_ATTRIBUTE_NOINLINE + bool tryIncrementAndPinNonAtomicSlow() { + abort(); + } + + LLVM_ATTRIBUTE_NOINLINE + bool tryIncrementSlow() { + abort(); + } + public: enum Initialized_t { Initialized }; @@ -247,16 +319,16 @@ class InlineRefCounts { // Refcount of a new object is 1. constexpr InlineRefCounts(Initialized_t) - : refCounts(InlineRefCountBits(1, 1)) { } + : refCounts(InlineRefCountBits(0, 1)) { } void init() { - refCounts.store(InlineRefCountBits(1, 1), relaxed); + refCounts.store(InlineRefCountBits(0, 1), relaxed); } /// Initialize for a stack promoted object. This prevents that the final /// release frees the memory of the object. void initForNotFreeing() { - refCounts.store(InlineRefCountBits(1, 2), relaxed); + refCounts.store(InlineRefCountBits(0, 2), relaxed); } @@ -266,14 +338,19 @@ class InlineRefCounts { InlineRefCountBits newbits; do { newbits = oldbits; - newbits.incrementStrongRefCount(); + bool fast = newbits.incrementStrongExtraRefCount(); + if (!fast) + return incrementSlow(oldbits); } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } void incrementNonAtomic() { - auto bits = refCounts.load(relaxed); - bits.incrementStrongRefCount(); - refCounts.store(bits, relaxed); + auto oldbits = refCounts.load(relaxed); + auto newbits = oldbits; + bool fast = newbits.incrementStrongExtraRefCount(); + if (!fast) + return incrementNonAtomicSlow(oldbits); + refCounts.store(newbits, relaxed); } // Increment the reference count by n. @@ -282,14 +359,19 @@ class InlineRefCounts { InlineRefCountBits newbits; do { newbits = oldbits; - newbits.incrementStrongRefCount(n); + bool fast = newbits.incrementStrongExtraRefCount(n); + if (!fast) + return incrementSlow(oldbits, n); } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } void incrementNonAtomic(uint32_t n) { - auto bits = refCounts.load(relaxed); - bits.incrementStrongRefCount(n); - refCounts.store(bits, relaxed); + auto oldbits = refCounts.load(relaxed); + auto newbits = oldbits; + bool fast = newbits.incrementStrongExtraRefCount(n); + if (!fast) + return incrementNonAtomicSlow(oldbits, n); + refCounts.store(newbits, relaxed); } // Try to simultaneously set the pinned flag and increment the @@ -306,14 +388,15 @@ class InlineRefCounts { InlineRefCountBits newbits; do { // If the flag is already set, just fail. - if (oldbits.getIsPinned()) { + if (oldbits.getIsPinned()) return false; - } // Try to simultaneously set the flag and increment the reference count. newbits = oldbits; newbits.setIsPinned(true); - newbits.incrementStrongRefCount(); + bool fast = newbits.incrementStrongExtraRefCount(); + if (!fast) + return tryIncrementAndPinSlow(); } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); return true; } @@ -322,13 +405,14 @@ class InlineRefCounts { auto bits = refCounts.load(relaxed); // If the flag is already set, just fail. - if (bits.getIsPinned()) { + if (bits.getIsPinned()) return false; - } // Try to simultaneously set the flag and increment the reference count. bits.setIsPinned(true); - bits.incrementStrongRefCount(); + bool fast = bits.incrementStrongExtraRefCount(); + if (!fast) + return tryIncrementAndPinNonAtomicSlow(); refCounts.store(bits, relaxed); return true; } @@ -338,12 +422,13 @@ class InlineRefCounts { auto oldbits = refCounts.load(relaxed); InlineRefCountBits newbits; do { - if (oldbits.getIsDeiniting()) { + if (oldbits.getIsDeiniting()) return false; - } newbits = oldbits; - newbits.incrementStrongRefCount(); + bool fast = newbits.incrementStrongExtraRefCount(); + if (!fast) + return tryIncrementSlow(); } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); return true; } @@ -353,25 +438,29 @@ class InlineRefCounts { // // Precondition: the pinned flag is set. bool decrementAndUnpinShouldDeinit() { - return doDecrementShouldDeinit(); + return doDecrementShouldDeinit(); } bool decrementAndUnpinShouldDeinitNonAtomic() { - return doDecrementShouldDeinitNonAtomic(); + return doDecrementShouldDeinitNonAtomic(); } // Decrement the reference count. // Return true if the caller should now deinit the object. bool decrementShouldDeinit() { - return doDecrementShouldDeinit(); + return doDecrementShouldDeinit(); } bool decrementShouldDeinitNonAtomic() { - return doDecrementShouldDeinitNonAtomic(); + return doDecrementShouldDeinitNonAtomic(); } bool decrementShouldDeinitN(uint32_t n) { - return doDecrementShouldDeinitN(n); + return doDecrementShouldDeinit(n); + } + + bool decrementShouldDeinitNNonAtomic(uint32_t n) { + return doDecrementShouldDeinitNonAtomic(n); } // Non-atomically release the last strong reference and mark the @@ -380,36 +469,42 @@ class InlineRefCounts { // Precondition: the reference count must be 1 void decrementFromOneAndDeinitNonAtomic() { auto bits = refCounts.load(relaxed); - assert(bits.getStrongRefCount() == 1 && "Expect a count of 1"); + assert(bits.getStrongExtraRefCount() == 0 && "Expect a refcount of 1"); - bits.setStrongRefCount(0); + bits.setStrongExtraRefCount(0); bits.setIsDeiniting(true); refCounts.store(bits, relaxed); } - bool decrementShouldDeinitNNonAtomic(uint32_t n) { - return doDecrementShouldDeinitNNonAtomic(n); - } - // Return the reference count. // Once deinit begins the reference count is undefined. uint32_t getCount() const { auto bits = refCounts.load(relaxed); - return bits.getStrongRefCount(); + return bits.getStrongExtraRefCount() + 1; } // Return whether the reference count is exactly 1. // Once deinit begins the reference count is undefined. bool isUniquelyReferenced() const { - return getCount() == 1; + auto bits = refCounts.load(relaxed); + assert(!bits.getIsDeiniting()); + if (bits.hasSideTable()) + abort(); + return bits.getStrongExtraRefCount() == 0; } // Return whether the reference count is exactly 1 or the pin flag // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { auto bits = refCounts.load(relaxed); - return (bits.getStrongRefCount() == 1 || bits.getIsPinned()); - // FIXME: rewrite with rotate if compiler doesn't do it. + assert(!bits.getIsDeiniting()); + if (bits.hasSideTable()) + abort(); + return (bits.getStrongExtraRefCount() == 0 || bits.getIsPinned()); + + // FIXME: check if generated code is efficient. + // We can't use the old rotate implementation below because + // the strong refcount field is now biased. // Rotating right by one sets the sign bit to the pinned bit. After // rotation, the dealloc flag is the least significant bit followed by the @@ -431,227 +526,71 @@ class InlineRefCounts { } private: - template - bool doDecrementShouldDeinit() { - auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; - - bool performDeinit; - do { - newbits = oldbits; - - // ClearPinnedFlag assumes the pinned flag is already set. - assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && - "unpinning reference that was not pinned"); - assert(oldbits.getStrongRefCount() >= 1 && - "releasing reference with a refcount of zero"); - - // FIXME: hand optimize these bit operations if necessary - if (ClearPinnedFlag) { - newbits.setIsPinned(false); - } - newbits.decrementStrongRefCount(); - performDeinit = (newbits.getStrongRefCount() == 0 && - !newbits.getIsDeiniting()); - if (performDeinit) { - newbits.setIsDeiniting(true); - } - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); - if (performDeinit) { - std::atomic_thread_fence(acquire); - } - return performDeinit; - - /* Old implementation for optimization reference: - - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - constexpr uint32_t quantum = - (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t newval = __atomic_sub_fetch(&strongRefCount, quantum, __ATOMIC_RELEASE); - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + quantum >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeinit() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - */ + // Second slow path of doDecrementShouldDeinit, where the + // object may have a side table entry. + template + bool doDecrementShouldDeinitSlow2(InlineRefCountBits oldbits) { + abort(); } - - template - bool doDecrementShouldDeinitNonAtomic() { - auto oldbits = refCounts.load(relaxed); + + // First slow path of doDecrementShouldDeinit, where the object + // may need to be deinited. + // Side table paths are handled in doDecrementShouldDeinitSlow2(). + // FIXME: can we do the non-atomic thing here? + template + bool doDecrementShouldDeinitSlow1(InlineRefCountBits oldbits, + uint32_t dec = 1) { InlineRefCountBits newbits; - - // FIXME: can probably do this without CAS once zeroing weak references - // are pushed to the side table. Then the presence of inline RC will - // prove that the object is not already weakly-referenced so there can't - // be a race vs a weak load; and presumably no other thread can see - // the object so there can't be a race vs a weak store. bool performDeinit; do { newbits = oldbits; - - // ClearPinnedFlag assumes the pinned flag is already set. - assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && - "unpinning reference that was not pinned"); - assert(oldbits.getStrongRefCount() >= 1 && - "releasing reference with a refcount of zero"); - - // FIXME: hand optimize these bit operations if necessary - if (ClearPinnedFlag) { - newbits.setIsPinned(false); + + bool fast = newbits.decrementStrongExtraRefCount(dec); + if (fast) { + // Decrement completed normally. New refcount is not zero. + performDeinit = false; } - newbits.decrementStrongRefCount(); - performDeinit = (newbits.getStrongRefCount() == 0 && - !newbits.getIsDeiniting()); - if (performDeinit) { + else if (oldbits.hasSideTable()) { + // Decrement failed because we're on some other slow path. + return doDecrementShouldDeinitSlow2(oldbits); + } else { + // Decrement underflowed. Begin deinit. + newbits = oldbits; // Undo failed decrement of newbits. + performDeinit = true; + newbits.setStrongExtraRefCount(0); newbits.setIsDeiniting(true); + if (clearPinnedFlag) + newbits.setIsPinned(false); } } while (! refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); - if (performDeinit) { + if (performDeinit) std::atomic_thread_fence(acquire); - } - return performDeinit; - - /* Old implementation for optimization reference: - - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - constexpr uint32_t quantum = - (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t val = __atomic_load_n(&strongRefCount, __ATOMIC_RELAXED); - val -= quantum; - __atomic_store_n(&strongRefCount, val, __ATOMIC_RELEASE); - uint32_t newval = strongRefCount; - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + quantum >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeinit() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - */ + return performDeinit; } - - template - bool doDecrementShouldDeinitN(uint32_t n) { + + template + bool doDecrementShouldDeinit(uint32_t n = 1) { auto oldbits = refCounts.load(relaxed); InlineRefCountBits newbits; - bool performDeinit; do { newbits = oldbits; + bool fast = newbits.decrementStrongExtraRefCount(n); + if (!fast) + return + doDecrementShouldDeinitSlow1(oldbits, n); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); - // ClearPinnedFlag assumes the pinned flag is already set. - assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && - "unpinning reference that was not pinned"); - assert(oldbits.getStrongRefCount() >= n && - "releasing reference with a refcount of zero"); - - // FIXME: hand optimize these bit operations if necessary - if (ClearPinnedFlag) { - newbits.setIsPinned(false); - } - newbits.decrementStrongRefCount(n); - performDeinit = (newbits.getStrongRefCount() == 0 && - !newbits.getIsDeiniting()); - if (performDeinit) { - newbits.setIsDeiniting(true); - } - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); - if (performDeinit) { - std::atomic_thread_fence(acquire); - } - return performDeinit; - - - /* Old implementation for optimization reference: - - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t newval = __atomic_sub_fetch(&strongRefCount, delta, __ATOMIC_RELEASE); - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + delta >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeinit() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - */ + return false; // don't deinit } - template - bool doDecrementShouldDeinitNNonAtomic(uint32_t n) { + template + bool doDecrementShouldDeinitNonAtomic(uint32_t n = 1) { auto oldbits = refCounts.load(relaxed); InlineRefCountBits newbits; @@ -661,71 +600,16 @@ class InlineRefCounts { // be a race vs a weak load; and presumably no other thread can see // the object so there can't be a race vs a weak store. - bool performDeinit; do { newbits = oldbits; - - // ClearPinnedFlag assumes the pinned flag is already set. - assert(!(ClearPinnedFlag && !oldbits.getIsPinned()) && - "unpinning reference that was not pinned"); - assert(oldbits.getStrongRefCount() >= n && - "releasing reference with a refcount of zero"); - - // FIXME: hand optimize these bit operations if necessary - if (ClearPinnedFlag) { - newbits.setIsPinned(false); - } - newbits.decrementStrongRefCount(n); - performDeinit = (newbits.getStrongRefCount() == 0 && - !newbits.getIsDeiniting()); - if (performDeinit) { - newbits.setIsDeiniting(true); - } + bool fast = newbits.decrementStrongExtraRefCount(n); + if (!fast) + return + doDecrementShouldDeinitSlow1(oldbits); } while (! refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); - if (performDeinit) { - std::atomic_thread_fence(acquire); - } - return performDeinit; - - /* Old implementation for optimization reference: - - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t val = __atomic_load_n(&strongRefCount, __ATOMIC_RELAXED); - val -= delta; - __atomic_store_n(&strongRefCount, val, __ATOMIC_RELEASE); - uint32_t newval = val; - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + delta >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeinit() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&strongRefCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - */ + return false; // don't deinit } diff --git a/stdlib/public/runtime/Errors.cpp b/stdlib/public/runtime/Errors.cpp index d989d2e4f0477..a9a9e21c2344a 100644 --- a/stdlib/public/runtime/Errors.cpp +++ b/stdlib/public/runtime/Errors.cpp @@ -271,3 +271,11 @@ swift_deletedMethodError() { swift::fatalError(/* flags = */ 0, "fatal error: call of deleted method\n"); } + + +// Crash due to a retain count overflow. +// FIXME: can't pass the object's address from InlineRefCounts without hacks +void swift::swift_abortRetainOverflow() { + swift::fatalError(FatalErrorFlags::ReportBacktrace, + "fatal error: object was retained too many times"); +} diff --git a/unittests/runtime/Refcounting.cpp b/unittests/runtime/Refcounting.cpp index dbea5c7d0f703..51b22d9bd7cbf 100644 --- a/unittests/runtime/Refcounting.cpp +++ b/unittests/runtime/Refcounting.cpp @@ -246,3 +246,78 @@ TEST(RefcountingTest, nonatomic_unknown_retain_release_n) { EXPECT_EQ(1u, swift_retainCount(object)); } + +//////////////////////////////////////////// +// Max retain count and overflow checking // +//////////////////////////////////////////// + + +template +static void retainALot(TestObject *object, size_t &deallocated, + uint64_t count) { + for (uint64_t i = 0; i < count; i++) { + if (atomic) swift_retain(object); + else swift_nonatomic_retain(object); + EXPECT_EQ(0u, deallocated); + } +} + +template +static void releaseALot(TestObject *object, size_t &deallocated, + uint64_t count) { + for (uint64_t i = 0; i < count; i++) { + if (atomic) swift_release(object); + else swift_nonatomic_release(object); + EXPECT_EQ(0u, deallocated); + } +} + +// 32-3 bits of extra retain count, plus 1 for the implicit retain +const uint64_t maxRC = 1ULL << (32 - 3); + +TEST(RefcountingTest, retain_max) { + size_t deallocated = 0; + auto object = allocTestObject(&deallocated, 1); + + // RC is 1. + // Retain to maxRC, release back to 1, then release and verify deallocation. + retainALot(object, deallocated, maxRC - 1); + releaseALot(object, deallocated, maxRC - 1); + EXPECT_EQ(0u, deallocated); + swift_release(object); + EXPECT_EQ(1u, deallocated); +} + +TEST(RefcountingTest, retain_overflow_DeathTest) { + size_t deallocated = 0; + auto object = allocTestObject(&deallocated, 1); + + // RC is 1. Retain to maxRC, then retain again and verify overflow error. + retainALot(object, deallocated, maxRC - 1); + EXPECT_EQ(0u, deallocated); + ASSERT_DEATH(swift_retain(object), "swift_abortRetainOverflow"); +} + +TEST(RefcountingTest, nonatomic_retain_max) { + size_t deallocated = 0; + auto object = allocTestObject(&deallocated, 1); + + // RC is 1. + // Retain to maxRC, release back to 1, then release and verify deallocation. + retainALot(object, deallocated, maxRC - 1); + releaseALot(object, deallocated, maxRC - 1); + EXPECT_EQ(0u, deallocated); + swift_nonatomic_release(object); + EXPECT_EQ(1u, deallocated); +} + +TEST(RefcountingTest, nonatomic_retain_overflow_DeathTest) { + size_t deallocated = 0; + auto object = allocTestObject(&deallocated, 1); + + // RC is 1. Retain to maxRC, then retain again and verify overflow error. + retainALot(object, deallocated, maxRC - 1); + EXPECT_EQ(0u, deallocated); + ASSERT_DEATH(swift_nonatomic_retain(object), "swift_abortRetainOverflow"); +} + From 46228cf6afeec7a3d52f3efb695f0bb2f813f559 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 8 Sep 2016 18:34:00 -0700 Subject: [PATCH 04/38] WIP: Side table. Separate weak refcount. --- include/swift/Runtime/Errors.h | 24 + include/swift/Runtime/HeapObject.h | 308 +++++------ include/swift/Runtime/Metadata.h | 2 +- stdlib/public/SwiftShims/RefCount.h | 733 +++++++++++++++++--------- stdlib/public/runtime/CMakeLists.txt | 1 + stdlib/public/runtime/Errors.cpp | 1 + stdlib/public/runtime/HeapObject.cpp | 230 +++----- stdlib/public/runtime/MetadataImpl.h | 3 + stdlib/public/runtime/Private.h | 17 + stdlib/public/runtime/RefCount.cpp | 127 +++++ stdlib/public/runtime/Reflection.mm | 1 + stdlib/public/runtime/SwiftObject.mm | 172 ++---- stdlib/public/runtime/WeakReference.h | 348 ++++++++++++ unittests/runtime/weak.mm | 7 + 14 files changed, 1250 insertions(+), 724 deletions(-) create mode 100644 include/swift/Runtime/Errors.h create mode 100644 stdlib/public/runtime/RefCount.cpp create mode 100644 stdlib/public/runtime/WeakReference.h diff --git a/include/swift/Runtime/Errors.h b/include/swift/Runtime/Errors.h new file mode 100644 index 0000000000000..fcce4736e1ffe --- /dev/null +++ b/include/swift/Runtime/Errors.h @@ -0,0 +1,24 @@ +//===--- Errors.n - Error reporting utilities -------------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Utilities for reporting errors to stderr, system console, and crash logs. +// +//===----------------------------------------------------------------------===// + +#include "swift/Basic/LLVM.h" + +namespace swift { + +LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE +void swift_abortRetainOverflow(); + +} diff --git a/include/swift/Runtime/HeapObject.h b/include/swift/Runtime/HeapObject.h index 30451881d11bb..4969a5aa9cd53 100644 --- a/include/swift/Runtime/HeapObject.h +++ b/include/swift/Runtime/HeapObject.h @@ -233,18 +233,6 @@ extern "C" void (*SWIFT_CC(RegisterPreservingCC) _swift_nonatomic_retain_n)(HeapObject *object, uint32_t n); -static inline void _swift_retain_inlined(HeapObject *object) { - if (object) { - object->refCounts.increment(); - } -} - -static inline void _swift_nonatomic_retain_inlined(HeapObject *object) { - if (object) { - object->refCounts.incrementNonAtomic(); - } -} - /// Atomically increments the reference count of an object, unless it has /// already been destroyed. Returns nil if the object is dead. SWIFT_RT_ENTRY_VISIBILITY @@ -532,22 +520,22 @@ struct UnownedReference { HeapObject *Value; }; -/// Increment the weak/unowned retain count. +/// Increment the unowned retain count. SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift_unownedRetain(HeapObject *value) SWIFT_CC(RegisterPreservingCC); -/// Decrement the weak/unowned retain count. +/// Decrement the unowned retain count. SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift_unownedRelease(HeapObject *value) SWIFT_CC(RegisterPreservingCC); -/// Increment the weak/unowned retain count by n. +/// Increment the unowned retain count by n. SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift_unownedRetain_n(HeapObject *value, int n) SWIFT_CC(RegisterPreservingCC); -/// Decrement the weak/unowned retain count by n. +/// Decrement the unowned retain count by n. SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift_unownedRelease_n(HeapObject *value, int n) SWIFT_CC(RegisterPreservingCC); @@ -560,7 +548,7 @@ extern "C" void swift_unownedRetainStrong(HeapObject *value) /// Increment the strong retain count of an object which may have been /// deallocated, aborting if it has been deallocated, and decrement its -/// weak/unowned reference count. +/// unowned reference count. SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift_unownedRetainStrongAndRelease(HeapObject *value) SWIFT_CC(RegisterPreservingCC); @@ -630,20 +618,13 @@ static inline void swift_unownedTakeAssign(UnownedReference *dest, swift_unownedRelease(oldValue); } + /*****************************************************************************/ /****************************** WEAK REFERENCES ******************************/ /*****************************************************************************/ -/// A weak reference value object. This is ABI. -struct WeakReference { - uintptr_t Value; -}; - -/// Return true if this is a native weak reference -/// -/// \param ref - never null -/// \return true if ref is a native weak reference -bool isNativeSwiftWeakReference(WeakReference *ref); +// Defined in Runtime/WeakReference.h +class WeakReference; /// Initialize a weak reference. /// @@ -710,6 +691,121 @@ extern "C" void swift_weakCopyAssign(WeakReference *dest, WeakReference *src); SWIFT_RUNTIME_EXPORT extern "C" void swift_weakTakeAssign(WeakReference *dest, WeakReference *src); + +/*****************************************************************************/ +/************************** UNKNOWN WEAK REFERENCES **************************/ +/*****************************************************************************/ + +#if SWIFT_OBJC_INTEROP + +/// Initialize a weak reference. +/// +/// \param ref - never null +/// \param value - not necessarily a native Swift object; can be null +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakInit(WeakReference *ref, void *value); + +/// Assign a new value to a weak reference. +/// +/// \param ref - never null +/// \param value - not necessarily a native Swift object; can be null +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakAssign(WeakReference *ref, void *value); + +/// Load a value from a weak reference, much like swift_weakLoadStrong +/// but without requiring the variable to refer to a native Swift object. +/// +/// \param ref - never null +/// \return can be null +SWIFT_RUNTIME_EXPORT +extern "C" void *swift_unknownWeakLoadStrong(WeakReference *ref); + +/// Load a value from a weak reference as if by +/// swift_unknownWeakLoadStrong, but leaving the reference in an +/// uninitialized state. +/// +/// \param ref - never null +/// \return can be null +SWIFT_RUNTIME_EXPORT +extern "C" void *swift_unknownWeakTakeStrong(WeakReference *ref); + +/// Destroy a weak reference variable that might not refer to a native +/// Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakDestroy(WeakReference *object); + +/// Copy-initialize a weak reference variable from one that might not +/// refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakCopyInit(WeakReference *dest, + WeakReference *src); + +/// Take-initialize a weak reference variable from one that might not +/// refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakTakeInit(WeakReference *dest, + WeakReference *src); + +/// Copy-assign a weak reference variable from another when either +/// or both variables might not refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakCopyAssign(WeakReference *dest, + WeakReference *src); + +/// Take-assign a weak reference variable from another when either +/// or both variables might not refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakTakeAssign(WeakReference *dest, + WeakReference *src); + +// SWIFT_OBJC_INTEROP +#else +// not SWIFT_OBJC_INTEROP + +static inline void swift_unknownWeakInit(WeakReference *ref, void *value) { + swift_weakInit(ref, static_cast(value)); +} + +static inline void swift_unknownWeakAssign(WeakReference *ref, void *value) { + swift_weakAssign(ref, static_cast(value)); +} + +static inline void *swift_unknownWeakLoadStrong(WeakReference *ref) { + return static_cast(swift_weakLoadStrong(ref)); +} + +static inline void *swift_unknownWeakTakeStrong(WeakReference *ref) { + return static_cast(swift_weakTakeStrong(ref)); +} + +static inline void swift_unknownWeakDestroy(WeakReference *object) { + swift_weakDestroy(object); +} + +static inline void swift_unknownWeakCopyInit(WeakReference *dest, + WeakReference *src) { + swift_weakCopyInit(dest, src); +} + +static inline void swift_unknownWeakTakeInit(WeakReference *dest, + WeakReference *src) { + swift_weakTakeInit(dest, src); +} + +static inline void swift_unknownWeakCopyAssign(WeakReference *dest, + WeakReference *src) { + swift_weakCopyAssign(dest, src); +} + +static inline void swift_unknownWeakTakeAssign(WeakReference *dest, + WeakReference *src) { + swift_weakTakeAssign(dest, src); +} + +// not SWIFT_OBJC_INTEROP +#endif + + /*****************************************************************************/ /************************* OTHER REFERENCE-COUNTING **************************/ /*****************************************************************************/ @@ -849,164 +945,6 @@ static inline void swift_nonatomic_unknownRelease_n(void *value, int n) #endif /* SWIFT_OBJC_INTEROP */ -/*****************************************************************************/ -/************************** UNKNOWN WEAK REFERENCES **************************/ -/*****************************************************************************/ - -#if SWIFT_OBJC_INTEROP - -/// Initialize a weak reference. -/// -/// \param ref - never null -/// \param value - not necessarily a native Swift object; can be null -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakInit(WeakReference *ref, void *value); - -#else - -static inline void swift_unknownWeakInit(WeakReference *ref, void *value) { - swift_weakInit(ref, static_cast(value)); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Assign a new value to a weak reference. -/// -/// \param ref - never null -/// \param value - not necessarily a native Swift object; can be null -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakAssign(WeakReference *ref, void *value); - -#else - -static inline void swift_unknownWeakAssign(WeakReference *ref, void *value) { - swift_weakAssign(ref, static_cast(value)); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Load a value from a weak reference, much like swift_weakLoadStrong -/// but without requiring the variable to refer to a native Swift object. -/// -/// \param ref - never null -/// \return can be null -SWIFT_RUNTIME_EXPORT -extern "C" void *swift_unknownWeakLoadStrong(WeakReference *ref); - -#else - -static inline void *swift_unknownWeakLoadStrong(WeakReference *ref) { - return static_cast(swift_weakLoadStrong(ref)); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Load a value from a weak reference as if by -/// swift_unknownWeakLoadStrong, but leaving the reference in an -/// uninitialized state. -/// -/// \param ref - never null -/// \return can be null -SWIFT_RUNTIME_EXPORT -extern "C" void *swift_unknownWeakTakeStrong(WeakReference *ref); - -#else - -static inline void *swift_unknownWeakTakeStrong(WeakReference *ref) { - return static_cast(swift_weakTakeStrong(ref)); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Destroy a weak reference variable that might not refer to a native -/// Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakDestroy(WeakReference *object); - -#else - -static inline void swift_unknownWeakDestroy(WeakReference *object) { - swift_weakDestroy(object); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Copy-initialize a weak reference variable from one that might not -/// refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakCopyInit(WeakReference *dest, - WeakReference *src); - -#else - -static inline void swift_unknownWeakCopyInit(WeakReference *dest, - WeakReference *src) { - swift_weakCopyInit(dest, src); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Take-initialize a weak reference variable from one that might not -/// refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakTakeInit(WeakReference *dest, - WeakReference *src); - -#else - -static inline void swift_unknownWeakTakeInit(WeakReference *dest, - WeakReference *src) { - swift_weakTakeInit(dest, src); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Copy-assign a weak reference variable from another when either -/// or both variables might not refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakCopyAssign(WeakReference *dest, - WeakReference *src); - -#else - -static inline void swift_unknownWeakCopyAssign(WeakReference *dest, - WeakReference *src) { - swift_weakCopyAssign(dest, src); -} - -#endif /* SWIFT_OBJC_INTEROP */ - -#if SWIFT_OBJC_INTEROP - -/// Take-assign a weak reference variable from another when either -/// or both variables might not refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakTakeAssign(WeakReference *dest, - WeakReference *src); - -#else - -static inline void swift_unknownWeakTakeAssign(WeakReference *dest, - WeakReference *src) { - swift_weakTakeAssign(dest, src); -} - -#endif /* SWIFT_OBJC_INTEROP */ - /*****************************************************************************/ /************************ UNKNOWN UNOWNED REFERENCES *************************/ /*****************************************************************************/ diff --git a/include/swift/Runtime/Metadata.h b/include/swift/Runtime/Metadata.h index 34a846cc42e16..e7a1663ed2a2e 100644 --- a/include/swift/Runtime/Metadata.h +++ b/include/swift/Runtime/Metadata.h @@ -134,7 +134,7 @@ using TargetFarRelativeIndirectablePointer = typename Runtime::template FarRelativeIndirectablePointer; struct HeapObject; -struct WeakReference; +class WeakReference; template struct TargetMetadata; using Metadata = TargetMetadata; diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index d102b1404887c..098bbe383110e 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -34,20 +34,53 @@ typedef struct { #include "llvm/Support/Compiler.h" #include "swift/Basic/type_traits.h" +#include "swift/Runtime/Config.h" +namespace swift { + struct HeapObject; + class HeapObjectSideTableEntry; +} + +// FIXME: HACK copied from HeapObject.cpp +extern "C" LLVM_LIBRARY_VISIBILITY void +_swift_release_dealloc(swift::HeapObject *object) + SWIFT_CC(RegisterPreservingCC_IMPL) + __attribute__((__noinline__, __used__)); + +namespace swift { + +// FIXME: There are lots of asserts here which hurts Assert performance a lot. + +// FIXME: many `relaxed` in this file should be `consume`, +// but (1) the compiler doesn't support `consume` directly, +// and (2) the compiler promotes `consume` to `acquire` instead which +// is overkill on our CPUs. But this might leave us vulnerable to +// compiler optimizations that `relaxed` allows but `consume` ought not. #define relaxed std::memory_order_relaxed #define acquire std::memory_order_acquire #define release std::memory_order_release +#define consume std::memory_order_consume + + +// RefCountIsInline: refcount stored in an object +// RefCountNotInline: refcount stored in an object's side table entry +enum RefCountInlinedness { RefCountNotInline = false, RefCountIsInline = true }; + +enum ClearPinnedFlag { DontClearPinnedFlag = false, DoClearPinnedFlag = true }; + +enum PerformDeinit { DontPerformDeinit = false, DoPerformDeinit = true }; -// Error reporting functions -namespace swift { - LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE - void swift_abortRetainOverflow(); -}; // Basic encoding of refcount and flag data into the object's header. // FIXME: Specialize this for a 32-bit field on 32-bit architectures. -class InlineRefCountBits { +template +class RefCountBitsT { + + friend class RefCountBitsT; + friend class RefCountBitsT; + + static const RefCountInlinedness Inlinedness = refcountIsInline; + uint64_t bits; // Layout of bits. @@ -57,6 +90,7 @@ class InlineRefCountBits { # define ShiftAfterField(name) (name##Shift + name##BitCount) enum : uint64_t { + // FIXME: isFreeing bit UnownedRefCountShift = 0, UnownedRefCountBitCount = 32, UnownedRefCountMask = MaskForField(UnownedRefCount), @@ -77,22 +111,26 @@ class InlineRefCountBits { UseSlowRCBitCount = 1, UseSlowRCMask = MaskForField(UseSlowRC), - // FIXME: 63 bits but MSB must always be zero. Handle that differently. SideTableShift = 0, - SideTableBitCount = 63, - SideTableMask = MaskForField(SideTable) + SideTableBitCount = 62, + SideTableMask = MaskForField(SideTable), + + SideTableMarkShift = SideTableBitCount, + SideTableMarkBitCount = 1, + SideTableMarkMask = MaskForField(SideTableMark) }; static_assert(StrongExtraRefCountShift == IsDeinitingShift + 1, "IsDeiniting must be LSB-wards of StrongExtraRefCount"); static_assert(UseSlowRCShift + UseSlowRCBitCount == sizeof(bits)*8, "UseSlowRC must be MSB"); - static_assert(SideTableBitCount + UseSlowRCBitCount == sizeof(bits)*8, - "wrong bit count for InlineRefCountBits side table version"); + static_assert(SideTableBitCount + SideTableMarkBitCount + + UseSlowRCBitCount == sizeof(bits)*8, + "wrong bit count for RefCountBits side table encoding"); static_assert(UnownedRefCountBitCount + IsPinnedBitCount + IsDeinitingBitCount + StrongExtraRefCountBitCount + UseSlowRCBitCount == sizeof(bits)*8, - "wrong bit count for InlineRefCountBits refcount version"); + "wrong bit count for RefCountBits refcount encoding"); # undef MaskForField # undef ShiftAfterField @@ -101,7 +139,7 @@ class InlineRefCountBits { # define SetField(name, val) \ bits = (bits & ~name##Mask) | (((uint64_t(val) << name##Shift) & name##Mask)) - // InlineRefCountBits uses always_inline everywhere + // RefCountBits uses always_inline everywhere // to improve performance of debug builds. private: @@ -115,29 +153,82 @@ class InlineRefCountBits { SetField(UseSlowRC, value); } + + // Returns true if the decrement is a fast-path result. + // Returns false if the decrement should fall back to some slow path + // (for example, because UseSlowRC is set + // or because the refcount is now zero and should deinit). + template + LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + bool doDecrementStrongExtraRefCount(uint32_t dec) { + if (!hasSideTable()) { + // Can't check these assertions with side table present. + + // clearPinnedFlag assumes the flag is already set. + if (clearPinnedFlag) + assert(getIsPinned() && "unpinning reference that was not pinned"); + + if (getIsDeiniting()) + assert(getStrongExtraRefCount() >= dec && + "releasing reference whose refcount is already zero"); + else + assert(getStrongExtraRefCount() + 1 >= dec && + "releasing reference whose refcount is already zero"); + } + + uint64_t unpin = clearPinnedFlag ? (uint64_t(1) << IsPinnedShift) : 0; + // This deliberately underflows by borrowing from the UseSlowRC field. + bits -= unpin + (uint64_t(dec) << StrongExtraRefCountShift); + return (int64_t(bits) >= 0); + } + public: LLVM_ATTRIBUTE_ALWAYS_INLINE - InlineRefCountBits() = default; + RefCountBitsT() = default; LLVM_ATTRIBUTE_ALWAYS_INLINE - constexpr InlineRefCountBits(uint32_t strongExtraCount, uint32_t unownedCount) + constexpr + RefCountBitsT(uint32_t strongExtraCount, uint32_t unownedCount) : bits((uint64_t(strongExtraCount) << StrongExtraRefCountShift) | (uint64_t(unownedCount) << UnownedRefCountShift)) { } + LLVM_ATTRIBUTE_ALWAYS_INLINE + RefCountBitsT(HeapObjectSideTableEntry* side) + : bits((reinterpret_cast(side) >> 3) | + (1ULL << UseSlowRCShift) | + (1ULL << SideTableMarkShift)) + { + assert(refcountIsInline); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + RefCountBitsT(RefCountBitsT newbits) : bits(newbits.bits) { } + LLVM_ATTRIBUTE_ALWAYS_INLINE bool hasSideTable() const { // FIXME: change this when introducing immutable RC objects - return getUseSlowRC(); + bool hasSide = getUseSlowRC(); + + // Side table refcount must not point to another side table. + assert((refcountIsInline || !hasSide) && + "side table refcount must not have a side table entry of its own"); + + return hasSide; } LLVM_ATTRIBUTE_ALWAYS_INLINE - uintptr_t getSideTable() const { + HeapObjectSideTableEntry *getSideTable() const { assert(hasSideTable()); + // FIXME: overkill barrier? Otherwise technically need + // a consume re-load of the bits before dereferencing. + std::atomic_thread_fence(std::memory_order_acquire); + // Stored value is a shifted pointer. // FIXME: Don't hard-code this shift amount? - return uintptr_t(GetField(SideTable)) << 2; + return reinterpret_cast + (uintptr_t(GetField(SideTable)) << 3); } LLVM_ATTRIBUTE_ALWAYS_INLINE @@ -172,13 +263,15 @@ class InlineRefCountBits { } LLVM_ATTRIBUTE_ALWAYS_INLINE - void setSideTable(uintptr_t value) { + void setSideTable(HeapObjectSideTableEntry *side) { assert(hasSideTable()); // Stored value is a shifted pointer. // FIXME: Don't hard-code this shift amount? - uintptr_t storedValue = value >> 2; - assert(storedValue << 2 == value); + uintptr_t value = reinterpret_cast(side); + uintptr_t storedValue = value >> 3; + assert(storedValue << 3 == value); SetField(SideTable, storedValue); + SetField(SideTableMark, 1); } LLVM_ATTRIBUTE_ALWAYS_INLINE @@ -210,43 +303,29 @@ class InlineRefCountBits { // Returns false if the increment should fall back to some slow path // (for example, because UseSlowRC is set or because the refcount overflowed). LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT - bool incrementStrongExtraRefCount(uint32_t inc = 1) { + bool incrementStrongExtraRefCount(uint32_t inc) { // This deliberately overflows into the UseSlowRC field. bits += uint64_t(inc) << StrongExtraRefCountShift; return (int64_t(bits) >= 0); } - LLVM_ATTRIBUTE_ALWAYS_INLINE - void incrementUnownedRefCount(uint32_t inc = 1) { - setUnownedRefCount(getUnownedRefCount() + inc); - } - - // Returns true if the decrement is a fast-path result. - // Returns false if the decrement should fall back to some slow path - // (for example, because UseSlowRC is set - // or because the refcount is now zero and should deinit). - template + // FIXME: I don't understand why I can't make clearPinned a template argument + // (compiler balks at calls from class RefCounts that way) LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT - bool decrementStrongExtraRefCount(uint32_t dec = 1) { - // ClearPinnedFlag assumes the flag is already set. - if (ClearPinnedFlag) - assert(getIsPinned() && "unpinning reference that was not pinned"); - - if (getIsDeiniting()) - assert(getStrongExtraRefCount() >= dec && - "releasing reference whose refcount is already zero"); - else - assert(getStrongExtraRefCount() + 1 >= dec && - "releasing reference whose refcount is already zero"); + bool decrementStrongExtraRefCount(uint32_t dec, bool clearPinned = false) { + if (clearPinned) + return doDecrementStrongExtraRefCount(dec); + else + return doDecrementStrongExtraRefCount(dec); + } - uint64_t unpin = ClearPinnedFlag ? (uint64_t(1) << IsPinnedShift) : 0; - // This deliberately underflows by borrowing from the UseSlowRC field. - bits -= unpin + (uint64_t(dec) << StrongExtraRefCountShift); - return (int64_t(bits) >= 0); + LLVM_ATTRIBUTE_ALWAYS_INLINE + void incrementUnownedRefCount(uint32_t inc) { + setUnownedRefCount(getUnownedRefCount() + inc); } LLVM_ATTRIBUTE_ALWAYS_INLINE - void decrementUnownedRefCount(uint32_t dec = 1) { + void decrementUnownedRefCount(uint32_t dec) { setUnownedRefCount(getUnownedRefCount() - dec); } @@ -254,6 +333,51 @@ class InlineRefCountBits { # undef SetField }; +typedef RefCountBitsT InlineRefCountBits; + +class SideTableRefCountBits : public RefCountBitsT +{ + uint32_t weakBits; + + public: + LLVM_ATTRIBUTE_ALWAYS_INLINE + SideTableRefCountBits() = default; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + constexpr + SideTableRefCountBits(uint32_t strongExtraCount, uint32_t unownedCount) + : RefCountBitsT(strongExtraCount, unownedCount) + , weakBits(0) + { } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + SideTableRefCountBits(HeapObjectSideTableEntry* side) = delete; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + SideTableRefCountBits(InlineRefCountBits newbits) + : RefCountBitsT(newbits), weakBits(0) + { } + + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void incrementWeakRefCount() { + weakBits++; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool decrementWeakRefCount() { + assert(weakBits > 0); + weakBits--; + return weakBits == 0; + } + + // Side table ref count never has a side table of its own. + bool hasSideTable() { + return false; + } +}; + + // Barriers // // Strong refcount increment is unordered with respect to other memory locations @@ -264,113 +388,73 @@ class InlineRefCountBits { // -dealloc code. This ensures that the deinit code sees all modifications // of the object's contents that were made before the object was released. -class InlineRefCounts { - std::atomic refCounts; - - // Template parameters. - enum ClearPinnedFlag { DontClearPinnedFlag = false, - DoClearPinnedFlag = true }; - enum Atomicity { NonAtomic = false, Atomic = true }; +template +class RefCounts { + std::atomic refCounts; // Out-of-line slow paths. LLVM_ATTRIBUTE_NOINLINE - void incrementSlow(InlineRefCountBits oldbits, uint32_t n = 1) { - if (oldbits.hasSideTable()) { - // Out-of-line slow path. - abort(); - } else { - swift::swift_abortRetainOverflow(); - } - } + void incrementSlow(RefCountBits oldbits, uint32_t inc); LLVM_ATTRIBUTE_NOINLINE - void incrementNonAtomicSlow(InlineRefCountBits oldbits, uint32_t n = 1) { - if (oldbits.hasSideTable()) { - // Out-of-line slow path. - abort(); - } else { - swift::swift_abortRetainOverflow(); - } - } + void incrementNonAtomicSlow(RefCountBits oldbits, uint32_t inc); LLVM_ATTRIBUTE_NOINLINE - bool tryIncrementAndPinSlow() { - abort(); - } + bool tryIncrementAndPinSlow(); LLVM_ATTRIBUTE_NOINLINE - bool tryIncrementAndPinNonAtomicSlow() { - abort(); - } + bool tryIncrementAndPinNonAtomicSlow(); LLVM_ATTRIBUTE_NOINLINE - bool tryIncrementSlow() { - abort(); - } + bool tryIncrementSlow(RefCountBits oldbits); public: enum Initialized_t { Initialized }; - // InlineRefCounts must be trivially constructible to avoid ObjC++ - // destruction overhead at runtime. Use InlineRefCounts(Initialized) + // RefCounts must be trivially constructible to avoid ObjC++ + // destruction overhead at runtime. Use RefCounts(Initialized) // to produce an initialized instance. - InlineRefCounts() = default; + RefCounts() = default; // Refcount of a new object is 1. - constexpr InlineRefCounts(Initialized_t) - : refCounts(InlineRefCountBits(0, 1)) { } + constexpr RefCounts(Initialized_t) + : refCounts(RefCountBits(0, 1)) { } void init() { - refCounts.store(InlineRefCountBits(0, 1), relaxed); + refCounts.store(RefCountBits(0, 1), relaxed); } /// Initialize for a stack promoted object. This prevents that the final /// release frees the memory of the object. void initForNotFreeing() { - refCounts.store(InlineRefCountBits(0, 2), relaxed); - } - - - // Increment the reference count. - void increment() { - auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; - do { - newbits = oldbits; - bool fast = newbits.incrementStrongExtraRefCount(); - if (!fast) - return incrementSlow(oldbits); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + refCounts.store(RefCountBits(0, 2), relaxed); } - void incrementNonAtomic() { - auto oldbits = refCounts.load(relaxed); - auto newbits = oldbits; - bool fast = newbits.incrementStrongExtraRefCount(); - if (!fast) - return incrementNonAtomicSlow(oldbits); - refCounts.store(newbits, relaxed); + // Initialize from another refcount bits. + // Only inline -> out-of-line is allowed (used for new side table entries). + void init(InlineRefCountBits newBits) { + refCounts.store(newBits, relaxed); } - // Increment the reference count by n. - void increment(uint32_t n) { + // Increment the reference count. + void increment(uint32_t inc = 1) { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; do { newbits = oldbits; - bool fast = newbits.incrementStrongExtraRefCount(n); + bool fast = newbits.incrementStrongExtraRefCount(inc); if (!fast) - return incrementSlow(oldbits, n); + return incrementSlow(oldbits, inc); } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } - void incrementNonAtomic(uint32_t n) { + void incrementNonAtomic(uint32_t inc = 1) { auto oldbits = refCounts.load(relaxed); auto newbits = oldbits; - bool fast = newbits.incrementStrongExtraRefCount(n); + bool fast = newbits.incrementStrongExtraRefCount(inc); if (!fast) - return incrementNonAtomicSlow(oldbits, n); + return incrementNonAtomicSlow(oldbits, inc); refCounts.store(newbits, relaxed); } @@ -385,16 +469,16 @@ class InlineRefCounts { // Postcondition: the flag is set. bool tryIncrementAndPin() { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; do { // If the flag is already set, just fail. - if (oldbits.getIsPinned()) + if (!oldbits.hasSideTable() && oldbits.getIsPinned()) return false; // Try to simultaneously set the flag and increment the reference count. newbits = oldbits; newbits.setIsPinned(true); - bool fast = newbits.incrementStrongExtraRefCount(); + bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementAndPinSlow(); } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); @@ -405,12 +489,12 @@ class InlineRefCounts { auto bits = refCounts.load(relaxed); // If the flag is already set, just fail. - if (bits.getIsPinned()) + if (!bits.hasSideTable() && bits.getIsPinned()) return false; // Try to simultaneously set the flag and increment the reference count. bits.setIsPinned(true); - bool fast = bits.incrementStrongExtraRefCount(); + bool fast = bits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementAndPinNonAtomicSlow(); refCounts.store(bits, relaxed); @@ -420,47 +504,48 @@ class InlineRefCounts { // Increment the reference count, unless the object is deiniting. bool tryIncrement() { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; do { - if (oldbits.getIsDeiniting()) + if (!oldbits.hasSideTable() && oldbits.getIsDeiniting()) return false; newbits = oldbits; - bool fast = newbits.incrementStrongExtraRefCount(); + bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) - return tryIncrementSlow(); + return tryIncrementSlow(oldbits); } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); return true; } // Simultaneously clear the pinned flag and decrement the reference - // count. + // count. Call _swift_release_dealloc() if the reference count goes to zero. // // Precondition: the pinned flag is set. - bool decrementAndUnpinShouldDeinit() { - return doDecrementShouldDeinit(); + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementAndUnpinAndMaybeDeinit() { + doDecrement(1); } - bool decrementAndUnpinShouldDeinitNonAtomic() { - return doDecrementShouldDeinitNonAtomic(); + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementAndUnpinAndMaybeDeinitNonAtomic() { + doDecrementNonAtomic(1); } // Decrement the reference count. // Return true if the caller should now deinit the object. - bool decrementShouldDeinit() { - return doDecrementShouldDeinit(); - } - - bool decrementShouldDeinitNonAtomic() { - return doDecrementShouldDeinitNonAtomic(); + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool decrementShouldDeinit(uint32_t dec) { + return doDecrement(dec); } - bool decrementShouldDeinitN(uint32_t n) { - return doDecrementShouldDeinit(n); + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementAndMaybeDeinit(uint32_t dec) { + doDecrement(dec); } - bool decrementShouldDeinitNNonAtomic(uint32_t n) { - return doDecrementShouldDeinitNonAtomic(n); + LLVM_ATTRIBUTE_ALWAYS_INLINE + void decrementAndMaybeDeinitNonAtomic(uint32_t dec) { + doDecrementNonAtomic(dec); } // Non-atomically release the last strong reference and mark the @@ -469,8 +554,11 @@ class InlineRefCounts { // Precondition: the reference count must be 1 void decrementFromOneAndDeinitNonAtomic() { auto bits = refCounts.load(relaxed); + if (bits.hasSideTable()) + abort(); + + assert(!bits.getIsDeiniting()); assert(bits.getStrongExtraRefCount() == 0 && "Expect a refcount of 1"); - bits.setStrongExtraRefCount(0); bits.setIsDeiniting(true); refCounts.store(bits, relaxed); @@ -480,6 +568,9 @@ class InlineRefCounts { // Once deinit begins the reference count is undefined. uint32_t getCount() const { auto bits = refCounts.load(relaxed); + if (bits.hasSideTable()) + abort(); + assert(!bits.getIsDeiniting()); // FIXME: can we assert this? return bits.getStrongExtraRefCount() + 1; } @@ -487,9 +578,9 @@ class InlineRefCounts { // Once deinit begins the reference count is undefined. bool isUniquelyReferenced() const { auto bits = refCounts.load(relaxed); - assert(!bits.getIsDeiniting()); if (bits.hasSideTable()) abort(); + assert(!bits.getIsDeiniting()); return bits.getStrongExtraRefCount() == 0; } @@ -497,9 +588,9 @@ class InlineRefCounts { // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { auto bits = refCounts.load(relaxed); - assert(!bits.getIsDeiniting()); if (bits.hasSideTable()) abort(); + assert(!bits.getIsDeiniting()); return (bits.getStrongExtraRefCount() == 0 || bits.getIsPinned()); // FIXME: check if generated code is efficient. @@ -522,43 +613,56 @@ class InlineRefCounts { // Return true if the object has started deiniting. bool isDeiniting() const { auto bits = refCounts.load(relaxed); - return bits.getIsDeiniting(); + if (bits.hasSideTable()) + return bits.getSideTable()->isDeiniting(); + else + return bits.getIsDeiniting(); + } + + /// Return true if the object can be freed directly right now. + /// This is used in swift_deallocObject(). + /// Can be freed now means: + /// no side table + /// unowned reference count is 1 + /// The object is assumed to be deiniting with no strong references already. + bool canBeFreedNow() const { + auto bits = refCounts.load(relaxed); + return (!bits.hasSideTable() && bits.getIsDeiniting() && bits.getStrongExtraRefCount() == 0 && bits.getUnownedRefCount() == 1); + // FIXME: make sure no-assert build optimizes this } -private: + private: - // Second slow path of doDecrementShouldDeinit, where the + // Second slow path of doDecrement, where the // object may have a side table entry. - template - bool doDecrementShouldDeinitSlow2(InlineRefCountBits oldbits) { - abort(); - } + template + bool doDecrementSideTable(RefCountBits oldbits, uint32_t dec); - // First slow path of doDecrementShouldDeinit, where the object - // may need to be deinited. - // Side table paths are handled in doDecrementShouldDeinitSlow2(). - // FIXME: can we do the non-atomic thing here? - template - bool doDecrementShouldDeinitSlow1(InlineRefCountBits oldbits, - uint32_t dec = 1) { - InlineRefCountBits newbits; + // First slow path of doDecrement, where the object may need to be deinited. + // Side table is handled in the second slow path, doDecrementSideTable(). + template + bool doDecrementSlow(RefCountBits oldbits, uint32_t dec) { + RefCountBits newbits; - bool performDeinit; + bool deinitNow; do { newbits = oldbits; - bool fast = newbits.decrementStrongExtraRefCount(dec); + bool fast = newbits.decrementStrongExtraRefCount(dec, clearPinnedFlag); if (fast) { // Decrement completed normally. New refcount is not zero. - performDeinit = false; + deinitNow = false; } else if (oldbits.hasSideTable()) { // Decrement failed because we're on some other slow path. - return doDecrementShouldDeinitSlow2(oldbits); - } else { + return doDecrementSideTable(oldbits, dec); + } + else { // Decrement underflowed. Begin deinit. + deinitNow = true; + assert(!oldbits.getIsDeiniting()); newbits = oldbits; // Undo failed decrement of newbits. - performDeinit = true; newbits.setStrongExtraRefCount(0); newbits.setIsDeiniting(true); if (clearPinnedFlag) @@ -566,134 +670,293 @@ class InlineRefCounts { } } while (! refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); - if (performDeinit) + if (performDeinit && deinitNow) { std::atomic_thread_fence(acquire); + _swift_release_dealloc(getHeapObject()); + } - return performDeinit; + return deinitNow; } - template - bool doDecrementShouldDeinit(uint32_t n = 1) { + public: // FIXME: access control hack + + // Fast path of atomic strong decrement. + // + // Deinit is optionally handled directly instead of always deferring to + // the caller because the compiler can optimize this arrangement better. + template + bool doDecrement(uint32_t dec) { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; do { newbits = oldbits; - bool fast = newbits.decrementStrongExtraRefCount(n); + bool fast = newbits.decrementStrongExtraRefCount(dec, clearPinnedFlag); if (!fast) - return - doDecrementShouldDeinitSlow1(oldbits, n); + // Slow paths include side table; deinit; underflow + return doDecrementSlow(oldbits, dec); } while (!refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); return false; // don't deinit } + + private: - template - bool doDecrementShouldDeinitNonAtomic(uint32_t n = 1) { - auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; - - // FIXME: can probably do this without CAS once zeroing weak references - // are pushed to the side table. Then the presence of inline RC will - // prove that the object is not already weakly-referenced so there can't - // be a race vs a weak load; and presumably no other thread can see - // the object so there can't be a race vs a weak store. - - do { - newbits = oldbits; - bool fast = newbits.decrementStrongExtraRefCount(n); - if (!fast) - return - doDecrementShouldDeinitSlow1(oldbits); - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); - - return false; // don't deinit - } + // This is independently specialized below for inline and out-of-line use. + template + bool doDecrementNonAtomic(uint32_t dec); // UNOWNED public: // Increment the unowned reference count. - void incrementUnowned() { + void incrementUnowned(uint32_t inc) { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; do { - newbits = oldbits; - newbits.incrementUnownedRefCount(); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); - } + if (oldbits.hasSideTable()) + return oldbits.getSideTable()->incrementUnowned(inc); - // Increment the unowned reference count by n. - void incrementUnowned(uint32_t n) { - auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; - do { newbits = oldbits; - newbits.incrementUnownedRefCount(n); + newbits.incrementUnownedRefCount(inc); + // FIXME: overflow check? } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } // Decrement the unowned reference count. // Return true if the caller should free the object. - bool decrementUnownedShouldFree() { + bool decrementUnownedShouldFree(uint32_t dec) { auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; + RefCountBits newbits; bool performFree; do { - newbits = oldbits; + if (oldbits.hasSideTable()) + return oldbits.getSideTable()->decrementUnownedShouldFree(dec); - assert(oldbits.getUnownedRefCount() >= 1 && - "releasing reference with an unowned refcount of zero"); - - // FIXME: hand optimize these bit operations if necessary - newbits.decrementUnownedRefCount(); + newbits = oldbits; + newbits.decrementUnownedRefCount(dec); performFree = (newbits.getUnownedRefCount() == 0); + // FIXME: underflow check? } while (! refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); return performFree; } - // Decrement the unowned reference count. - // Return true if the caller should free the object. - bool decrementUnownedShouldFreeN(uint32_t n) { - auto oldbits = refCounts.load(relaxed); - InlineRefCountBits newbits; - - bool performFree; - do { - newbits = oldbits; + // Return unowned reference count. + // Note that this is not equal to the number of outstanding unowned pointers. + uint32_t getUnownedCount() const { + auto bits = refCounts.load(relaxed); + if (bits.hasSideTable()) + return bits.getSideTable()->getUnownedCount(); + else + return bits.getUnownedRefCount(); + } - assert(oldbits.getUnownedRefCount() >= n && - "releasing reference with an unowned refcount of zero"); - // FIXME: hand optimize these bit operations if necessary - newbits.decrementUnownedRefCount(n); - performFree = (newbits.getUnownedRefCount() == 0); - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); - return performFree; - } + // WEAK + + public: + // Returns the object's side table entry (creating it if necessary) with + // its weak ref count incremented. + // Returns nullptr if the object is already deiniting. + // Use this when creating a new weak reference to an object. + HeapObjectSideTableEntry* formWeakReference(); + // Increment the weak reference count. + void incrementWeak() { + // FIXME + } + bool decrementWeakShouldCleanUp() { + // FIXME + return false; + } + // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. - uint32_t getUnownedCount() const { - auto bits = refCounts.load(relaxed); - return bits.getUnownedRefCount(); - } + // FIXME: inline fast path when there are no weak references outstanding + uint32_t getWeakCount() const; + + + private: + HeapObject *getHeapObject() const; + + HeapObjectSideTableEntry* allocateSideTable(); }; +typedef RefCounts InlineRefCounts; +typedef RefCounts SideTableRefCounts; + static_assert(swift::IsTriviallyConstructible::value, "InlineRefCounts must be trivially initializable"); static_assert(std::is_trivially_destructible::value, "InlineRefCounts must be trivially destructible"); + +class HeapObjectSideTableEntry { + std::atomic object; + SideTableRefCounts refCounts; + + public: + HeapObjectSideTableEntry(HeapObject *newObject) + : object(newObject), refCounts() + { } + + HeapObject* tryRetain() { + if (refCounts.tryIncrement()) + return object.load(); // FIXME barrier + else + return nullptr; + } + + void initRefCounts(InlineRefCountBits newbits) { + refCounts.init(newbits); + } + + HeapObject *unsafeGetObject() const { + return object.load(relaxed); + } + + void incrementStrong(uint32_t inc) { + refCounts.increment(inc); + } + + template + bool decrementStrong(uint32_t dec) { + return refCounts.doDecrement(dec); + } + + bool isDeiniting() const { + return refCounts.isDeiniting(); + } + + bool tryIncrement() { + return refCounts.tryIncrement(); + } + + // UNOWNED + + void incrementUnowned(uint32_t inc) { + return refCounts.incrementUnowned(inc); + } + + bool decrementUnownedShouldFree(uint32_t dec) { + bool shouldFree = refCounts.decrementUnownedShouldFree(dec); + if (shouldFree) { + // FIXME: Delete the side table if the weak count is zero. + } + + return shouldFree; + } + + uint32_t getUnownedCount() const { + return refCounts.getUnownedCount(); + } + + + // WEAK + + LLVM_ATTRIBUTE_UNUSED_RESULT + HeapObjectSideTableEntry* incrementWeak() { + // incrementWeak need not be atomic w.r.t. concurrent deinit initiation. + // The client can't actually get a reference to the object without + // going through tryRetain(). tryRetain is the one that needs to be + // atomic w.r.t. concurrent deinit initiation. + // The check here is merely an optimization. + if (refCounts.isDeiniting()) + return nullptr; + refCounts.incrementWeak(); + return this; + } + + void decrementWeak() { + // FIXME: assertions + // FIXME: optimize barriers + bool cleanup = refCounts.decrementWeakShouldCleanUp(); + if (!cleanup) + return; + + // Weak ref count is now zero. Maybe delete the side table entry. + abort(); + } +}; + + +// Inline version of non-atomic strong decrement. +// This version can actually be non-atomic. +template <> +template +LLVM_ATTRIBUTE_ALWAYS_INLINE +inline bool RefCounts::doDecrementNonAtomic(uint32_t dec) { + + // We can get away without atomicity here. + // The caller claims that there are no other threads with strong references + // to this object. + // We can non-atomically check that there are no outstanding unowned or + // weak references, and if nobody else has a strong reference then + // nobody else can form a new unowned or weak reference. + // Therefore there is no other thread that can be concurrently + // manipulating this object's retain counts. + + auto oldbits = refCounts.load(relaxed); + + // Use slow path if we can't guarantee atomicity. + if (oldbits.hasSideTable() || oldbits.getUnownedRefCount() != 1) + return doDecrementSlow(oldbits, dec); + + auto newbits = oldbits; + bool fast = newbits.decrementStrongExtraRefCount(dec, clearPinnedFlag); + if (!fast) + return doDecrementSlow(oldbits, dec); + + refCounts.store(newbits, relaxed); + return false; // don't deinit +} + +// Out-of-line version of non-atomic strong decrement. +// This version needs to be atomic because of the +// threat of concurrent read of a weak reference. +template <> +template +inline bool RefCounts:: +doDecrementNonAtomic(uint32_t dec) { + return doDecrement(dec); +} + +// SideTableRefCountBits specialization intentionally does not exist. +template <> +template +inline bool RefCounts:: +doDecrementSideTable(InlineRefCountBits oldbits, uint32_t dec) { + auto side = oldbits.getSideTable(); + return side->decrementStrong(dec); +} + +template <> inline +HeapObject* RefCounts::getHeapObject() const { + auto prefix = ((char *)this - sizeof(void*)); + return (HeapObject *)prefix; +} + +template <> inline +HeapObject* RefCounts::getHeapObject() const { + auto prefix = ((char *)this - sizeof(void*)); + return *(HeapObject **)prefix; +} + + +// namespace swift +} + +// for use by SWIFT_HEAPOBJECT_NON_OBJC_MEMBERS +typedef swift::InlineRefCounts InlineRefCounts; + #undef relaxed #undef acquire #undef release +#undef consume // __cplusplus #endif diff --git a/stdlib/public/runtime/CMakeLists.txt b/stdlib/public/runtime/CMakeLists.txt index 732013ad1c701..1e3087922aa59 100644 --- a/stdlib/public/runtime/CMakeLists.txt +++ b/stdlib/public/runtime/CMakeLists.txt @@ -53,6 +53,7 @@ set(swift_runtime_sources Once.cpp Portability.cpp ProtocolConformance.cpp + RefCount.cpp ReflectionNative.cpp RuntimeEntrySymbols.cpp SwiftObjectNative.cpp) diff --git a/stdlib/public/runtime/Errors.cpp b/stdlib/public/runtime/Errors.cpp index a9a9e21c2344a..77367ad1f215b 100644 --- a/stdlib/public/runtime/Errors.cpp +++ b/stdlib/public/runtime/Errors.cpp @@ -30,6 +30,7 @@ #include #endif #include +#include "swift/Runtime/Errors.h" #include "swift/Runtime/Debug.h" #include "swift/Runtime/Mutex.h" #include "swift/Basic/Demangle.h" diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index a96cd48fb91c6..689f83bd95c69 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/MathExtras.h" #include "MetadataCache.h" #include "Private.h" +#include "WeakReference.h" #include "swift/Runtime/Debug.h" #include #include @@ -91,7 +92,11 @@ swift::swift_verifyEndOfLifetime(HeapObject *object) { if (object->refCounts.getUnownedCount() != 1) swift::fatalError(/* flags = */ 0, - "fatal error: weak/unowned reference to stack object\n"); + "fatal error: unowned reference to stack object\n"); + + if (object->refCounts.getWeakCount() != 0) + swift::fatalError(/* flags = */ 0, + "fatal error: weak reference to stack object\n"); } /// \brief Allocate a reference-counted object on the heap that @@ -210,11 +215,6 @@ OpaqueValue *swift::swift_projectBox(HeapObject *o) { return metadata->project(o); } -// Forward-declare this, but define it after swift_release. -extern "C" LLVM_LIBRARY_VISIBILITY void -_swift_release_dealloc(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) - __attribute__((__noinline__, __used__)); - SWIFT_RT_ENTRY_VISIBILITY extern "C" void swift::swift_retain(HeapObject *object) @@ -231,7 +231,8 @@ void swift::swift_nonatomic_retain(HeapObject *object) { SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_retain)(HeapObject *object) { - _swift_nonatomic_retain_inlined(object); + if (object) + object->refCounts.incrementNonAtomic(1); } SWIFT_RT_ENTRY_VISIBILITY @@ -243,17 +244,16 @@ void swift::swift_nonatomic_release(HeapObject *object) { SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_release)(HeapObject *object) { - if (object && object->refCounts.decrementShouldDeinitNonAtomic()) { - // TODO: Use non-atomic _swift_release_dealloc? - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndMaybeDeinitNonAtomic(1); } SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_retain)(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - _swift_retain_inlined(object); + if (object) + object->refCounts.increment(1); } SWIFT_RT_ENTRY_VISIBILITY @@ -299,9 +299,8 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_release)(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCounts.decrementShouldDeinit()) { - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndMaybeDeinit(1); } SWIFT_RT_ENTRY_VISIBILITY @@ -314,9 +313,8 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_release_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCounts.decrementShouldDeinitN(n)) { - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndMaybeDeinit(n); } void swift::swift_setDeallocating(HeapObject *object) { @@ -333,9 +331,8 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_release_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCounts.decrementShouldDeinitNNonAtomic(n)) { - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndMaybeDeinitNonAtomic(n); } size_t swift::swift_retainCount(HeapObject *object) { @@ -352,7 +349,7 @@ void swift::swift_unownedRetain(HeapObject *object) if (!object) return; - object->refCounts.incrementUnowned(); + object->refCounts.incrementUnowned(1); } SWIFT_RT_ENTRY_VISIBILITY @@ -361,12 +358,13 @@ void swift::swift_unownedRelease(HeapObject *object) if (!object) return; - if (object->refCounts.decrementUnownedShouldFree()) { - // Only class objects can be weak-retained and weak-released. - auto metadata = object->metadata; - assert(metadata->isClassObject()); - auto classMetadata = static_cast(metadata); - assert(classMetadata->isTypeMetadata()); + // Only class objects can be unowned-retained and unowned-released. + assert(object->metadata->isClassObject()); + assert(static_cast(object->metadata)->isTypeMetadata()); + + if (object->refCounts.decrementUnownedShouldFree(1)) { + auto classMetadata = static_cast(object->metadata); + SWIFT_RT_ENTRY_CALL(swift_slowDealloc) (object, classMetadata->getInstanceSize(), classMetadata->getInstanceAlignMask()); @@ -390,18 +388,19 @@ void swift::swift_unownedRelease_n(HeapObject *object, int n) if (!object) return; - if (object->refCounts.decrementUnownedShouldFreeN(n)) { - // Only class objects can be weak-retained and weak-released. - auto metadata = object->metadata; - assert(metadata->isClassObject()); - auto classMetadata = static_cast(metadata); - assert(classMetadata->isTypeMetadata()); + // Only class objects can be unowned-retained and unowned-released. + assert(object->metadata->isClassObject()); + assert(static_cast(object->metadata)->isTypeMetadata()); + + if (object->refCounts.decrementUnownedShouldFree(n)) { + auto classMetadata = static_cast(object->metadata); SWIFT_RT_ENTRY_CALL(swift_slowDealloc) (object, classMetadata->getInstanceSize(), classMetadata->getInstanceAlignMask()); } } + SWIFT_RT_ENTRY_VISIBILITY HeapObject *swift::swift_tryPin(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { @@ -421,9 +420,8 @@ HeapObject *swift::swift_tryPin(HeapObject *object) SWIFT_RT_ENTRY_VISIBILITY void swift::swift_unpin(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCounts.decrementAndUnpinShouldDeinit()) { - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndUnpinAndMaybeDeinit(); } SWIFT_RT_ENTRY_VISIBILITY @@ -451,9 +449,8 @@ HeapObject *swift::swift_nonatomic_tryPin(HeapObject *object) SWIFT_RT_ENTRY_VISIBILITY void swift::swift_nonatomic_unpin(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object && object->refCounts.decrementAndUnpinShouldDeinitNonAtomic()) { - _swift_release_dealloc(object); - } + if (object) + object->refCounts.decrementAndUnpinAndMaybeDeinitNonAtomic(); } SWIFT_RT_ENTRY_IMPL_VISIBILITY @@ -486,7 +483,7 @@ void swift::swift_unownedRetainStrong(HeapObject *object) if (!object) return; assert(object->refCounts.getUnownedCount() && - "object is not currently weakly retained"); + "object is not currently unowned-retained"); if (! object->refCounts.tryIncrement()) _swift_abortRetainUnowned(object); @@ -499,13 +496,13 @@ swift::swift_unownedRetainStrongAndRelease(HeapObject *object) if (!object) return; assert(object->refCounts.getUnownedCount() && - "object is not currently weakly retained"); + "object is not currently unowned-retained"); if (! object->refCounts.tryIncrement()) _swift_abortRetainUnowned(object); // This should never cause a deallocation. - bool dealloc = object->refCounts.decrementUnownedShouldFree(); + bool dealloc = object->refCounts.decrementUnownedShouldFree(1); assert(!dealloc && "retain-strong-and-release caused dealloc?"); (void) dealloc; } @@ -513,13 +510,13 @@ swift::swift_unownedRetainStrongAndRelease(HeapObject *object) void swift::swift_unownedCheck(HeapObject *object) { if (!object) return; assert(object->refCounts.getUnownedCount() && - "object is not currently weakly retained"); + "object is not currently unowned-retained"); if (object->refCounts.isDeiniting()) _swift_abortRetainUnowned(object); } -// Declared extern "C" LLVM_LIBRARY_VISIBILITY above. +// Declared extern "C" LLVM_LIBRARY_VISIBILITY in RefCount.h void _swift_release_dealloc(HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) { asFullMetadata(object->metadata)->destroy(object); @@ -603,7 +600,7 @@ extern "C" void swift_deallocPartialClassInstance(HeapObject *object, #endif // The strong reference count should be +1 -- tear down the object - bool shouldDeallocate = object->refCounts.decrementShouldDeinit(); + bool shouldDeallocate = object->refCounts.decrementShouldDeinit(1); assert(shouldDeallocate); (void) shouldDeallocate; swift_deallocClassInstance(object, allocatedSize, allocatedAlignMask); @@ -637,6 +634,7 @@ void swift::swift_deallocObject(HeapObject *object, // If we are tracking leaks, stop tracking this object. SWIFT_LEAKS_STOP_TRACKING_OBJECT(object); + // Drop the initial weak retain of the object. // // If the outstanding weak retain count is 1 (i.e. only the initial @@ -702,7 +700,14 @@ void swift::swift_deallocObject(HeapObject *object, // release, we will fall back on swift_unownedRelease, which does an // atomic decrement (and has the ability to reconstruct // allocatedSize and allocatedAlignMask). - if (object->refCounts.getUnownedCount() == 1) { + // + // Note: This shortcut is NOT an optimization. + // Some allocations passed to swift_deallocObject() are not compatible + // with swift_unownedRelease() because they do not have ClassMetadata. + + // FIXME: reexamine and repair this optimization + + if (object->refCounts.canBeFreedNow()) { SWIFT_RT_ENTRY_CALL(swift_slowDealloc) (object, allocatedSize, allocatedAlignMask); @@ -711,148 +716,45 @@ void swift::swift_deallocObject(HeapObject *object, } } -enum: uintptr_t { - WR_NATIVE = 1<<(swift::heap_object_abi::ObjCReservedLowBits), - WR_READING = 1<<(swift::heap_object_abi::ObjCReservedLowBits+1), - - WR_NATIVEMASK = WR_NATIVE | swift::heap_object_abi::ObjCReservedBitsMask, -}; - -static_assert(WR_READING < alignof(void*), - "weakref lock bit mustn't interfere with real pointer bits"); - -enum: short { - WR_SPINLIMIT = 64, -}; - -bool swift::isNativeSwiftWeakReference(WeakReference *ref) { - return (ref->Value & WR_NATIVEMASK) == WR_NATIVE; +void swift::_swift_abortRetainUnowned(const void *object) { + (void)object; + swift::crash("attempted to retain deallocated object"); } + void swift::swift_weakInit(WeakReference *ref, HeapObject *value) { - ref->Value = (uintptr_t)value | WR_NATIVE; - SWIFT_RT_ENTRY_CALL(swift_unownedRetain)(value); + ref->nativeInit(value); } -void swift::swift_weakAssign(WeakReference *ref, HeapObject *newValue) { - SWIFT_RT_ENTRY_CALL(swift_unownedRetain)(newValue); - auto oldValue = (HeapObject*) (ref->Value & ~WR_NATIVE); - ref->Value = (uintptr_t)newValue | WR_NATIVE; - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(oldValue); +void swift::swift_weakAssign(WeakReference *ref, HeapObject *value) { + ref->nativeAssign(value); } HeapObject *swift::swift_weakLoadStrong(WeakReference *ref) { - if (ref->Value == (uintptr_t)nullptr) { - return nullptr; - } - - // ref might be visible to other threads - auto ptr = __atomic_fetch_or(&ref->Value, WR_READING, __ATOMIC_RELAXED); - while (ptr & WR_READING) { - short c = 0; - while (__atomic_load_n(&ref->Value, __ATOMIC_RELAXED) & WR_READING) { - if (++c == WR_SPINLIMIT) { - std::this_thread::yield(); - c -= 1; - } - } - ptr = __atomic_fetch_or(&ref->Value, WR_READING, __ATOMIC_RELAXED); - } - - auto object = (HeapObject*)(ptr & ~WR_NATIVE); - if (object == nullptr) { - __atomic_store_n(&ref->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); - return nullptr; - } - if (object->refCounts.isDeiniting()) { - __atomic_store_n(&ref->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); - return nullptr; - } - auto result = swift_tryRetain(object); - __atomic_store_n(&ref->Value, ptr, __ATOMIC_RELAXED); - return result; + return ref->nativeLoadStrong(); } HeapObject *swift::swift_weakTakeStrong(WeakReference *ref) { - auto object = (HeapObject*) (ref->Value & ~WR_NATIVE); - if (object == nullptr) return nullptr; - auto result = swift_tryRetain(object); - ref->Value = (uintptr_t)nullptr; - swift_unownedRelease(object); - return result; + return ref->nativeTakeStrong(); } void swift::swift_weakDestroy(WeakReference *ref) { - auto tmp = (HeapObject*) (ref->Value & ~WR_NATIVE); - ref->Value = (uintptr_t)nullptr; - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(tmp); + ref->nativeDestroy(); } void swift::swift_weakCopyInit(WeakReference *dest, WeakReference *src) { - if (src->Value == (uintptr_t)nullptr) { - dest->Value = (uintptr_t)nullptr; - return; - } - - // src might be visible to other threads - auto ptr = __atomic_fetch_or(&src->Value, WR_READING, __ATOMIC_RELAXED); - while (ptr & WR_READING) { - short c = 0; - while (__atomic_load_n(&src->Value, __ATOMIC_RELAXED) & WR_READING) { - if (++c == WR_SPINLIMIT) { - std::this_thread::yield(); - c -= 1; - } - } - ptr = __atomic_fetch_or(&src->Value, WR_READING, __ATOMIC_RELAXED); - } - - auto object = (HeapObject*)(ptr & ~WR_NATIVE); - if (object == nullptr) { - __atomic_store_n(&src->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); - dest->Value = (uintptr_t)nullptr; - } else if (object->refCounts.isDeiniting()) { - __atomic_store_n(&src->Value, (uintptr_t)nullptr, __ATOMIC_RELAXED); - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); - dest->Value = (uintptr_t)nullptr; - } else { - SWIFT_RT_ENTRY_CALL(swift_unownedRetain)(object); - __atomic_store_n(&src->Value, ptr, __ATOMIC_RELAXED); - dest->Value = (uintptr_t)object | WR_NATIVE; - } + dest->nativeCopyInit(src); } void swift::swift_weakTakeInit(WeakReference *dest, WeakReference *src) { - auto object = (HeapObject*) (src->Value & ~WR_NATIVE); - if (object == nullptr) { - dest->Value = (uintptr_t)nullptr; - } else if (object->refCounts.isDeiniting()) { - dest->Value = (uintptr_t)nullptr; - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); - } else { - dest->Value = (uintptr_t)object | WR_NATIVE; - } - src->Value = (uintptr_t)nullptr; + dest->nativeTakeInit(src); } void swift::swift_weakCopyAssign(WeakReference *dest, WeakReference *src) { - if (dest->Value) { - auto object = (HeapObject*) (dest->Value & ~WR_NATIVE); - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); - } - swift_weakCopyInit(dest, src); + dest->nativeCopyAssign(src); } void swift::swift_weakTakeAssign(WeakReference *dest, WeakReference *src) { - if (dest->Value) { - auto object = (HeapObject*) (dest->Value & ~WR_NATIVE); - SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); - } - swift_weakTakeInit(dest, src); + dest->nativeTakeAssign(src); } -void swift::_swift_abortRetainUnowned(const void *object) { - (void)object; - swift::crash("attempted to retain deallocated object"); -} diff --git a/stdlib/public/runtime/MetadataImpl.h b/stdlib/public/runtime/MetadataImpl.h index f67798d96ebd7..f5db7a5d7bd86 100644 --- a/stdlib/public/runtime/MetadataImpl.h +++ b/stdlib/public/runtime/MetadataImpl.h @@ -47,6 +47,9 @@ #if SWIFT_OBJC_INTEROP #include "swift/Runtime/ObjCBridge.h" #endif + +#include "WeakReference.h" + #include #include diff --git a/stdlib/public/runtime/Private.h b/stdlib/public/runtime/Private.h index 28b16c2bfac9b..b077821e10c7e 100644 --- a/stdlib/public/runtime/Private.h +++ b/stdlib/public/runtime/Private.h @@ -127,6 +127,23 @@ namespace swift { LLVM_LIBRARY_VISIBILITY bool usesNativeSwiftReferenceCounting(const ClassMetadata *theClass); + static inline + bool objectUsesNativeSwiftReferenceCounting(const void *object) { + assert(!isObjCTaggedPointerOrNull(object)); +#if SWIFT_HAS_OPAQUE_ISAS + // Fast path for opaque ISAs. We don't want to call + // _swift_getClassOfAllocated as that will call object_getClass. + // Instead we can look at the bits in the ISA and tell if its a + // non-pointer opaque ISA which means it is definitely an ObjC + // object and doesn't use native swift reference counting. + if (_swift_isNonPointerIsaObjCClass(object)) + return false; + return usesNativeSwiftReferenceCounting(_swift_getClassOfAllocatedFromPointer(object)); +#else + return usesNativeSwiftReferenceCounting(_swift_getClassOfAllocated(object)); +#endif + } + /// Get the superclass pointer value used for Swift root classes. /// Note that this function may return a nullptr on non-objc platforms, /// where there is no common root class. rdar://problem/18987058 diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp new file mode 100644 index 0000000000000..645eee8fcc209 --- /dev/null +++ b/stdlib/public/runtime/RefCount.cpp @@ -0,0 +1,127 @@ +//===--- RefCount.cpp -----------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Runtime/HeapObject.h" +#include "swift/Runtime/Errors.h" + +#define relaxed std::memory_order_relaxed +#define acquire std::memory_order_acquire +#define release std::memory_order_release +#define consume std::memory_order_consume + + +namespace swift { + +template +void RefCounts::incrementSlow(RefCountBits oldbits, + uint32_t n) { + if (oldbits.hasSideTable()) { + // Out-of-line slow path. + auto side = oldbits.getSideTable(); + side->incrementStrong(n); + } + else { + // Retain count overflow. + swift::swift_abortRetainOverflow(); + } +} +template void RefCounts::incrementSlow(InlineRefCountBits oldbits, uint32_t n); +template void RefCounts::incrementSlow(SideTableRefCountBits oldbits, uint32_t n); + +template +void RefCounts::incrementNonAtomicSlow(RefCountBits oldbits, + uint32_t n) { + if (oldbits.hasSideTable()) { + // Out-of-line slow path. + auto side = oldbits.getSideTable(); + side->incrementStrong(n); // FIXME: can there be a nonatomic impl? + } else { + swift::swift_abortRetainOverflow(); + } +} +template void RefCounts::incrementNonAtomicSlow(InlineRefCountBits oldbits, uint32_t n); +template void RefCounts::incrementNonAtomicSlow(SideTableRefCountBits oldbits, uint32_t n); + +template +bool RefCounts::tryIncrementAndPinSlow() { + abort(); +} +template bool RefCounts::tryIncrementAndPinSlow(); +template bool RefCounts::tryIncrementAndPinSlow(); + +template +bool RefCounts::tryIncrementAndPinNonAtomicSlow() { + abort(); +} +template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); +template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); + + +// SideTableRefCountBits specialization intentionall does not exist. +template +bool RefCounts::tryIncrementSlow(RefCountBits oldbits) { + if (oldbits.hasSideTable()) + return oldbits.getSideTable()->tryIncrement(); + else + swift::swift_abortRetainOverflow(); +} +template bool RefCounts::tryIncrementSlow(InlineRefCountBits oldbits); +template bool RefCounts::tryIncrementSlow(SideTableRefCountBits oldbits); + +// SideTableRefCountBits specialization intentionally does not exist. +template <> +HeapObjectSideTableEntry* RefCounts::allocateSideTable() +{ + HeapObjectSideTableEntry *side = new HeapObjectSideTableEntry(getHeapObject()); + + auto newbits = InlineRefCountBits(side); + + auto oldbits = refCounts.load(relaxed); + + do { + if (oldbits.hasSideTable()) { + // Read before delete to streamline barriers. + auto result = oldbits.getSideTable(); + delete side; + return result; + } + + // FIXME: assert not deiniting or something? + + // FIXME: barriers? + side->initRefCounts(oldbits); + + } while (! refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); + return side; +} + + + +// SideTableRefCountBits specialization intentionally does not exist. +template <> +HeapObjectSideTableEntry* RefCounts::formWeakReference() +{ + auto bits = refCounts.load(relaxed); + if (!bits.hasSideTable() && bits.getIsDeiniting()) { + // Already past the start of deinit. Do nothing. + return nil; + } + + auto side = allocateSideTable(); + return side->incrementWeak(); +} + +// namespace swift +} + + diff --git a/stdlib/public/runtime/Reflection.mm b/stdlib/public/runtime/Reflection.mm index 3b77a7bebdac6..0a261dd738fad 100644 --- a/stdlib/public/runtime/Reflection.mm +++ b/stdlib/public/runtime/Reflection.mm @@ -20,6 +20,7 @@ #include "swift/Runtime/Debug.h" #include "swift/Runtime/Portability.h" #include "Private.h" +#include "WeakReference.h" #include #include #include diff --git a/stdlib/public/runtime/SwiftObject.mm b/stdlib/public/runtime/SwiftObject.mm index aa7c454dbc544..e4bec127a3454 100644 --- a/stdlib/public/runtime/SwiftObject.mm +++ b/stdlib/public/runtime/SwiftObject.mm @@ -34,6 +34,7 @@ #include "../SwiftShims/RuntimeShims.h" #include "Private.h" #include "SwiftObject.h" +#include "WeakReference.h" #include "swift/Runtime/Debug.h" #if SWIFT_OBJC_INTEROP #include @@ -419,25 +420,11 @@ - (BOOL)isNSValue__ { return NO; } static uintptr_t const objectPointerIsObjCBit = 0x00000002U; #endif -static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { - assert(!isObjCTaggedPointerOrNull(object)); -#if SWIFT_HAS_OPAQUE_ISAS - // Fast path for opaque ISAs. We don't want to call _swift_getClassOfAllocated - // as that will call object_getClass. Instead we can look at the bits in the - // ISA and tell if its a non-pointer opaque ISA which means it is definitely - // an ObjC object and doesn't use native swift reference counting. - if (_swift_isNonPointerIsaObjCClass(object)) - return false; - return usesNativeSwiftReferenceCounting(_swift_getClassOfAllocatedFromPointer(object)); -#endif - return usesNativeSwiftReferenceCounting(_swift_getClassOfAllocated(object)); -} - SWIFT_RUNTIME_EXPORT void swift::swift_unknownRetain_n(void *object, int n) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) { + if (objectUsesNativeSwiftReferenceCounting(object)) { swift_retain_n(static_cast(object), n); return; } @@ -449,7 +436,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_unknownRelease_n(void *object, int n) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) + if (objectUsesNativeSwiftReferenceCounting(object)) return swift_release_n(static_cast(object), n); for (int i = 0; i < n; ++i) objc_release(static_cast(object)); @@ -459,7 +446,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_unknownRetain(void *object) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) { + if (objectUsesNativeSwiftReferenceCounting(object)) { swift_retain(static_cast(object)); return; } @@ -470,7 +457,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_unknownRelease(void *object) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) + if (objectUsesNativeSwiftReferenceCounting(object)) return SWIFT_RT_ENTRY_CALL(swift_release)(static_cast(object)); return objc_release(static_cast(object)); } @@ -479,7 +466,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_nonatomic_unknownRetain_n(void *object, int n) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) { + if (objectUsesNativeSwiftReferenceCounting(object)) { swift_nonatomic_retain_n(static_cast(object), n); return; } @@ -491,7 +478,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_nonatomic_unknownRelease_n(void *object, int n) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) + if (objectUsesNativeSwiftReferenceCounting(object)) return swift_nonatomic_release_n(static_cast(object), n); for (int i = 0; i < n; ++i) objc_release(static_cast(object)); @@ -501,7 +488,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_nonatomic_unknownRetain(void *object) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) { + if (objectUsesNativeSwiftReferenceCounting(object)) { swift_nonatomic_retain(static_cast(object)); return; } @@ -512,7 +499,7 @@ static bool usesNativeSwiftReferenceCounting_allocated(const void *object) { void swift::swift_nonatomic_unknownRelease(void *object) SWIFT_CC(DefaultCC_IMPL) { if (isObjCTaggedPointerOrNull(object)) return; - if (usesNativeSwiftReferenceCounting_allocated(object)) + if (objectUsesNativeSwiftReferenceCounting(object)) return SWIFT_RT_ENTRY_CALL(swift_release)(static_cast(object)); return objc_release(static_cast(object)); } @@ -714,7 +701,7 @@ static bool isNonNative_unTagged_bridgeObject(void *object) { #if SWIFT_OBJC_INTEROP /*****************************************************************************/ -/**************************** UNOWNED REFERENCES *****************************/ +/************************ UNKNOWN UNOWNED REFERENCES *************************/ /*****************************************************************************/ // Swift's native unowned references are implemented purely with @@ -821,7 +808,7 @@ static bool classof(const UnownedReference *ref) { static bool isObjCForUnownedReference(void *value) { return (isObjCTaggedPointer(value) || - !usesNativeSwiftReferenceCounting_allocated(value)); + !objectUsesNativeSwiftReferenceCounting(value)); } void swift::swift_unknownUnownedInit(UnownedReference *dest, void *value) { @@ -949,138 +936,45 @@ static bool isObjCForUnownedReference(void *value) { } /*****************************************************************************/ -/****************************** WEAK REFERENCES ******************************/ +/************************** UNKNOWN WEAK REFERENCES **************************/ /*****************************************************************************/ -// FIXME: these are not really valid implementations; they assume too -// much about the implementation of ObjC weak references, and the -// loads from ->Value can race with clears by the runtime. - -static void doWeakInit(WeakReference *addr, void *value, bool valueIsNative) { - assert(value != nullptr); - if (valueIsNative) { - swift_weakInit(addr, (HeapObject*) value); - } else { - objc_initWeak((id*) &addr->Value, (id) value); - } -} - -static void doWeakDestroy(WeakReference *addr, bool valueIsNative) { - if (valueIsNative) { - swift_weakDestroy(addr); - } else { - objc_destroyWeak((id*) &addr->Value); - } -} - -void swift::swift_unknownWeakInit(WeakReference *addr, void *value) { - if (isObjCTaggedPointerOrNull(value)) { - addr->Value = (uintptr_t) value; - return; - } - doWeakInit(addr, value, usesNativeSwiftReferenceCounting_allocated(value)); +void swift::swift_unknownWeakInit(WeakReference *ref, void *value) { + return ref->unknownInit(value); } -void swift::swift_unknownWeakAssign(WeakReference *addr, void *newValue) { - // If the incoming value is not allocated, this is just a destroy - // and re-initialize. - if (isObjCTaggedPointerOrNull(newValue)) { - swift_unknownWeakDestroy(addr); - addr->Value = (uintptr_t) newValue; - return; - } - - bool newIsNative = usesNativeSwiftReferenceCounting_allocated(newValue); - - // If the existing value is not allocated, this is just an initialize. - void *oldValue = (void*) addr->Value; - if (isObjCTaggedPointerOrNull(oldValue)) - return doWeakInit(addr, newValue, newIsNative); - - bool oldIsNative = isNativeSwiftWeakReference(addr); - - // If they're both native, we can use the native function. - if (oldIsNative && newIsNative) - return swift_weakAssign(addr, (HeapObject*) newValue); - - // If neither is native, we can use the ObjC function. - if (!oldIsNative && !newIsNative) - return (void) objc_storeWeak((id*) &addr->Value, (id) newValue); - - // Otherwise, destroy according to one set of semantics and - // re-initialize with the other. - doWeakDestroy(addr, oldIsNative); - doWeakInit(addr, newValue, newIsNative); +void swift::swift_unknownWeakAssign(WeakReference *ref, void *value) { + return ref->unknownAssign(value); } -void *swift::swift_unknownWeakLoadStrong(WeakReference *addr) { - if (isNativeSwiftWeakReference(addr)) { - return swift_weakLoadStrong(addr); - } - - void *value = (void*) addr->Value; - if (isObjCTaggedPointerOrNull(value)) return value; - - return (void*) objc_loadWeakRetained((id*) &addr->Value); +void *swift::swift_unknownWeakLoadStrong(WeakReference *ref) { + return ref->unknownLoadStrong(); } -void *swift::swift_unknownWeakTakeStrong(WeakReference *addr) { - if (isNativeSwiftWeakReference(addr)) { - return swift_weakTakeStrong(addr); - } - - void *value = (void*) addr->Value; - if (isObjCTaggedPointerOrNull(value)) return value; - - void *result = (void*) objc_loadWeakRetained((id*) &addr->Value); - objc_destroyWeak((id*) &addr->Value); - return result; +void *swift::swift_unknownWeakTakeStrong(WeakReference *ref) { + return ref->unknownTakeStrong(); } -void swift::swift_unknownWeakDestroy(WeakReference *addr) { - if (isNativeSwiftWeakReference(addr)) { - return swift_weakDestroy(addr); - } - - id object = (id) addr->Value; - if (isObjCTaggedPointerOrNull(object)) return; - objc_destroyWeak((id*) &addr->Value); +void swift::swift_unknownWeakDestroy(WeakReference *ref) { + ref->unknownDestroy(); } void swift::swift_unknownWeakCopyInit(WeakReference *dest, WeakReference *src) { - if (isNativeSwiftWeakReference(src)) { - return swift_weakCopyInit(dest, src); - } - - id object = (id) src->Value; - if (isObjCTaggedPointerOrNull(object)) { - dest->Value = (uintptr_t) object; - } else { - objc_copyWeak((id*) &dest->Value, (id*) src); - } + dest->unknownCopyInit(src); } void swift::swift_unknownWeakTakeInit(WeakReference *dest, WeakReference *src) { - if (isNativeSwiftWeakReference(src)) { - return swift_weakTakeInit(dest, src); - } - - id object = (id) src->Value; - if (isObjCTaggedPointerOrNull(object)) { - dest->Value = (uintptr_t) object; - } else { - objc_moveWeak((id*) &dest->Value, (id*) &src->Value); - } + dest->unknownTakeInit(src); } -void swift::swift_unknownWeakCopyAssign(WeakReference *dest, WeakReference *src) { - if (dest == src) return; - swift_unknownWeakDestroy(dest); - swift_unknownWeakCopyInit(dest, src); +void swift::swift_unknownWeakCopyAssign(WeakReference *dest, + WeakReference *src) { + dest->unknownCopyAssign(src); } -void swift::swift_unknownWeakTakeAssign(WeakReference *dest, WeakReference *src) { - if (dest == src) return; - swift_unknownWeakDestroy(dest); - swift_unknownWeakTakeInit(dest, src); +void swift::swift_unknownWeakTakeAssign(WeakReference *dest, + WeakReference *src) { + dest->unknownTakeAssign(src); } + +// SWIFT_OBJC_INTEROP #endif /*****************************************************************************/ @@ -1336,7 +1230,7 @@ static bool usesNativeSwiftReferenceCounting_nonNull( ) { assert(object != nullptr); return !isObjCTaggedPointer(object) && - usesNativeSwiftReferenceCounting_allocated(object); + objectUsesNativeSwiftReferenceCounting(object); } #endif diff --git a/stdlib/public/runtime/WeakReference.h b/stdlib/public/runtime/WeakReference.h new file mode 100644 index 0000000000000..5ca3e662828b1 --- /dev/null +++ b/stdlib/public/runtime/WeakReference.h @@ -0,0 +1,348 @@ +//===--- WeakReference.h - Swift weak references ----------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Swift weak reference implementation. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_RUNTIME_WEAKREFERENCE_H +#define SWIFT_RUNTIME_WEAKREFERENCE_H + +#include "swift/Runtime/Config.h" +#include "swift/Runtime/HeapObject.h" +#include "swift/Runtime/Metadata.h" + +#if SWIFT_OBJC_INTEROP +#include "swift/Runtime/ObjCBridge.h" +#endif + +#include "Private.h" + +#include + +namespace swift { + +// Note: This implementation of unknown weak makes several assumptions +// about ObjC's weak variables implementation: +// * Nil is stored verbatim. +// * Tagged pointer objects are stored verbatim with no side table entry. +// * Ordinary objects are stored with the LSB two bits (64-bit) or +// one bit (32-bit) all clear. The stored value otherwise need not be +// the pointed-to object. +// +// The Swift 3 implementation of unknown weak makes the following +// additional assumptions: +// * Ordinary objects are stored *verbatim* with the LSB *three* bits (64-bit) +// or *two* bits (32-bit) all clear. + +// Thread-safety: +// +// Reading a weak reference must be thread-safe with respect to: +// * concurrent readers +// * concurrent weak reference zeroing due to deallocation of the +// pointed-to object +// * concurrent ObjC readers or zeroing (for non-native weak storage) +// +// Reading a weak reference is NOT thread-safe with respect to: +// * concurrent writes to the weak variable other than zeroing +// * concurrent destruction of the weak variable +// +// Writing a weak reference must be thread-safe with respect to: +// * concurrent weak reference zeroing due to deallocation of the +// pointed-to object +// * concurrent ObjC zeroing (for non-native weak storage) +// +// Writing a weak reference is NOT thread-safe with respect to: +// * concurrent reads +// * concurrent writes other than zeroing + +class WeakReferenceBits { +#if SWIFT_OBJC_INTEROP + // NativeMarker: ObjC low bits all zero, next bit 1 + enum : uintptr_t { + NativeMarkerValue = + uintptr_t(1) << swift::heap_object_abi::ObjCReservedLowBits, + NativeMarkerMask = + NativeMarkerValue | (NativeMarkerValue - 1) + }; +#else + // No NativeMarkerValue needed. + enum : uintptr_t { + NativeMarkerValue = 0, NativeMarkerMask = 0 + }; +#endif + + static_assert(NativeMarkerMask < alignof(void*), + "native marker bit must not interfere with pointer bits"); + + uintptr_t bits; + + public: + LLVM_ATTRIBUTE_ALWAYS_INLINE + WeakReferenceBits(HeapObjectSideTableEntry *newValue) { + setNativeOrNull(newValue); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isNativeOrNull() const { + return bits == 0 || (bits & NativeMarkerMask) == NativeMarkerValue; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + HeapObjectSideTableEntry *getNativeOrNull() const { + assert(isNativeOrNull()); + if (bits == 0) + return nullptr; + else + return + reinterpret_cast(bits & ~NativeMarkerMask); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + void setNativeOrNull(HeapObjectSideTableEntry *newValue) { + assert((uintptr_t(newValue) & NativeMarkerMask) == 0); + if (newValue) + bits = uintptr_t(newValue) | NativeMarkerValue; + else + bits = 0; + } +}; + + +class WeakReference { + union { + std::atomic nativeValue; + id nonnativeValue; + }; + + void destroyOldNativeBits(WeakReferenceBits oldBits) { + auto oldSide = oldBits.getNativeOrNull(); + if (oldSide) + oldSide->decrementWeak(); + } + HeapObject *nativeLoadStrongFromBits(WeakReferenceBits bits) { + auto side = bits.getNativeOrNull(); + return side ? side->tryRetain() : nullptr; + } + void nativeCopyInitFromBits(WeakReferenceBits srcBits) { + auto side = srcBits.getNativeOrNull(); + if (side) + side = side->incrementWeak(); + + nativeValue.store(WeakReferenceBits(side), std::memory_order_relaxed); + } + + public: + + WeakReference() = default; + + WeakReference(std::nullptr_t) + : nativeValue(WeakReferenceBits(nullptr)) { } + + WeakReference(const WeakReference& rhs) = delete; + + + void nativeInit(HeapObject *object) { + auto side = object ? object->refCounts.formWeakReference() : nullptr; + nativeValue.store(WeakReferenceBits(side), std::memory_order_relaxed); + } + + void nativeDestroy() { + auto oldBits = nativeValue.load(std::memory_order_relaxed); + nativeValue.store(nullptr, std::memory_order_relaxed); + destroyOldNativeBits(oldBits); + } + + void nativeAssign(HeapObject *newObject) { + if (newObject) { + assert(objectUsesNativeSwiftReferenceCounting(newObject) && + "weak assign native with non-native new object"); + } + + auto newSide = + newObject ? newObject->refCounts.formWeakReference() : nullptr; + auto newBits = WeakReferenceBits(newSide); + + auto oldBits = nativeValue.load(std::memory_order_relaxed); + nativeValue.store(newBits, std::memory_order_relaxed); + + assert(oldBits.isNativeOrNull() && + "weak assign native with non-native old object"); + destroyOldNativeBits(oldBits); + } + + HeapObject *nativeLoadStrong() { + auto bits = nativeValue.load(std::memory_order_relaxed); + return nativeLoadStrongFromBits(bits); + } + + HeapObject *nativeTakeStrong() { + auto bits = nativeValue.load(std::memory_order_relaxed); + nativeValue.store(nullptr, std::memory_order_relaxed); + return nativeLoadStrongFromBits(bits); + } + + void nativeCopyInit(WeakReference *src) { + auto srcBits = src->nativeValue.load(std::memory_order_relaxed); + return nativeCopyInitFromBits(srcBits); + } + + void nativeTakeInit(WeakReference *src) { + auto srcBits = src->nativeValue.load(std::memory_order_relaxed); + assert(srcBits.isNativeOrNull()); + src->nativeValue.store(nullptr, std::memory_order_relaxed); + nativeValue.store(srcBits, std::memory_order_relaxed); + } + + void nativeCopyAssign(WeakReference *src) { + if (this == src) return; + nativeDestroy(); + nativeCopyInit(src); + } + + void nativeTakeAssign(WeakReference *src) { + if (this == src) return; + nativeDestroy(); + nativeTakeInit(src); + } + +#if SWIFT_OBJC_INTEROP + private: + void nonnativeInit(id object) { + objc_initWeak(&nonnativeValue, object); + } + + void initWithNativeness(void *object, bool isNative) { + if (isNative) + nativeInit(static_cast(object)); + else + nonnativeInit(static_cast(object)); + } + + void nonnativeDestroy() { + objc_destroyWeak(&nonnativeValue); + } + + void destroyWithNativeness(bool isNative) { + if (isNative) + nativeDestroy(); + else + nonnativeDestroy(); + } + + public: + + void unknownInit(void *object) { + if (isObjCTaggedPointerOrNull(object)) { + nonnativeValue = static_cast(object); + } else { + bool isNative = objectUsesNativeSwiftReferenceCounting(object); + initWithNativeness(object, isNative); + } + } + + void unknownDestroy() { + auto oldBits = nativeValue.load(std::memory_order_relaxed); + destroyWithNativeness(oldBits.isNativeOrNull()); + } + + void unknownAssign(void *newObject) { + // If the new value is not allocated, simply destroy any old value. + if (isObjCTaggedPointerOrNull(newObject)) { + unknownDestroy(); + nonnativeValue = static_cast(newObject); + return; + } + + bool newIsNative = objectUsesNativeSwiftReferenceCounting(newObject); + + auto oldBits = nativeValue.load(std::memory_order_relaxed); + bool oldIsNative = oldBits.isNativeOrNull(); + + // If they're both native, use the native function. + if (oldIsNative && newIsNative) + return nativeAssign(static_cast(newObject)); + + // If neither is native, use ObjC. + if (!oldIsNative && !newIsNative) + return (void) objc_storeWeak(&nonnativeValue, static_cast(newObject)); + + // They don't match. Destroy and re-initialize. + destroyWithNativeness(oldIsNative); + initWithNativeness(newObject, newIsNative); + } + + void *unknownLoadStrong() { + auto bits = nativeValue.load(std::memory_order_relaxed); + if (bits.isNativeOrNull()) + return nativeLoadStrongFromBits(bits); + else + return objc_loadWeakRetained(&nonnativeValue); + } + + void *unknownTakeStrong() { + auto bits = nativeValue.load(std::memory_order_relaxed); + if (bits.isNativeOrNull()) { + nativeValue.store(nullptr, std::memory_order_relaxed); + auto result = nativeLoadStrongFromBits(bits); + destroyOldNativeBits(bits); + return result; + } + else { + id result = objc_loadWeakRetained(&nonnativeValue); + objc_destroyWeak(&nonnativeValue); + return result; + } + } + + void unknownCopyInit(WeakReference *src) { + auto srcBits = src->nativeValue.load(std::memory_order_relaxed); + if (srcBits.isNativeOrNull()) + nativeCopyInitFromBits(srcBits); + else + objc_copyWeak(&nonnativeValue, &src->nonnativeValue); + } + + void unknownTakeInit(WeakReference *src) { + auto srcBits = src->nativeValue.load(std::memory_order_relaxed); + if (srcBits.isNativeOrNull()) + nativeTakeInit(src); + else + objc_moveWeak(&nonnativeValue, &src->nonnativeValue); + } + + void unknownCopyAssign(WeakReference *src) { + if (this == src) return; + unknownDestroy(); + unknownCopyInit(src); + } + + void unknownTakeAssign(WeakReference *src) { + if (this == src) return; + unknownDestroy(); + unknownTakeInit(src); + } + +// SWIFT_OBJC_INTEROP +#endif + +}; + +static_assert(sizeof(WeakReference) == sizeof(void*), + "incorrect WeakReference size"); +static_assert(alignof(WeakReference) == alignof(void*), + "incorrect WeakReference alignment"); + +// namespace swift +} + +#endif diff --git a/unittests/runtime/weak.mm b/unittests/runtime/weak.mm index 28838ded3dc65..0ada9dcb5caa0 100644 --- a/unittests/runtime/weak.mm +++ b/unittests/runtime/weak.mm @@ -14,10 +14,17 @@ #include #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" +#include "swift/Runtime/Metadata.h" #include "gtest/gtest.h" using namespace swift; +// A fake definition of Swift runtime's WeakReference. +// This has the proper size and alignment which is all we need. +namespace swift { +class WeakReference { void *value __attribute__((unused)); }; +} + // Declare some Objective-C stuff. extern "C" void objc_release(id); From c734c9acb07ce49b0698dada2c3637ee6d4b4806 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 1 Sep 2016 15:39:46 -0700 Subject: [PATCH 05/38] Add long test infrastructure for test/Unit. Use it for max-refcount tests. --- .../runtime/LongTests/LongRefcounting.cpp | 47 ++++-------- unittests/runtime/Refcounting.cpp | 76 ------------------- 2 files changed, 16 insertions(+), 107 deletions(-) diff --git a/unittests/runtime/LongTests/LongRefcounting.cpp b/unittests/runtime/LongTests/LongRefcounting.cpp index 2e1c0f119d84d..f5498f44897be 100644 --- a/unittests/runtime/LongTests/LongRefcounting.cpp +++ b/unittests/runtime/LongTests/LongRefcounting.cpp @@ -73,8 +73,8 @@ static void releaseALot(TestObject *object, size_t &deallocated, } } -// 32-2 bits of retain count. -const uint64_t maxRC = (1ULL << (32 - 2)) - 1; +// 32-3 bits of extra retain count, plus 1 for the implicit retain +const uint64_t maxRC = 1ULL << (32 - 3); TEST(LongRefcountingTest, retain_max) { size_t deallocated = 0; @@ -83,14 +83,22 @@ TEST(LongRefcountingTest, retain_max) { // RC is 1. // Retain to maxRC, release back to 1, then release and verify deallocation. retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), maxRC); releaseALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), 1u); EXPECT_EQ(0u, deallocated); swift_release(object); EXPECT_EQ(1u, deallocated); } +TEST(LongRefcountingTest, retain_overflow_DeathTest) { + size_t deallocated = 0; + auto object = allocTestObject(&deallocated, 1); + + // RC is 1. Retain to maxRC, then retain again and verify overflow error. + retainALot(object, deallocated, maxRC - 1); + EXPECT_EQ(0u, deallocated); + ASSERT_DEATH(swift_retain(object), "swift_abortRetainOverflow"); +} + TEST(LongRefcountingTest, nonatomic_retain_max) { size_t deallocated = 0; auto object = allocTestObject(&deallocated, 1); @@ -98,42 +106,19 @@ TEST(LongRefcountingTest, nonatomic_retain_max) { // RC is 1. // Retain to maxRC, release back to 1, then release and verify deallocation. retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), maxRC); releaseALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), 1u); EXPECT_EQ(0u, deallocated); swift_nonatomic_release(object); EXPECT_EQ(1u, deallocated); } -TEST(RefcountingTest, retain_overflow) { +TEST(LongRefcountingTest, nonatomic_retain_overflow_DeathTest) { size_t deallocated = 0; auto object = allocTestObject(&deallocated, 1); - // RC is 1. Retain to maxRC, then retain again and verify overflow. - retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), maxRC); - EXPECT_EQ(0u, deallocated); - - // There is no overflow enforcement in the runtime today. - // Instead we check that the retain count wrapped around. - swift_retain(object); - EXPECT_EQ(swift_retainCount(object), 0u); - EXPECT_EQ(0u, deallocated); -} - -TEST(RefcountingTest, nonatomic_retain_overflow) { - size_t deallocated = 0; - auto object = allocTestObject(&deallocated, 1); - - // RC is 1. Retain to maxRC, then retain again and verify overflow. + // RC is 1. Retain to maxRC, then retain again and verify overflow error. retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(swift_retainCount(object), maxRC); - EXPECT_EQ(0u, deallocated); - - // There is no overflow enforcement in the runtime today. - // Instead we check that the retain count wrapped around. - swift_nonatomic_retain(object); - EXPECT_EQ(swift_retainCount(object), 0u); EXPECT_EQ(0u, deallocated); + ASSERT_DEATH(swift_nonatomic_retain(object), "swift_abortRetainOverflow"); } + diff --git a/unittests/runtime/Refcounting.cpp b/unittests/runtime/Refcounting.cpp index 51b22d9bd7cbf..a540be4b664d6 100644 --- a/unittests/runtime/Refcounting.cpp +++ b/unittests/runtime/Refcounting.cpp @@ -245,79 +245,3 @@ TEST(RefcountingTest, nonatomic_unknown_retain_release_n) { EXPECT_EQ(0u, value); EXPECT_EQ(1u, swift_retainCount(object)); } - - -//////////////////////////////////////////// -// Max retain count and overflow checking // -//////////////////////////////////////////// - - -template -static void retainALot(TestObject *object, size_t &deallocated, - uint64_t count) { - for (uint64_t i = 0; i < count; i++) { - if (atomic) swift_retain(object); - else swift_nonatomic_retain(object); - EXPECT_EQ(0u, deallocated); - } -} - -template -static void releaseALot(TestObject *object, size_t &deallocated, - uint64_t count) { - for (uint64_t i = 0; i < count; i++) { - if (atomic) swift_release(object); - else swift_nonatomic_release(object); - EXPECT_EQ(0u, deallocated); - } -} - -// 32-3 bits of extra retain count, plus 1 for the implicit retain -const uint64_t maxRC = 1ULL << (32 - 3); - -TEST(RefcountingTest, retain_max) { - size_t deallocated = 0; - auto object = allocTestObject(&deallocated, 1); - - // RC is 1. - // Retain to maxRC, release back to 1, then release and verify deallocation. - retainALot(object, deallocated, maxRC - 1); - releaseALot(object, deallocated, maxRC - 1); - EXPECT_EQ(0u, deallocated); - swift_release(object); - EXPECT_EQ(1u, deallocated); -} - -TEST(RefcountingTest, retain_overflow_DeathTest) { - size_t deallocated = 0; - auto object = allocTestObject(&deallocated, 1); - - // RC is 1. Retain to maxRC, then retain again and verify overflow error. - retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(0u, deallocated); - ASSERT_DEATH(swift_retain(object), "swift_abortRetainOverflow"); -} - -TEST(RefcountingTest, nonatomic_retain_max) { - size_t deallocated = 0; - auto object = allocTestObject(&deallocated, 1); - - // RC is 1. - // Retain to maxRC, release back to 1, then release and verify deallocation. - retainALot(object, deallocated, maxRC - 1); - releaseALot(object, deallocated, maxRC - 1); - EXPECT_EQ(0u, deallocated); - swift_nonatomic_release(object); - EXPECT_EQ(1u, deallocated); -} - -TEST(RefcountingTest, nonatomic_retain_overflow_DeathTest) { - size_t deallocated = 0; - auto object = allocTestObject(&deallocated, 1); - - // RC is 1. Retain to maxRC, then retain again and verify overflow error. - retainALot(object, deallocated, maxRC - 1); - EXPECT_EQ(0u, deallocated); - ASSERT_DEATH(swift_nonatomic_retain(object), "swift_abortRetainOverflow"); -} - From 43c106366fad04e2289a853efbf1bb7b5f1d9248 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 15 Sep 2016 19:16:10 -0700 Subject: [PATCH 06/38] WIP: Fix debug build. Fix getHeapObject() for side table access. --- stdlib/public/SwiftShims/RefCount.h | 33 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 098bbe383110e..9ccdb57dfa119 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -776,8 +776,13 @@ class RefCounts { // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. - // FIXME: inline fast path when there are no weak references outstanding - uint32_t getWeakCount() const; + uint32_t getWeakCount() const { + auto bits = refCounts.load(relaxed); + if (bits.hasSideTable()) + return bits.getSideTable()->getWeakCount(); + else + return 0; + } private: @@ -804,6 +809,10 @@ class HeapObjectSideTableEntry { : object(newObject), refCounts() { } + static ptrdiff_t refCountsOffset() { + return offsetof(HeapObjectSideTableEntry, refCounts); + } + HeapObject* tryRetain() { if (refCounts.tryIncrement()) return object.load(); // FIXME barrier @@ -881,6 +890,10 @@ class HeapObjectSideTableEntry { // Weak ref count is now zero. Maybe delete the side table entry. abort(); } + + uint32_t getWeakCount() const { + return refCounts.getWeakCount(); + } }; @@ -925,7 +938,7 @@ doDecrementNonAtomic(uint32_t dec) { return doDecrement(dec); } -// SideTableRefCountBits specialization intentionally does not exist. + template <> template inline bool RefCounts:: @@ -934,15 +947,25 @@ doDecrementSideTable(InlineRefCountBits oldbits, uint32_t dec) { return side->decrementStrong(dec); } +template <> +template +inline bool RefCounts:: +doDecrementSideTable(SideTableRefCountBits oldbits, uint32_t dec) { + abort(); +} + + template <> inline HeapObject* RefCounts::getHeapObject() const { - auto prefix = ((char *)this - sizeof(void*)); + auto offset = sizeof(void *); + auto prefix = ((char *)this - offset); return (HeapObject *)prefix; } template <> inline HeapObject* RefCounts::getHeapObject() const { - auto prefix = ((char *)this - sizeof(void*)); + auto offset = HeapObjectSideTableEntry::refCountsOffset(); + auto prefix = ((char *)this - offset); return *(HeapObject **)prefix; } From d9793d5bb6aa9bf539af6d51d87a104470bed17f Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Tue, 11 Oct 2016 21:30:52 -0700 Subject: [PATCH 07/38] WIP: Delete side table entries. Document stuff. --- stdlib/public/SwiftShims/RefCount.h | 210 +++++++++++++++++++++++++-- stdlib/public/runtime/HeapObject.cpp | 2 + stdlib/public/runtime/RefCount.cpp | 41 ++++-- 3 files changed, 224 insertions(+), 29 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 9ccdb57dfa119..e054518b14ddd 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -36,12 +36,147 @@ typedef struct { #include "swift/Basic/type_traits.h" #include "swift/Runtime/Config.h" +/* + An object conceptually has three refcounts. These refcounts + are stored either "inline" in the field following the isa + or in a "side table entry" pointed to by the field following the isa. + + The strong RC counts strong references to the object. When the strong RC + reaches zero the object is deinited, unowned reference reads become errors, + and weak reference reads become nil. + + The unowned RC counts unowned references to the object. The unowned RC + also has an extra +1 on behalf of the strong references; this +1 is + decremented after deinit completes. When the unowned RC reaches zero + the object's allocation is freed. + + The weak RC counts weak references to the object. The weak RC also has an + extra +1 on behalf of the unowned references; this +1 is decremented + after the object's allocation is freed. When the weak RC reaches zero + the object's side table entry is freed. + + Objects initially start with no side table. They can gain a side table when: + * a weak reference is formed + and pending future implementation: + * strong RC or unowned RC overflows (inline RCs will be small on 32-bit) + * associated object storage is needed on an object + * etc + Gaining a side table entry is a one-way operation; an object with a side + table entry never loses it. This prevents some thread races. + + Strong and unowned variables point at the object. + Weak variables point at the object's side table. + + + Storage layout: + + HeapObject { + isa + InlineRefCounts { + atomic { + strong RC + unowned RC + flags + OR + HeapObjectSideTableEntry* + } + } + } + + HeapObjectSideTableEntry { + SideTableRefCounts { + object pointer + atomic { + strong RC + unowned RC + weak RC + flags + } + } + } + + InlineRefCounts and SideTableRefCounts share some implementation + via RefCounts. + + InlineRefCountBits and SideTableRefCountBits share some implementation + via RefCountBitsT. + + In general: The InlineRefCounts implementation tries to perform the + operation inline. If the object has a side table it calls the + HeapObjectSideTableEntry implementation which in turn calls the + SideTableRefCounts implementation. + Downside: this code is a bit twisted. + Upside: this code has less duplication than it might otherwise + + + Object lifecycle state machine: + + LIVE without side table + The object is alive. + Object's refcounts are initialized as 1 strong, 1 unowned, 1 weak. + No side table. No weak RC storage. + Strong variable operations work normally. + Unowned variable operations work normally. + Weak variable load can't happen. + Weak variable store adds the side table, becoming LIVE with side table. + When the strong RC reaches zero deinit() is called and the object + becomes DEINITING. + + LIVE with side table + Weak variable operations work normally. + Everything else is the same as LIVE. + + DEINITING without side table + deinit() is in progress on the object. + Strong variable operations have no effect. + Unowned variable load halts in swift_abortRetainUnowned(). + Unowned variable store works normally. + Weak variable load can't happen. + Weak variable store stores nil. + When deinit() completes, the generated code calls swift_deallocObject. + swift_deallocObject calls canBeFreedNow() checking for the fast path + of no weak or unowned references. + If canBeFreedNow() the object is freed and it becomes DEAD. + Otherwise, it decrements the unowned RC and the object becomes DEINITED. + + DEINITING with side table + Weak variable load returns nil. + Weak variable store stores nil. + canBeFreedNow() is always false, so it never transitions directly to DEAD. + Everything else is the same as DEINITING. + + DEINITED without side table + deinit() has completed but there are unowned references outstanding. + Strong variable operations can't happen. + Unowned variable store can't happen. + Unowned variable load halts in swift_abortRetainUnowned(). + Weak variable operations can't happen. + When the unowned RC reaches zero, the object is freed and it becomes DEAD. + + DEINITED with side table + Weak variable load returns nil. + Weak variable store can't happen. + When the unowned RC reaches zero, the object is freed, the weak RC is + decremented, and the object becomes FREED. + Everything else is the same as DEINITED. + + FREED without side table + This state never happens. + + FREED with side table + The object is freed but there are weak refs to the side table outstanding. + Strong variable operations can't happen. + Unowned variable operations can't happen. + Weak variable load returns nil. + Weak variable store can't happen. + When the weak RC reaches zero, the side table entry is freed and + the object becomes DEAD. + + DEAD + The object and its side table are gone. +*/ + namespace swift { struct HeapObject; class HeapObjectSideTableEntry; } -// FIXME: HACK copied from HeapObject.cpp +// FIXME: HACK: copied from HeapObject.cpp extern "C" LLVM_LIBRARY_VISIBILITY void _swift_release_dealloc(swift::HeapObject *object) SWIFT_CC(RegisterPreservingCC_IMPL) @@ -90,7 +225,6 @@ class RefCountBitsT { # define ShiftAfterField(name) (name##Shift + name##BitCount) enum : uint64_t { - // FIXME: isFreeing bit UnownedRefCountShift = 0, UnownedRefCountBitCount = 32, UnownedRefCountMask = MaskForField(UnownedRefCount), @@ -347,7 +481,8 @@ class SideTableRefCountBits : public RefCountBitsT constexpr SideTableRefCountBits(uint32_t strongExtraCount, uint32_t unownedCount) : RefCountBitsT(strongExtraCount, unownedCount) - , weakBits(0) + // weak refcount starts at 1 on behalf of the unowned count + , weakBits(1) { } LLVM_ATTRIBUTE_ALWAYS_INLINE @@ -355,7 +490,7 @@ class SideTableRefCountBits : public RefCountBitsT LLVM_ATTRIBUTE_ALWAYS_INLINE SideTableRefCountBits(InlineRefCountBits newbits) - : RefCountBitsT(newbits), weakBits(0) + : RefCountBitsT(newbits), weakBits(1) { } @@ -371,6 +506,11 @@ class SideTableRefCountBits : public RefCountBitsT return weakBits == 0; } + LLVM_ATTRIBUTE_ALWAYS_INLINE + uint32_t getWeakRefCount() { + return weakBits; + } + // Side table ref count never has a side table of its own. bool hasSideTable() { return false; @@ -425,8 +565,10 @@ class RefCounts { refCounts.store(RefCountBits(0, 1), relaxed); } - /// Initialize for a stack promoted object. This prevents that the final - /// release frees the memory of the object. + // Initialize for a stack promoted object. This prevents that the final + // release frees the memory of the object. + // FIXME: need to mark these and assert they never get a side table, + // because the extra unowned ref will keep the side table alive forever void initForNotFreeing() { refCounts.store(RefCountBits(0, 2), relaxed); } @@ -620,6 +762,7 @@ class RefCounts { } /// Return true if the object can be freed directly right now. + /// (transition DEINITING -> DEAD) /// This is used in swift_deallocObject(). /// Can be freed now means: /// no side table @@ -660,6 +803,7 @@ class RefCounts { } else { // Decrement underflowed. Begin deinit. + // LIVE -> DEINITING deinitNow = true; assert(!oldbits.getIsDeiniting()); newbits = oldbits; // Undo failed decrement of newbits. @@ -720,6 +864,7 @@ class RefCounts { return oldbits.getSideTable()->incrementUnowned(inc); newbits = oldbits; + assert(newbits.getUnownedRefCount() != 0); newbits.incrementUnownedRefCount(inc); // FIXME: overflow check? } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); @@ -738,7 +883,15 @@ class RefCounts { newbits = oldbits; newbits.decrementUnownedRefCount(dec); - performFree = (newbits.getUnownedRefCount() == 0); + if (newbits.getUnownedRefCount() == 0) { + // DEINITED -> FREED or DEINITED -> DEAD + // Caller will free the object. Weak decrement is handled by + // HeapObjectSideTableEntry::decrementUnownedShouldFree. + assert(newbits.getIsDeiniting()); + performFree = true; + } else { + performFree = false; + } // FIXME: underflow check? } while (! refCounts.compare_exchange_weak(oldbits, newbits, release, relaxed)); @@ -767,21 +920,40 @@ class RefCounts { // Increment the weak reference count. void incrementWeak() { - // FIXME + auto oldbits = refCounts.load(relaxed); + RefCountBits newbits; + do { + newbits = oldbits; + assert(newbits.getWeakRefCount() != 0); + newbits.incrementWeakRefCount(); + // FIXME: overflow check + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); } + bool decrementWeakShouldCleanUp() { - // FIXME - return false; + auto oldbits = refCounts.load(relaxed); + RefCountBits newbits; + + bool performFree; + do { + newbits = oldbits; + performFree = newbits.decrementWeakRefCount(); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + + return performFree; } // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. uint32_t getWeakCount() const { auto bits = refCounts.load(relaxed); - if (bits.hasSideTable()) + if (bits.hasSideTable()) { return bits.getSideTable()->getWeakCount(); - else - return 0; + } else { + // No weak refcount storage. Return only the weak increment held + // on behalf of the unowned count. + return bits.getUnownedRefCount() ? 1 : 0; + } } @@ -827,6 +999,8 @@ class HeapObjectSideTableEntry { HeapObject *unsafeGetObject() const { return object.load(relaxed); } + + // STRONG void incrementStrong(uint32_t inc) { refCounts.increment(inc); @@ -854,7 +1028,9 @@ class HeapObjectSideTableEntry { bool decrementUnownedShouldFree(uint32_t dec) { bool shouldFree = refCounts.decrementUnownedShouldFree(dec); if (shouldFree) { - // FIXME: Delete the side table if the weak count is zero. + // DEINITED -> FREED + // Caller will free the object. + decrementWeak(); } return shouldFree; @@ -887,8 +1063,10 @@ class HeapObjectSideTableEntry { if (!cleanup) return; - // Weak ref count is now zero. Maybe delete the side table entry. - abort(); + // Weak ref count is now zero. Delete the side table entry. + // FREED -> DEAD + assert(refCounts.getUnownedCount() == 0); + delete this; } uint32_t getWeakCount() const { diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index 689f83bd95c69..5a7e908626072 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -708,10 +708,12 @@ void swift::swift_deallocObject(HeapObject *object, // FIXME: reexamine and repair this optimization if (object->refCounts.canBeFreedNow()) { + // object state DEINITING -> DEAD SWIFT_RT_ENTRY_CALL(swift_slowDealloc) (object, allocatedSize, allocatedAlignMask); } else { + // object state DEINITING -> DEINITED SWIFT_RT_ENTRY_CALL(swift_unownedRelease)(object); } } diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index 645eee8fcc209..a80e1a935e817 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -66,7 +66,7 @@ template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); -// SideTableRefCountBits specialization intentionall does not exist. +// SideTableRefCountBits specialization intentionally does not exist. template bool RefCounts::tryIncrementSlow(RefCountBits oldbits) { if (oldbits.hasSideTable()) @@ -77,25 +77,44 @@ bool RefCounts::tryIncrementSlow(RefCountBits oldbits) { template bool RefCounts::tryIncrementSlow(InlineRefCountBits oldbits); template bool RefCounts::tryIncrementSlow(SideTableRefCountBits oldbits); + +// Return an object's side table, allocating it if necessary. +// Returns nil if the object is deiniting. // SideTableRefCountBits specialization intentionally does not exist. template <> HeapObjectSideTableEntry* RefCounts::allocateSideTable() { + auto oldbits = refCounts.load(relaxed); + + // Preflight failures before allocating a new side table. + if (oldbits.hasSideTable()) { + // Already have a side table. Return it. + return oldbits.getSideTable(); + } + else if (oldbits.getIsDeiniting()) { + // Already past the start of deinit. Do nothing. + return nil; + } + + // Preflight passed. Allocate a side table. + + // FIXME: custom side table allocator HeapObjectSideTableEntry *side = new HeapObjectSideTableEntry(getHeapObject()); auto newbits = InlineRefCountBits(side); - auto oldbits = refCounts.load(relaxed); - do { if (oldbits.hasSideTable()) { + // Already have a side table. Return it and delete ours. // Read before delete to streamline barriers. auto result = oldbits.getSideTable(); delete side; return result; } - - // FIXME: assert not deiniting or something? + else if (oldbits.getIsDeiniting()) { + // Already past the start of deinit. Do nothing. + return nil; + } // FIXME: barriers? side->initRefCounts(oldbits); @@ -106,19 +125,15 @@ HeapObjectSideTableEntry* RefCounts::allocateSideTable() } - // SideTableRefCountBits specialization intentionally does not exist. template <> HeapObjectSideTableEntry* RefCounts::formWeakReference() { - auto bits = refCounts.load(relaxed); - if (!bits.hasSideTable() && bits.getIsDeiniting()) { - // Already past the start of deinit. Do nothing. - return nil; - } - auto side = allocateSideTable(); - return side->incrementWeak(); + if (side) + return side->incrementWeak(); + else + return nil; } // namespace swift From df0020b2dc7d5fe0b59cfb2688781559039f0208 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 02:35:59 -0700 Subject: [PATCH 08/38] WIP: Record unowned-retain error message to the crash log and console. --- include/swift/Runtime/Errors.h | 5 ++++- stdlib/public/runtime/Errors.cpp | 14 ++++++++++++++ stdlib/public/runtime/HeapObject.cpp | 12 ++++-------- stdlib/public/runtime/Private.h | 3 --- stdlib/public/runtime/SwiftObject.mm | 5 +++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/include/swift/Runtime/Errors.h b/include/swift/Runtime/Errors.h index fcce4736e1ffe..6933fb1a519fb 100644 --- a/include/swift/Runtime/Errors.h +++ b/include/swift/Runtime/Errors.h @@ -1,4 +1,4 @@ -//===--- Errors.n - Error reporting utilities -------------------*- C++ -*-===// +//===--- Errors.h - Error reporting utilities -------------------*- C++ -*-===// // // This source file is part of the Swift.org open source project // @@ -21,4 +21,7 @@ namespace swift { LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE void swift_abortRetainOverflow(); +LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE +void swift_abortRetainUnowned(const void *object); + } diff --git a/stdlib/public/runtime/Errors.cpp b/stdlib/public/runtime/Errors.cpp index 77367ad1f215b..90059784cd2d8 100644 --- a/stdlib/public/runtime/Errors.cpp +++ b/stdlib/public/runtime/Errors.cpp @@ -280,3 +280,17 @@ void swift::swift_abortRetainOverflow() { swift::fatalError(FatalErrorFlags::ReportBacktrace, "fatal error: object was retained too many times"); } + +// Crash due to retain of a dead unowned reference. +// FIXME: can't pass the object's address from InlineRefCounts without hacks +void swift::swift_abortRetainUnowned(const void *object) { + if (object) { + swift::fatalError(FatalErrorFlags::ReportBacktrace, + "fatal error: attempted to read an unowned reference but " + "object %p was already deallocated", object); + } else { + swift::fatalError(FatalErrorFlags::ReportBacktrace, + "fatal error: attempted to read an unowned reference but " + "the object was already deallocated"); + } +} diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index 5a7e908626072..94496082c711b 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "swift/Basic/Lazy.h" +#include "swift/Runtime/Errors.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Heap.h" #include "swift/Runtime/Metadata.h" @@ -486,7 +487,7 @@ void swift::swift_unownedRetainStrong(HeapObject *object) "object is not currently unowned-retained"); if (! object->refCounts.tryIncrement()) - _swift_abortRetainUnowned(object); + swift::swift_abortRetainUnowned(object); } SWIFT_RT_ENTRY_VISIBILITY @@ -499,7 +500,7 @@ swift::swift_unownedRetainStrongAndRelease(HeapObject *object) "object is not currently unowned-retained"); if (! object->refCounts.tryIncrement()) - _swift_abortRetainUnowned(object); + swift::swift_abortRetainUnowned(object); // This should never cause a deallocation. bool dealloc = object->refCounts.decrementUnownedShouldFree(1); @@ -513,7 +514,7 @@ void swift::swift_unownedCheck(HeapObject *object) { "object is not currently unowned-retained"); if (object->refCounts.isDeiniting()) - _swift_abortRetainUnowned(object); + swift::swift_abortRetainUnowned(object); } // Declared extern "C" LLVM_LIBRARY_VISIBILITY in RefCount.h @@ -718,11 +719,6 @@ void swift::swift_deallocObject(HeapObject *object, } } -void swift::_swift_abortRetainUnowned(const void *object) { - (void)object; - swift::crash("attempted to retain deallocated object"); -} - void swift::swift_weakInit(WeakReference *ref, HeapObject *value) { ref->nativeInit(value); diff --git a/stdlib/public/runtime/Private.h b/stdlib/public/runtime/Private.h index b077821e10c7e..b78d0b7511c7a 100644 --- a/stdlib/public/runtime/Private.h +++ b/stdlib/public/runtime/Private.h @@ -42,9 +42,6 @@ namespace swift { const ProtocolDescriptor *theProtocol); #endif - extern "C" LLVM_LIBRARY_VISIBILITY LLVM_ATTRIBUTE_NORETURN - void _swift_abortRetainUnowned(const void *object); - /// Is the given value a valid alignment mask? static inline bool isAlignmentMask(size_t mask) { // mask == xyz01111... diff --git a/stdlib/public/runtime/SwiftObject.mm b/stdlib/public/runtime/SwiftObject.mm index e4bec127a3454..9a88861f6b75c 100644 --- a/stdlib/public/runtime/SwiftObject.mm +++ b/stdlib/public/runtime/SwiftObject.mm @@ -26,6 +26,7 @@ #include "swift/Basic/Demangle.h" #include "swift/Basic/LLVM.h" #include "swift/Basic/Lazy.h" +#include "swift/Runtime/Errors.h" #include "swift/Runtime/Heap.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" @@ -848,7 +849,7 @@ static bool isObjCForUnownedReference(void *value) { } else if (auto objcRef = dyn_cast(ref)) { auto result = (void*) objc_loadWeakRetained(&objcRef->storage()->WeakRef); if (result == nullptr) { - _swift_abortRetainUnowned(nullptr); + swift::swift_abortRetainUnowned(nullptr); } return result; } else { @@ -863,7 +864,7 @@ static bool isObjCForUnownedReference(void *value) { auto storage = objcRef->storage(); auto result = (void*) objc_loadWeakRetained(&objcRef->storage()->WeakRef); if (result == nullptr) { - _swift_abortRetainUnowned(nullptr); + swift::swift_abortRetainUnowned(nullptr); } delete storage; return result; From 9c7a88a5619addd91d1b073bd30fd42f6dd98799 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 02:41:47 -0700 Subject: [PATCH 09/38] WIP: Fix leak in swift_weakTakeStrong(). --- stdlib/public/runtime/WeakReference.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/stdlib/public/runtime/WeakReference.h b/stdlib/public/runtime/WeakReference.h index 5ca3e662828b1..4a4fb50857f10 100644 --- a/stdlib/public/runtime/WeakReference.h +++ b/stdlib/public/runtime/WeakReference.h @@ -129,10 +129,22 @@ class WeakReference { if (oldSide) oldSide->decrementWeak(); } + HeapObject *nativeLoadStrongFromBits(WeakReferenceBits bits) { auto side = bits.getNativeOrNull(); return side ? side->tryRetain() : nullptr; } + + HeapObject *nativeTakeStrongFromBits(WeakReferenceBits bits) { + auto side = bits.getNativeOrNull(); + if (side) { + side->decrementWeak(); + return side->tryRetain(); + } else { + return nullptr; + } + } + void nativeCopyInitFromBits(WeakReferenceBits srcBits) { auto side = srcBits.getNativeOrNull(); if (side) @@ -188,7 +200,7 @@ class WeakReference { HeapObject *nativeTakeStrong() { auto bits = nativeValue.load(std::memory_order_relaxed); nativeValue.store(nullptr, std::memory_order_relaxed); - return nativeLoadStrongFromBits(bits); + return nativeTakeStrongFromBits(bits); } void nativeCopyInit(WeakReference *src) { @@ -293,9 +305,7 @@ class WeakReference { auto bits = nativeValue.load(std::memory_order_relaxed); if (bits.isNativeOrNull()) { nativeValue.store(nullptr, std::memory_order_relaxed); - auto result = nativeLoadStrongFromBits(bits); - destroyOldNativeBits(bits); - return result; + return nativeTakeStrongFromBits(bits); } else { id result = objc_loadWeakRetained(&nonnativeValue); From d3a236394c6dc67c6d76e2388d839366353e4031 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 02:44:33 -0700 Subject: [PATCH 10/38] WIP: nits --- stdlib/public/SwiftShims/RefCount.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index e054518b14ddd..d9d3a33900a63 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -184,8 +184,6 @@ _swift_release_dealloc(swift::HeapObject *object) namespace swift { -// FIXME: There are lots of asserts here which hurts Assert performance a lot. - // FIXME: many `relaxed` in this file should be `consume`, // but (1) the compiler doesn't support `consume` directly, // and (2) the compiler promotes `consume` to `acquire` instead which @@ -512,6 +510,7 @@ class SideTableRefCountBits : public RefCountBitsT } // Side table ref count never has a side table of its own. + LLVM_ATTRIBUTE_ALWAYS_INLINE bool hasSideTable() { return false; } @@ -805,7 +804,7 @@ class RefCounts { // Decrement underflowed. Begin deinit. // LIVE -> DEINITING deinitNow = true; - assert(!oldbits.getIsDeiniting()); + assert(!oldbits.getIsDeiniting()); // FIXME: make this an error? newbits = oldbits; // Undo failed decrement of newbits. newbits.setStrongExtraRefCount(0); newbits.setIsDeiniting(true); From b8decdb1918c7f3fa2b09eb761a7054125ac2bca Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 02:45:01 -0700 Subject: [PATCH 11/38] WIP: Add refcounting lifecycle tests. --- .../runtime/LongTests/LongRefcounting.cpp | 692 +++++++++++++++++- 1 file changed, 682 insertions(+), 10 deletions(-) diff --git a/unittests/runtime/LongTests/LongRefcounting.cpp b/unittests/runtime/LongTests/LongRefcounting.cpp index f5498f44897be..d752026ce8fad 100644 --- a/unittests/runtime/LongTests/LongRefcounting.cpp +++ b/unittests/runtime/LongTests/LongRefcounting.cpp @@ -14,23 +14,89 @@ #include "swift/Runtime/Metadata.h" #include "gtest/gtest.h" +#ifdef __APPLE__ +// FIXME: is EXPECT_UNALLOCATED reliable enough for CI? +// EXPECT_ALLOCATED may fail falsely if the memory is re-allocated. +# include +# define EXPECT_ALLOCATED(p) EXPECT_NE(0u, malloc_size(p)) +# define EXPECT_UNALLOCATED(p) EXPECT_EQ(0u, malloc_size(p)) +#else +// FIXME: heap assertion for other platforms? +# define EXPECT_ALLOCATED(p) do {} while (0) +# define EXPECT_UNALLOCATED(p) do {} while (0) +#endif + using namespace swift; struct TestObject : HeapObject { + // *Addr = Value during deinit size_t *Addr; size_t Value; + + // Check lifecycle state DEINITING during deinit + bool CheckLifecycle; + + // Weak variable to check in CheckLifecycle. nullptr skips the check. + // On entry to deinit: must point to object + // On exit from deinit: is destroyed + WeakReference *WeakRef; + + TestObject(size_t *addr, size_t value) + : Addr(addr), Value(value), CheckLifecycle(false), WeakRef(nullptr) + { } }; -static void destroyTestObject(HeapObject *_object) { + +static void deinitTestObject(HeapObject *_object) { auto object = static_cast(_object); assert(object->Addr && "object already deallocated"); + + if (object->CheckLifecycle) { + // RC ok + swift_retain(object); + swift_retain(object); + swift_release(object); + swift_release(object); + // FIXME: RC underflow during deinit? + + // URC load crashes + // URC increment OK + // URC decrement OK + ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + swift_unownedRetain(object); + swift_unownedRetain(object); + swift_unownedRelease(object); + swift_unownedRelease(object); + + if (object->WeakRef) { + // WRC load is nil + // WRC increment is nil + // WRC decrement OK + + // WRC -1 + auto weak_value = swift_weakLoadStrong(object->WeakRef); + EXPECT_EQ(nullptr, weak_value); + swift_weakDestroy(object->WeakRef); + + // WRC no change + swift_weakInit(object->WeakRef, object); + weak_value = swift_weakLoadStrong(object->WeakRef); + EXPECT_EQ(nullptr, weak_value); + + // WRC no change + swift_weakInit(object->WeakRef, object); + weak_value = swift_weakLoadStrong(object->WeakRef); + EXPECT_EQ(nullptr, weak_value); + } + } + *object->Addr = object->Value; object->Addr = nullptr; swift_deallocObject(object, sizeof(TestObject), alignof(TestObject) - 1); } static const FullMetadata TestClassObjectMetadata = { - { { &destroyTestObject }, { &_TWVBo } }, + { { &deinitTestObject }, { &_TWVBo } }, { { { MetadataKind::Class } }, 0, /*rodata*/ 1, ClassFlags::UsesSwift1Refcounting, nullptr, 0, 0, 0, 0, 0 } }; @@ -38,13 +104,11 @@ static const FullMetadata TestClassObjectMetadata = { /// Create an object that, when deallocated, stores the given value to /// the given pointer. static TestObject *allocTestObject(size_t *addr, size_t value) { - auto result = - static_cast(swift_allocObject(&TestClassObjectMetadata, - sizeof(TestObject), - alignof(TestObject) - 1)); - result->Addr = addr; - result->Value = value; - return result; + auto buf = swift_allocObject(&TestClassObjectMetadata, + sizeof(TestObject), + alignof(TestObject) - 1); + + return new (buf) TestObject(addr, value); } @@ -52,7 +116,6 @@ static TestObject *allocTestObject(size_t *addr, size_t value) { // Max retain count and overflow checking // //////////////////////////////////////////// - template static void retainALot(TestObject *object, size_t &deallocated, uint64_t count) { @@ -122,3 +185,612 @@ TEST(LongRefcountingTest, nonatomic_retain_overflow_DeathTest) { ASSERT_DEATH(swift_nonatomic_retain(object), "swift_abortRetainOverflow"); } + +////////////////////// +// Object lifecycle // +////////////////////// + +// FIXME: use the real WeakReference definition +namespace swift { + +class WeakReference { + uintptr_t value; + + public: + void *getSideTable() { + return (void*)(value & ~3ULL); + } +}; + +} + +// Lifecycle paths. One test each. +// +// LIVE -> DEINITING -> DEAD, no side table +// LIVE -> DEINITING -> DEINITED -> DEAD, no side table +// +// LIVE -> DEINITING -> DEAD, with side table +// LIVE -> DEINITING -> DEINITED -> DEAD, with side table +// LIVE -> DEINITING -> FREED -> DEAD, with side table +// LIVE -> DEINITING -> DEINITED -> FREED -> DEAD, with side table + + +// LIVE -> DEINITING -> DEAD, no side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_no_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load can't happen + // WRC increment adds side table which is tested elsewhere + // WRC decrement can't happen + + // RC == 1 + // URC == 1 + // WRC == 1 + + swift_release(object); // DEINITING is in here + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(object); +} + + +// LIVE -> DEINITING -> DEINITED -> DEAD, no side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_no_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load can't happen + // WRC increment adds side table which is tested elsewhere + // WRC decrement can't happen + + // RC == 1 + // URC == 3 + // WRC == 1 + + swift_release(object); // DEINITING is in here + + // Object is DEINITED + // RC == 0 + // URC == 2 + // WRC == 1 + + EXPECT_EQ(1u, deinited); + EXPECT_ALLOCATED(object); + + // RC can't happen + + // WRC can't happen + + // URC load crashes + // URC increment can't happen + // URC decrement OK + ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + swift_unownedRelease(object); + EXPECT_ALLOCATED(object); + + // RC == 0 + // URC == 1 + // WRC == 1 + + swift_unownedRelease(object); + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(object); +} + + +// LIVE -> DEINITING -> DEAD, with side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_with_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + // Remaining releases are performed after the side table is allocated. + + // WRC load can't happen + // WRC increment adds side table + // WRC decrement can't happen + + WeakReference w; + swift_weakInit(&w, object); + + // Object is LIVE with side table + + void *side = w.getSideTable(); + EXPECT_ALLOCATED(side); + + WeakReference w_deinit; + swift_weakInit(&w_deinit, object); + object->WeakRef = &w_deinit; + // destroyed during deinit + + // RC increment ok + // RC decrement ok + swift_retain(object); + swift_retain(object); + swift_retain(object); + swift_release(object); + swift_release(object); + swift_release(object); + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + // ...and balancing from previously... + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load OK + // WRC increment OK + // WRC decrement OK + + WeakReference w2; + swift_weakInit(&w2, object); + HeapObject *weakValue = swift_weakTakeStrong(&w2); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + weakValue = swift_weakTakeStrong(&w); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + // RC == 1 + // URC == 1 + // WRC == 1 + + swift_release(object); // DEINITING is in here + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(side); + EXPECT_UNALLOCATED(object); +} + + +// LIVE -> DEINITING -> DEINITED -> DEAD, with side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_with_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + // Remaining releases are performed during DEINITED. + + // WRC load can't happen + // WRC increment adds side table + // WRC decrement can't happen + + WeakReference w; + swift_weakInit(&w, object); + + // Object is LIVE with side table + + void *side = w.getSideTable(); + EXPECT_ALLOCATED(side); + + WeakReference w_deinit; + swift_weakInit(&w_deinit, object); + object->WeakRef = &w_deinit; + // destroyed during deinit + + // RC increment ok + // RC decrement ok + swift_retain(object); + swift_retain(object); + swift_retain(object); + swift_release(object); + swift_release(object); + swift_release(object); + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load OK + // WRC increment OK + // WRC decrement OK + + WeakReference w2; + swift_weakInit(&w2, object); + HeapObject *weakValue = swift_weakTakeStrong(&w2); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + weakValue = swift_weakTakeStrong(&w); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + // RC == 1 + // URC == 3 + // WRC == 1 + + swift_release(object); // DEINITING is in here + + // Object is DEINITED + // RC == 0 + // URC == 2 + // WRC == 1 + + EXPECT_EQ(1u, deinited); + EXPECT_ALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC can't happen + + // WRC can't happen + + // URC load crashes + // URC increment can't happen + // URC decrement OK + ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + swift_unownedRelease(object); + EXPECT_ALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC == 0 + // URC == 1 + // WRC == 1 + + swift_unownedRelease(object); + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(object); + EXPECT_UNALLOCATED(side); +} + + +// LIVE -> DEINITING -> FREED -> DEAD, with side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_freed_with_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load can't happen + // WRC increment adds side table + // WRC decrement can't happen + + WeakReference w; + swift_weakInit(&w, object); + + // Object is LIVE with side table + + void *side = w.getSideTable(); + EXPECT_ALLOCATED(side); + + WeakReference w_deinit; + swift_weakInit(&w_deinit, object); + object->WeakRef = &w_deinit; + // destroyed during deinit + + // RC increment ok + // RC decrement ok + swift_retain(object); + swift_retain(object); + swift_retain(object); + swift_release(object); + swift_release(object); + swift_release(object); + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load OK + // WRC increment OK + // WRC decrement OK + + WeakReference w2; + swift_weakInit(&w2, object); + HeapObject *weakValue = swift_weakLoadStrong(&w2); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + weakValue = swift_weakLoadStrong(&w); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + // RC == 1 + // URC == 1 + // WRC == 3 + + swift_release(object); // DEINITING is in here + + // Object is FREED + // RC == 0 + // URC == 0 + // WRC == 2 + + EXPECT_EQ(1u, deinited); + EXPECT_UNALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC can't happen + + // URC can't happen + + // WRC load is nil + // WRC increment can't happen + // WRC decrement OK + + weakValue = swift_weakTakeStrong(&w2); + EXPECT_EQ(0, weakValue); + + // RC == 0 + // URC == 0 + // WRC == 1 + + weakValue = swift_weakTakeStrong(&w); + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(side); + EXPECT_EQ(0, weakValue); +} + + +// LIVE -> DEINITING -> DEINITED -> FREED -> DEAD, with side table +TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_freed_with_side_DeathTest) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + size_t deinited = 0; + auto object = allocTestObject(&deinited, 1); + object->CheckLifecycle = true; + + // Object is LIVE + + EXPECT_ALLOCATED(object); + // RC tested elsewhere + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + // Remaining releases are performed during DEINITED. + + // WRC load can't happen + // WRC increment adds side table + // WRC decrement can't happen + + WeakReference w; + swift_weakInit(&w, object); + + // Object is LIVE with side table + + void *side = w.getSideTable(); + EXPECT_ALLOCATED(side); + + WeakReference w_deinit; + swift_weakInit(&w_deinit, object); + object->WeakRef = &w_deinit; + // destroyed during deinit + + // RC increment ok + // RC decrement ok + swift_retain(object); + swift_retain(object); + swift_retain(object); + swift_release(object); + swift_release(object); + swift_release(object); + + // URC load OK + // URC increment OK + // URC decrement OK + swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRetain(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + swift_unownedRelease(object); swift_unownedCheck(object); + + // WRC load OK + // WRC increment OK + // WRC decrement OK + + WeakReference w2; + swift_weakInit(&w2, object); + HeapObject *weakValue = swift_weakLoadStrong(&w2); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + weakValue = swift_weakLoadStrong(&w); + EXPECT_EQ(weakValue, object); + swift_release(weakValue); + + // RC == 1 + // URC == 3 + // WRC == 3 + + swift_release(object); // DEINITING is in here + + // Object is DEINITED + // RC == 0 + // URC == 2 + // WRC == 3 + + EXPECT_EQ(1u, deinited); + EXPECT_ALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC can't happen + + // WRC load is nil + // WRC increment can't happen + // WRC decrement OK + + weakValue = swift_weakTakeStrong(&w2); + EXPECT_EQ(0, weakValue); + + // URC load crashes + // URC increment can't happen + // URC decrement OK + ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + swift_unownedRelease(object); + EXPECT_ALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC == 0 + // URC == 1 + // WRC == 2 + + swift_unownedRelease(object); + + // Object is FREED + // RC == 0 + // URC == 0 + // WRC == 1 + + EXPECT_EQ(1u, deinited); + EXPECT_UNALLOCATED(object); + EXPECT_ALLOCATED(side); + + // RC can't happen + + // URC can't happen + + // WRC load is nil + // WRC increment can't happen + // WRC decrement OK + + // RC == 0 + // URC == 0 + // WRC == 1 + + weakValue = swift_weakTakeStrong(&w); + + // Object is DEAD + // RC == 0 + // URC == 0 + // WRC == 0 + + EXPECT_UNALLOCATED(side); + EXPECT_EQ(0, weakValue); +} From 42e12f0eec2a6b8b6ca971f5c0ceb6f9532a3a5a Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 18:12:02 -0700 Subject: [PATCH 12/38] WIP: Fix Linux build. Fold Errors.h into Debug.h. --- include/swift/Runtime/Debug.h | 20 ++++++++++++------ include/swift/Runtime/Errors.h | 27 ------------------------- stdlib/public/SwiftShims/RefCount.h | 29 +++++++++++++++++++++------ stdlib/public/runtime/Errors.cpp | 1 - stdlib/public/runtime/HeapObject.cpp | 3 +-- stdlib/public/runtime/RefCount.cpp | 1 - stdlib/public/runtime/SwiftObject.mm | 1 - stdlib/public/runtime/WeakReference.h | 2 ++ 8 files changed, 40 insertions(+), 44 deletions(-) delete mode 100644 include/swift/Runtime/Errors.h diff --git a/include/swift/Runtime/Debug.h b/include/swift/Runtime/Debug.h index 856164f6202b5..049bff9940e0c 100644 --- a/include/swift/Runtime/Debug.h +++ b/include/swift/Runtime/Debug.h @@ -20,7 +20,6 @@ #include #include #include "swift/Runtime/Config.h" -#include "swift/Runtime/Metadata.h" #ifdef SWIFT_HAVE_CRASHREPORTERCLIENT @@ -63,6 +62,12 @@ static void CRSetCrashLogMessage(const char *) {} namespace swift { +// Duplicated from Metadata.h. We want to use this header +// in places that cannot themselves include Metadata.h. +struct InProcess; +template struct TargetMetadata; +using Metadata = TargetMetadata; + // swift::crash() halts with a crash log message, // but otherwise tries not to disturb register state. @@ -91,11 +96,6 @@ static inline void _failCorruptType(const Metadata *type) { LLVM_ATTRIBUTE_NORETURN extern void fatalError(uint32_t flags, const char *format, ...); - -struct InProcess; - -template struct TargetMetadata; -using Metadata = TargetMetadata; // swift_dynamicCastFailure halts using fatalError() // with a description of a failed cast's types. @@ -117,6 +117,14 @@ SWIFT_RUNTIME_EXPORT extern "C" void swift_reportError(uint32_t flags, const char *message); +// Halt due to an overflow in swift_retain(). +LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE +void swift_abortRetainOverflow(); + +// Halt due to reading an unowned reference to a dead object. +LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE +void swift_abortRetainUnowned(const void *object); + // namespace swift } diff --git a/include/swift/Runtime/Errors.h b/include/swift/Runtime/Errors.h deleted file mode 100644 index 6933fb1a519fb..0000000000000 --- a/include/swift/Runtime/Errors.h +++ /dev/null @@ -1,27 +0,0 @@ -//===--- Errors.h - Error reporting utilities -------------------*- C++ -*-===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -// -// Utilities for reporting errors to stderr, system console, and crash logs. -// -//===----------------------------------------------------------------------===// - -#include "swift/Basic/LLVM.h" - -namespace swift { - -LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE -void swift_abortRetainOverflow(); - -LLVM_ATTRIBUTE_NORETURN LLVM_ATTRIBUTE_NOINLINE -void swift_abortRetainUnowned(const void *object); - -} diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index d9d3a33900a63..830b1ac33e0c1 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -35,6 +35,7 @@ typedef struct { #include "llvm/Support/Compiler.h" #include "swift/Basic/type_traits.h" #include "swift/Runtime/Config.h" +#include "swift/Runtime/Debug.h" /* An object conceptually has three refcounts. These refcounts @@ -693,10 +694,10 @@ class RefCounts { // object as deiniting. // // Precondition: the reference count must be 1 - void decrementFromOneAndDeinitNonAtomic() { + void decrementFromOneNonAtomic() { auto bits = refCounts.load(relaxed); if (bits.hasSideTable()) - abort(); + return bits.getSideTable()->decrementFromOneNonAtomic(); assert(!bits.getIsDeiniting()); assert(bits.getStrongExtraRefCount() == 0 && "Expect a refcount of 1"); @@ -710,7 +711,8 @@ class RefCounts { uint32_t getCount() const { auto bits = refCounts.load(relaxed); if (bits.hasSideTable()) - abort(); + return bits.getSideTable()->getCount(); + assert(!bits.getIsDeiniting()); // FIXME: can we assert this? return bits.getStrongExtraRefCount() + 1; } @@ -720,7 +722,8 @@ class RefCounts { bool isUniquelyReferenced() const { auto bits = refCounts.load(relaxed); if (bits.hasSideTable()) - abort(); + return false; // FIXME: implement side table path if useful + assert(!bits.getIsDeiniting()); return bits.getStrongExtraRefCount() == 0; } @@ -730,7 +733,8 @@ class RefCounts { bool isUniquelyReferencedOrPinned() const { auto bits = refCounts.load(relaxed); if (bits.hasSideTable()) - abort(); + return false; // FIXME: implement side table path if useful + assert(!bits.getIsDeiniting()); return (bits.getStrongExtraRefCount() == 0 || bits.getIsPinned()); @@ -980,9 +984,12 @@ class HeapObjectSideTableEntry { : object(newObject), refCounts() { } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Winvalid-offsetof" static ptrdiff_t refCountsOffset() { return offsetof(HeapObjectSideTableEntry, refCounts); } +#pragma clang diagnostic pop HeapObject* tryRetain() { if (refCounts.tryIncrement()) @@ -1010,6 +1017,11 @@ class HeapObjectSideTableEntry { return refCounts.doDecrement(dec); } + void decrementFromOneNonAtomic() { + // FIXME: can there be a non-atomic implementation? + decrementStrong(1); + } + bool isDeiniting() const { return refCounts.isDeiniting(); } @@ -1018,6 +1030,10 @@ class HeapObjectSideTableEntry { return refCounts.tryIncrement(); } + uint32_t getCount() const { + return refCounts.getCount(); + } + // UNOWNED void incrementUnowned(uint32_t inc) { @@ -1128,7 +1144,8 @@ template <> template inline bool RefCounts:: doDecrementSideTable(SideTableRefCountBits oldbits, uint32_t dec) { - abort(); + swift::crash("side table refcount must not have " + "a side table entry of its own"); } diff --git a/stdlib/public/runtime/Errors.cpp b/stdlib/public/runtime/Errors.cpp index 90059784cd2d8..d525edeaa78bf 100644 --- a/stdlib/public/runtime/Errors.cpp +++ b/stdlib/public/runtime/Errors.cpp @@ -30,7 +30,6 @@ #include #endif #include -#include "swift/Runtime/Errors.h" #include "swift/Runtime/Debug.h" #include "swift/Runtime/Mutex.h" #include "swift/Basic/Demangle.h" diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index 94496082c711b..d987e1a373713 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -15,7 +15,6 @@ //===----------------------------------------------------------------------===// #include "swift/Basic/Lazy.h" -#include "swift/Runtime/Errors.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Heap.h" #include "swift/Runtime/Metadata.h" @@ -319,7 +318,7 @@ void SWIFT_RT_ENTRY_IMPL(swift_release_n)(HeapObject *object, uint32_t n) } void swift::swift_setDeallocating(HeapObject *object) { - object->refCounts.decrementFromOneAndDeinitNonAtomic(); + object->refCounts.decrementFromOneNonAtomic(); } SWIFT_RT_ENTRY_VISIBILITY diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index a80e1a935e817..e56481670e1e2 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "swift/Runtime/HeapObject.h" -#include "swift/Runtime/Errors.h" #define relaxed std::memory_order_relaxed #define acquire std::memory_order_acquire diff --git a/stdlib/public/runtime/SwiftObject.mm b/stdlib/public/runtime/SwiftObject.mm index 9a88861f6b75c..4087799dc0f53 100644 --- a/stdlib/public/runtime/SwiftObject.mm +++ b/stdlib/public/runtime/SwiftObject.mm @@ -26,7 +26,6 @@ #include "swift/Basic/Demangle.h" #include "swift/Basic/LLVM.h" #include "swift/Basic/Lazy.h" -#include "swift/Runtime/Errors.h" #include "swift/Runtime/Heap.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" diff --git a/stdlib/public/runtime/WeakReference.h b/stdlib/public/runtime/WeakReference.h index 4a4fb50857f10..044d630f75486 100644 --- a/stdlib/public/runtime/WeakReference.h +++ b/stdlib/public/runtime/WeakReference.h @@ -121,7 +121,9 @@ class WeakReferenceBits { class WeakReference { union { std::atomic nativeValue; +#if SWIFT_OBJC_INTEROP id nonnativeValue; +#endif }; void destroyOldNativeBits(WeakReferenceBits oldBits) { From 6a0b6aa80736f3d43f6cda46d308128da3ab248e Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 13 Oct 2016 18:29:42 -0700 Subject: [PATCH 13/38] WIP: Fix Linux build. --- stdlib/public/runtime/WeakReference.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stdlib/public/runtime/WeakReference.h b/stdlib/public/runtime/WeakReference.h index 044d630f75486..fdc959e903fb9 100644 --- a/stdlib/public/runtime/WeakReference.h +++ b/stdlib/public/runtime/WeakReference.h @@ -87,6 +87,9 @@ class WeakReferenceBits { uintptr_t bits; public: + LLVM_ATTRIBUTE_ALWAYS_INLINE + WeakReferenceBits() { } + LLVM_ATTRIBUTE_ALWAYS_INLINE WeakReferenceBits(HeapObjectSideTableEntry *newValue) { setNativeOrNull(newValue); From 6113aab83c45f9f1cf53cb9e543914f2999051c5 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Fri, 14 Oct 2016 14:18:06 -0700 Subject: [PATCH 14/38] WIP: Fix Linux build. --- stdlib/public/runtime/RefCount.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index e56481670e1e2..fee5713f4b4dd 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -78,7 +78,7 @@ template bool RefCounts::tryIncrementSlow(SideTableRefCou // Return an object's side table, allocating it if necessary. -// Returns nil if the object is deiniting. +// Returns null if the object is deiniting. // SideTableRefCountBits specialization intentionally does not exist. template <> HeapObjectSideTableEntry* RefCounts::allocateSideTable() @@ -92,7 +92,7 @@ HeapObjectSideTableEntry* RefCounts::allocateSideTable() } else if (oldbits.getIsDeiniting()) { // Already past the start of deinit. Do nothing. - return nil; + return nullptr; } // Preflight passed. Allocate a side table. @@ -112,7 +112,7 @@ HeapObjectSideTableEntry* RefCounts::allocateSideTable() } else if (oldbits.getIsDeiniting()) { // Already past the start of deinit. Do nothing. - return nil; + return nullptr; } // FIXME: barriers? @@ -132,7 +132,7 @@ HeapObjectSideTableEntry* RefCounts::formWeakReference() if (side) return side->incrementWeak(); else - return nil; + return nullptr; } // namespace swift From 095a795287baa2538c6c734116dbb69003615c9d Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Mon, 17 Oct 2016 17:00:16 -0700 Subject: [PATCH 15/38] WIP: Fix 32-bit build. --- stdlib/public/SwiftShims/RefCount.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 830b1ac33e0c1..89de70a8e81a0 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -220,7 +220,7 @@ class RefCountBitsT { // Layout of bits. // field value = (bits & mask) >> shift -# define MaskForField(name) (((1UL< Date: Tue, 18 Oct 2016 17:01:28 -0700 Subject: [PATCH 16/38] WIP: Fix some whitespace style. --- stdlib/public/SwiftShims/RefCount.h | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 89de70a8e81a0..66df5475aa67c 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -275,7 +275,7 @@ class RefCountBitsT { // RefCountBits uses always_inline everywhere // to improve performance of debug builds. - private: + private: LLVM_ATTRIBUTE_ALWAYS_INLINE bool getUseSlowRC() const { return bool(GetField(UseSlowRC)); @@ -315,7 +315,7 @@ class RefCountBitsT { return (int64_t(bits) >= 0); } - public: + public: LLVM_ATTRIBUTE_ALWAYS_INLINE RefCountBitsT() = default; @@ -472,7 +472,7 @@ class SideTableRefCountBits : public RefCountBitsT { uint32_t weakBits; - public: + public: LLVM_ATTRIBUTE_ALWAYS_INLINE SideTableRefCountBits() = default; @@ -549,7 +549,7 @@ class RefCounts { LLVM_ATTRIBUTE_NOINLINE bool tryIncrementSlow(RefCountBits oldbits); - public: + public: enum Initialized_t { Initialized }; // RefCounts must be trivially constructible to avoid ObjC++ @@ -623,7 +623,7 @@ class RefCounts { bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementAndPinSlow(); - } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); return true; } @@ -655,7 +655,7 @@ class RefCounts { bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementSlow(oldbits); - } while (! refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); return true; } @@ -777,7 +777,7 @@ class RefCounts { // FIXME: make sure no-assert build optimizes this } - private: + private: // Second slow path of doDecrement, where the // object may have a side table entry. @@ -815,8 +815,8 @@ class RefCounts { if (clearPinnedFlag) newbits.setIsPinned(false); } - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); if (performDeinit && deinitNow) { std::atomic_thread_fence(acquire); _swift_release_dealloc(getHeapObject()); @@ -848,7 +848,7 @@ class RefCounts { return false; // don't deinit } - private: + private: // This is independently specialized below for inline and out-of-line use. template @@ -857,7 +857,7 @@ class RefCounts { // UNOWNED - public: + public: // Increment the unowned reference count. void incrementUnowned(uint32_t inc) { auto oldbits = refCounts.load(relaxed); @@ -896,8 +896,8 @@ class RefCounts { performFree = false; } // FIXME: underflow check? - } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + release, relaxed)); return performFree; } @@ -914,7 +914,7 @@ class RefCounts { // WEAK - public: + public: // Returns the object's side table entry (creating it if necessary) with // its weak ref count incremented. // Returns nullptr if the object is already deiniting. @@ -960,7 +960,7 @@ class RefCounts { } - private: + private: HeapObject *getHeapObject() const; HeapObjectSideTableEntry* allocateSideTable(); @@ -979,7 +979,7 @@ class HeapObjectSideTableEntry { std::atomic object; SideTableRefCounts refCounts; - public: + public: HeapObjectSideTableEntry(HeapObject *newObject) : object(newObject), refCounts() { } From 23eff3f33b67576ae2d884ab81f312ed99e711c8 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 3 Nov 2016 17:04:37 -0700 Subject: [PATCH 17/38] WIP: Introduce fake_memory_order_consume. Improve some memory barriers. --- stdlib/public/SwiftShims/RefCount.h | 130 +++++++++++++++------------- stdlib/public/runtime/RefCount.cpp | 12 ++- 2 files changed, 77 insertions(+), 65 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 66df5475aa67c..9928d6829866f 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -185,15 +185,16 @@ _swift_release_dealloc(swift::HeapObject *object) namespace swift { -// FIXME: many `relaxed` in this file should be `consume`, -// but (1) the compiler doesn't support `consume` directly, -// and (2) the compiler promotes `consume` to `acquire` instead which -// is overkill on our CPUs. But this might leave us vulnerable to -// compiler optimizations that `relaxed` allows but `consume` ought not. -#define relaxed std::memory_order_relaxed -#define acquire std::memory_order_acquire -#define release std::memory_order_release -#define consume std::memory_order_consume +// FIXME: some operations here should be memory_order_consume, +// but (1) the compiler doesn't support consume directly, +// and (2) the compiler implements consume as acquire which +// is unnecessarily slow on some of our CPUs. +// Such operations are written here as fake_memory_order_consume. +// We map them to memory_order_relaxed. This might leave us vulnerable to +// compiler optimizations. In addition, the other dependency-tracking +// annotations that would be required for real memory_order_consume +// are not present. +#define fake_memory_order_consume std::memory_order_relaxed // RefCountIsInline: refcount stored in an object @@ -354,10 +355,7 @@ class RefCountBitsT { LLVM_ATTRIBUTE_ALWAYS_INLINE HeapObjectSideTableEntry *getSideTable() const { assert(hasSideTable()); - // FIXME: overkill barrier? Otherwise technically need - // a consume re-load of the bits before dereferencing. - std::atomic_thread_fence(std::memory_order_acquire); - + // Stored value is a shifted pointer. // FIXME: Don't hard-code this shift amount? return reinterpret_cast @@ -527,6 +525,15 @@ class SideTableRefCountBits : public RefCountBitsT // an acquire fence is performed before beginning Swift deinit or ObjC // -dealloc code. This ensures that the deinit code sees all modifications // of the object's contents that were made before the object was released. +// +// Unowned and weak increment and decrement are all unordered. +// There is no deinit equivalent for these counts so no fence is needed. +// +// Accessing the side table requires that refCounts be accessed with +// a load-consume. Only code that is guaranteed not to try dereferencing +// the side table may perform a load-relaxed of refCounts. +// Similarly, storing the new side table pointer into refCounts is a +// store-release, but most other stores into refCounts are store-relaxed. template class RefCounts { @@ -562,7 +569,7 @@ class RefCounts { : refCounts(RefCountBits(0, 1)) { } void init() { - refCounts.store(RefCountBits(0, 1), relaxed); + refCounts.store(RefCountBits(0, 1), std::memory_order_relaxed); } // Initialize for a stack promoted object. This prevents that the final @@ -570,34 +577,35 @@ class RefCounts { // FIXME: need to mark these and assert they never get a side table, // because the extra unowned ref will keep the side table alive forever void initForNotFreeing() { - refCounts.store(RefCountBits(0, 2), relaxed); + refCounts.store(RefCountBits(0, 2), std::memory_order_relaxed); } // Initialize from another refcount bits. // Only inline -> out-of-line is allowed (used for new side table entries). void init(InlineRefCountBits newBits) { - refCounts.store(newBits, relaxed); + refCounts.store(newBits, std::memory_order_relaxed); } // Increment the reference count. void increment(uint32_t inc = 1) { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { newbits = oldbits; bool fast = newbits.incrementStrongExtraRefCount(inc); if (!fast) return incrementSlow(oldbits, inc); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); } void incrementNonAtomic(uint32_t inc = 1) { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); auto newbits = oldbits; bool fast = newbits.incrementStrongExtraRefCount(inc); if (!fast) return incrementNonAtomicSlow(oldbits, inc); - refCounts.store(newbits, relaxed); + refCounts.store(newbits, std::memory_order_relaxed); } // Try to simultaneously set the pinned flag and increment the @@ -610,7 +618,7 @@ class RefCounts { // // Postcondition: the flag is set. bool tryIncrementAndPin() { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { // If the flag is already set, just fail. @@ -623,12 +631,13 @@ class RefCounts { bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementAndPinSlow(); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); return true; } bool tryIncrementAndPinNonAtomic() { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); // If the flag is already set, just fail. if (!bits.hasSideTable() && bits.getIsPinned()) @@ -639,13 +648,13 @@ class RefCounts { bool fast = bits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementAndPinNonAtomicSlow(); - refCounts.store(bits, relaxed); + refCounts.store(bits, std::memory_order_relaxed); return true; } // Increment the reference count, unless the object is deiniting. bool tryIncrement() { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { if (!oldbits.hasSideTable() && oldbits.getIsDeiniting()) @@ -655,7 +664,8 @@ class RefCounts { bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) return tryIncrementSlow(oldbits); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); return true; } @@ -695,7 +705,7 @@ class RefCounts { // // Precondition: the reference count must be 1 void decrementFromOneNonAtomic() { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return bits.getSideTable()->decrementFromOneNonAtomic(); @@ -703,13 +713,13 @@ class RefCounts { assert(bits.getStrongExtraRefCount() == 0 && "Expect a refcount of 1"); bits.setStrongExtraRefCount(0); bits.setIsDeiniting(true); - refCounts.store(bits, relaxed); + refCounts.store(bits, std::memory_order_relaxed); } // Return the reference count. // Once deinit begins the reference count is undefined. uint32_t getCount() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return bits.getSideTable()->getCount(); @@ -720,7 +730,7 @@ class RefCounts { // Return whether the reference count is exactly 1. // Once deinit begins the reference count is undefined. bool isUniquelyReferenced() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return false; // FIXME: implement side table path if useful @@ -731,7 +741,7 @@ class RefCounts { // Return whether the reference count is exactly 1 or the pin flag // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return false; // FIXME: implement side table path if useful @@ -757,7 +767,7 @@ class RefCounts { // Return true if the object has started deiniting. bool isDeiniting() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return bits.getSideTable()->isDeiniting(); else @@ -772,8 +782,11 @@ class RefCounts { /// unowned reference count is 1 /// The object is assumed to be deiniting with no strong references already. bool canBeFreedNow() const { - auto bits = refCounts.load(relaxed); - return (!bits.hasSideTable() && bits.getIsDeiniting() && bits.getStrongExtraRefCount() == 0 && bits.getUnownedRefCount() == 1); + auto bits = refCounts.load(fake_memory_order_consume); + return (!bits.hasSideTable() && + bits.getIsDeiniting() && + bits.getStrongExtraRefCount() == 0 && + bits.getUnownedRefCount() == 1); // FIXME: make sure no-assert build optimizes this } @@ -816,9 +829,10 @@ class RefCounts { newbits.setIsPinned(false); } } while (!refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + std::memory_order_release, + std::memory_order_relaxed)); if (performDeinit && deinitNow) { - std::atomic_thread_fence(acquire); + std::atomic_thread_fence(std::memory_order_acquire); _swift_release_dealloc(getHeapObject()); } @@ -833,7 +847,7 @@ class RefCounts { // the caller because the compiler can optimize this arrangement better. template bool doDecrement(uint32_t dec) { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { @@ -843,7 +857,8 @@ class RefCounts { // Slow paths include side table; deinit; underflow return doDecrementSlow(oldbits, dec); } while (!refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + std::memory_order_release, + std::memory_order_relaxed)); return false; // don't deinit } @@ -860,7 +875,7 @@ class RefCounts { public: // Increment the unowned reference count. void incrementUnowned(uint32_t inc) { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { if (oldbits.hasSideTable()) @@ -870,13 +885,14 @@ class RefCounts { assert(newbits.getUnownedRefCount() != 0); newbits.incrementUnownedRefCount(inc); // FIXME: overflow check? - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); } // Decrement the unowned reference count. // Return true if the caller should free the object. bool decrementUnownedShouldFree(uint32_t dec) { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; bool performFree; @@ -897,14 +913,14 @@ class RefCounts { } // FIXME: underflow check? } while (!refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + std::memory_order_relaxed)); return performFree; } // Return unowned reference count. // Note that this is not equal to the number of outstanding unowned pointers. uint32_t getUnownedCount() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) return bits.getSideTable()->getUnownedCount(); else @@ -923,25 +939,27 @@ class RefCounts { // Increment the weak reference count. void incrementWeak() { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; do { newbits = oldbits; assert(newbits.getWeakRefCount() != 0); newbits.incrementWeakRefCount(); // FIXME: overflow check - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); } bool decrementWeakShouldCleanUp() { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); RefCountBits newbits; bool performFree; do { newbits = oldbits; - performFree = newbits.decrementWeakRefCount(); - } while (!refCounts.compare_exchange_weak(oldbits, newbits, relaxed)); + performFree = newbits.decrementWeakRefCount(); + } while (!refCounts.compare_exchange_weak(oldbits, newbits, + std::memory_order_relaxed)); return performFree; } @@ -949,7 +967,7 @@ class RefCounts { // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. uint32_t getWeakCount() const { - auto bits = refCounts.load(relaxed); + auto bits = refCounts.load(fake_memory_order_consume); if (bits.hasSideTable()) { return bits.getSideTable()->getWeakCount(); } else { @@ -976,6 +994,7 @@ static_assert(std::is_trivially_destructible::value, class HeapObjectSideTableEntry { + // FIXME: does object need to be atomic? std::atomic object; SideTableRefCounts refCounts; @@ -993,7 +1012,7 @@ class HeapObjectSideTableEntry { HeapObject* tryRetain() { if (refCounts.tryIncrement()) - return object.load(); // FIXME barrier + return object.load(std::memory_order_relaxed); else return nullptr; } @@ -1003,7 +1022,7 @@ class HeapObjectSideTableEntry { } HeapObject *unsafeGetObject() const { - return object.load(relaxed); + return object.load(std::memory_order_relaxed); } // STRONG @@ -1106,7 +1125,7 @@ inline bool RefCounts::doDecrementNonAtomic(uint32_t dec) { // Therefore there is no other thread that can be concurrently // manipulating this object's retain counts. - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); // Use slow path if we can't guarantee atomicity. if (oldbits.hasSideTable() || oldbits.getUnownedRefCount() != 1) @@ -1117,7 +1136,7 @@ inline bool RefCounts::doDecrementNonAtomic(uint32_t dec) { if (!fast) return doDecrementSlow(oldbits, dec); - refCounts.store(newbits, relaxed); + refCounts.store(newbits, std::memory_order_relaxed); return false; // don't deinit } @@ -1170,11 +1189,6 @@ HeapObject* RefCounts::getHeapObject() const { // for use by SWIFT_HEAPOBJECT_NON_OBJC_MEMBERS typedef swift::InlineRefCounts InlineRefCounts; -#undef relaxed -#undef acquire -#undef release -#undef consume - // __cplusplus #endif diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index fee5713f4b4dd..3b9b28cfdeaf2 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -12,10 +12,8 @@ #include "swift/Runtime/HeapObject.h" -#define relaxed std::memory_order_relaxed -#define acquire std::memory_order_acquire -#define release std::memory_order_release -#define consume std::memory_order_consume +// See note about memory_order_consume in SwiftShims/RefCount.h. +#define fake_memory_order_consume std::memory_order_relaxed namespace swift { @@ -83,7 +81,7 @@ template bool RefCounts::tryIncrementSlow(SideTableRefCou template <> HeapObjectSideTableEntry* RefCounts::allocateSideTable() { - auto oldbits = refCounts.load(relaxed); + auto oldbits = refCounts.load(fake_memory_order_consume); // Preflight failures before allocating a new side table. if (oldbits.hasSideTable()) { @@ -115,11 +113,11 @@ HeapObjectSideTableEntry* RefCounts::allocateSideTable() return nullptr; } - // FIXME: barriers? side->initRefCounts(oldbits); } while (! refCounts.compare_exchange_weak(oldbits, newbits, - release, relaxed)); + std::memory_order_release, + std::memory_order_relaxed)); return side; } From bdbb9bee4e4c3f7ff3aabe8b654ba7f931dec22b Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 3 Nov 2016 17:38:51 -0700 Subject: [PATCH 18/38] WIP: Hide a group of assertions behind an NDEBUG check. --- stdlib/public/SwiftShims/RefCount.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 9928d6829866f..4fd76d82b15f6 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -295,6 +295,7 @@ class RefCountBitsT { template LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT bool doDecrementStrongExtraRefCount(uint32_t dec) { +#ifndef NDEBUG if (!hasSideTable()) { // Can't check these assertions with side table present. @@ -309,6 +310,7 @@ class RefCountBitsT { assert(getStrongExtraRefCount() + 1 >= dec && "releasing reference whose refcount is already zero"); } +#endif uint64_t unpin = clearPinnedFlag ? (uint64_t(1) << IsPinnedShift) : 0; // This deliberately underflows by borrowing from the UseSlowRC field. From 4515d702d1473e9196aac29a1125970d4183f83f Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 3 Nov 2016 23:12:05 -0700 Subject: [PATCH 19/38] WIP: Compact 32-bit inline refcounts. --- stdlib/public/SwiftShims/HeapObject.h | 5 + stdlib/public/SwiftShims/RefCount.h | 273 +++++++++++++++++++------- 2 files changed, 204 insertions(+), 74 deletions(-) diff --git a/stdlib/public/SwiftShims/HeapObject.h b/stdlib/public/SwiftShims/HeapObject.h index 3c2f9175c1ce5..f3858c1963462 100644 --- a/stdlib/public/SwiftShims/HeapObject.h +++ b/stdlib/public/SwiftShims/HeapObject.h @@ -57,6 +57,11 @@ static_assert(swift::IsTriviallyConstructible::value, "HeapObject must be trivially initializable"); static_assert(std::is_trivially_destructible::value, "HeapObject must be trivially destructible"); +// FIXME: small header for 32-bit +//static_assert(sizeof(HeapObject) == 2*sizeof(void*), +// "HeapObject must be two pointers long"); +static_assert(alignof(HeapObject) == alignof(void*), + "HeapObject must be pointer-aligned"); } // end namespace swift #endif diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 4fd76d82b15f6..d8e2812311f34 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -206,54 +206,116 @@ enum ClearPinnedFlag { DontClearPinnedFlag = false, DoClearPinnedFlag = true }; enum PerformDeinit { DontPerformDeinit = false, DoPerformDeinit = true }; -// Basic encoding of refcount and flag data into the object's header. -// FIXME: Specialize this for a 32-bit field on 32-bit architectures. -template -class RefCountBitsT { +// Raw storage of refcount bits, depending on pointer size and inlinedness. +// 32-bit inline refcount is 32-bits. All others are 64-bits. - friend class RefCountBitsT; - friend class RefCountBitsT; - - static const RefCountInlinedness Inlinedness = refcountIsInline; - - uint64_t bits; +template +struct RefCountBitsInt; - // Layout of bits. - // field value = (bits & mask) >> shift - -# define MaskForField(name) (((1ULL< +struct RefCountBitsInt { + typedef uint64_t Type; + typedef int64_t SignedType; +}; - enum : uint64_t { - UnownedRefCountShift = 0, - UnownedRefCountBitCount = 32, - UnownedRefCountMask = MaskForField(UnownedRefCount), +// 32-bit out of line +template <> +struct RefCountBitsInt { + typedef uint64_t Type; + typedef int64_t SignedType; +}; - IsPinnedShift = ShiftAfterField(UnownedRefCount), - IsPinnedBitCount = 1, - IsPinnedMask = MaskForField(IsPinned), +// 32-bit inline +template <> +struct RefCountBitsInt { + typedef uint32_t Type; + typedef int32_t SignedType; +}; - IsDeinitingShift = ShiftAfterField(IsPinned), - IsDeinitingBitCount = 1, - IsDeinitingMask = MaskForField(IsDeiniting), - StrongExtraRefCountShift = ShiftAfterField(IsDeiniting), - StrongExtraRefCountBitCount = 29, - StrongExtraRefCountMask = MaskForField(StrongExtraRefCount), +// Layout of refcount bits. +// field value = (bits & mask) >> shift +// FIXME: redo this abstraction more cleanly + +# define maskForField(name) (((uint64_t(1)< +struct RefCountBitOffsets; - SideTableShift = 0, - SideTableBitCount = 62, - SideTableMask = MaskForField(SideTable), +// 64-bit inline +// 64-bit out of line +// 32-bit out of line +template <> +struct RefCountBitOffsets<8> { + static const size_t UnownedRefCountShift = 0; + static const size_t UnownedRefCountBitCount = 32; + static const uint64_t UnownedRefCountMask = maskForField(UnownedRefCount); + + static const size_t IsPinnedShift = shiftAfterField(UnownedRefCount); + static const size_t IsPinnedBitCount = 1; + static const uint64_t IsPinnedMask = maskForField(IsPinned); + + static const size_t IsDeinitingShift = shiftAfterField(IsPinned); + static const size_t IsDeinitingBitCount = 1; + static const uint64_t IsDeinitingMask = maskForField(IsDeiniting); + + static const size_t StrongExtraRefCountShift = shiftAfterField(IsDeiniting); + static const size_t StrongExtraRefCountBitCount = 29; + static const uint64_t StrongExtraRefCountMask = maskForField(StrongExtraRefCount); + + static const size_t UseSlowRCShift = shiftAfterField(StrongExtraRefCount); + static const size_t UseSlowRCBitCount = 1; + static const uint64_t UseSlowRCMask = maskForField(UseSlowRC); + + static const size_t SideTableShift = 0; + static const size_t SideTableBitCount = 62; + static const uint64_t SideTableMask = maskForField(SideTable); + static const size_t SideTableUnusedLowBits = 3; + + static const size_t SideTableMarkShift = SideTableBitCount; + static const size_t SideTableMarkBitCount = 1; + static const uint64_t SideTableMarkMask = maskForField(SideTableMark); +}; + +// 32-bit inline +template <> +struct RefCountBitOffsets<4> { + static const size_t UnownedRefCountShift = 0; + static const size_t UnownedRefCountBitCount = 8; + static const uint32_t UnownedRefCountMask = maskForField(UnownedRefCount); + + static const size_t IsPinnedShift = shiftAfterField(UnownedRefCount); + static const size_t IsPinnedBitCount = 1; + static const uint32_t IsPinnedMask = maskForField(IsPinned); + + static const size_t IsDeinitingShift = shiftAfterField(IsPinned); + static const size_t IsDeinitingBitCount = 1; + static const uint32_t IsDeinitingMask = maskForField(IsDeiniting); + + static const size_t StrongExtraRefCountShift = shiftAfterField(IsDeiniting); + static const size_t StrongExtraRefCountBitCount = 21; + static const uint32_t StrongExtraRefCountMask = maskForField(StrongExtraRefCount); + + static const size_t UseSlowRCShift = shiftAfterField(StrongExtraRefCount); + static const size_t UseSlowRCBitCount = 1; + static const uint32_t UseSlowRCMask = maskForField(UseSlowRC); + + static const size_t SideTableShift = 0; + static const size_t SideTableBitCount = 30; + static const uint32_t SideTableMask = maskForField(SideTable); + static const size_t SideTableUnusedLowBits = 2; + + static const size_t SideTableMarkShift = SideTableBitCount; + static const size_t SideTableMarkBitCount = 1; + static const uint32_t SideTableMarkMask = maskForField(SideTableMark); +}; - SideTableMarkShift = SideTableBitCount, - SideTableMarkBitCount = 1, - SideTableMarkMask = MaskForField(SideTableMark) - }; +/* + FIXME: reinstate these assertions static_assert(StrongExtraRefCountShift == IsDeinitingShift + 1, "IsDeiniting must be LSB-wards of StrongExtraRefCount"); static_assert(UseSlowRCShift + UseSlowRCBitCount == sizeof(bits)*8, @@ -265,13 +327,42 @@ class RefCountBitsT { IsDeinitingBitCount + StrongExtraRefCountBitCount + UseSlowRCBitCount == sizeof(bits)*8, "wrong bit count for RefCountBits refcount encoding"); -# undef MaskForField -# undef ShiftAfterField +*/ +# undef maskForField +# undef shiftAfterField -# define GetField(name) \ - ((bits & name##Mask) >> name##Shift) -# define SetField(name, val) \ - bits = (bits & ~name##Mask) | (((uint64_t(val) << name##Shift) & name##Mask)) + +// Basic encoding of refcount and flag data into the object's header. +template +class RefCountBitsT { + + friend class RefCountBitsT; + friend class RefCountBitsT; + + static const RefCountInlinedness Inlinedness = refcountIsInline; + + typedef typename RefCountBitsInt::Type + BitsType; + typedef typename RefCountBitsInt::SignedType + SignedBitsType; + typedef RefCountBitOffsets + Offsets; + + BitsType bits; + + // "Bitfield" accessors. + +# define getFieldIn(bits, offsets, name) \ + ((bits & offsets::name##Mask) >> offsets::name##Shift) +# define setFieldIn(bits, offsets, name, val) \ + bits = ((bits & ~offsets::name##Mask) | \ + (((BitsType(val) << offsets::name##Shift) & offsets::name##Mask))) + +# define getField(name) getFieldIn(bits, Offsets, name) +# define setField(name, val) setFieldIn(bits, Offsets, name, val) +# define copyFieldFrom(src, name) \ + setFieldIn(bits, Offsets, name, \ + getFieldIn(src.bits, decltype(src)::Offsets, name)) // RefCountBits uses always_inline everywhere // to improve performance of debug builds. @@ -279,12 +370,12 @@ class RefCountBitsT { private: LLVM_ATTRIBUTE_ALWAYS_INLINE bool getUseSlowRC() const { - return bool(GetField(UseSlowRC)); + return bool(getField(UseSlowRC)); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setUseSlowRC(bool value) { - SetField(UseSlowRC, value); + setField(UseSlowRC, value); } @@ -312,10 +403,12 @@ class RefCountBitsT { } #endif - uint64_t unpin = clearPinnedFlag ? (uint64_t(1) << IsPinnedShift) : 0; + BitsType unpin = (clearPinnedFlag + ? (BitsType(1) << Offsets::IsPinnedShift) + : 0); // This deliberately underflows by borrowing from the UseSlowRC field. - bits -= unpin + (uint64_t(dec) << StrongExtraRefCountShift); - return (int64_t(bits) >= 0); + bits -= unpin + (BitsType(dec) << Offsets::StrongExtraRefCountShift); + return (SignedBitsType(bits) >= 0); } public: @@ -326,21 +419,40 @@ class RefCountBitsT { LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr RefCountBitsT(uint32_t strongExtraCount, uint32_t unownedCount) - : bits((uint64_t(strongExtraCount) << StrongExtraRefCountShift) | - (uint64_t(unownedCount) << UnownedRefCountShift)) + : bits((BitsType(strongExtraCount) << Offsets::StrongExtraRefCountShift) | + (BitsType(unownedCount) << Offsets::UnownedRefCountShift)) { } LLVM_ATTRIBUTE_ALWAYS_INLINE RefCountBitsT(HeapObjectSideTableEntry* side) - : bits((reinterpret_cast(side) >> 3) | - (1ULL << UseSlowRCShift) | - (1ULL << SideTableMarkShift)) + : bits((reinterpret_cast(side) >> Offsets::SideTableUnusedLowBits) + | (BitsType(1) << Offsets::UseSlowRCShift) + | (BitsType(1) << Offsets::SideTableMarkShift)) { assert(refcountIsInline); } LLVM_ATTRIBUTE_ALWAYS_INLINE - RefCountBitsT(RefCountBitsT newbits) : bits(newbits.bits) { } + RefCountBitsT(RefCountBitsT newbits) { + bits = 0; + + if (refcountIsInline || sizeof(newbits) == sizeof(*this)) { + // this and newbits are both inline + // OR this is out-of-line but the same layout as inline. + // (FIXME: use something cleaner than sizeof for same-layout test) + // Copy the bits directly. + bits = newbits.bits; + } + else { + // this is out-of-line and not the same layout as inline newbits. + // Copy field-by-field. + copyFieldFrom(newbits, UnownedRefCount); + copyFieldFrom(newbits, IsPinned); + copyFieldFrom(newbits, IsDeiniting); + copyFieldFrom(newbits, StrongExtraRefCount); + copyFieldFrom(newbits, UseSlowRC); + } + } LLVM_ATTRIBUTE_ALWAYS_INLINE bool hasSideTable() const { @@ -359,33 +471,32 @@ class RefCountBitsT { assert(hasSideTable()); // Stored value is a shifted pointer. - // FIXME: Don't hard-code this shift amount? return reinterpret_cast - (uintptr_t(GetField(SideTable)) << 3); + (uintptr_t(getField(SideTable)) << Offsets::SideTableUnusedLowBits); } LLVM_ATTRIBUTE_ALWAYS_INLINE uint32_t getUnownedRefCount() const { assert(!hasSideTable()); - return uint32_t(GetField(UnownedRefCount)); + return uint32_t(getField(UnownedRefCount)); } LLVM_ATTRIBUTE_ALWAYS_INLINE bool getIsPinned() const { assert(!hasSideTable()); - return bool(GetField(IsPinned)); + return bool(getField(IsPinned)); } LLVM_ATTRIBUTE_ALWAYS_INLINE bool getIsDeiniting() const { assert(!hasSideTable()); - return bool(GetField(IsDeiniting)); + return bool(getField(IsDeiniting)); } LLVM_ATTRIBUTE_ALWAYS_INLINE uint32_t getStrongExtraRefCount() const { assert(!hasSideTable()); - return uint32_t(GetField(StrongExtraRefCount)); + return uint32_t(getField(StrongExtraRefCount)); } @@ -399,36 +510,35 @@ class RefCountBitsT { void setSideTable(HeapObjectSideTableEntry *side) { assert(hasSideTable()); // Stored value is a shifted pointer. - // FIXME: Don't hard-code this shift amount? uintptr_t value = reinterpret_cast(side); - uintptr_t storedValue = value >> 3; - assert(storedValue << 3 == value); - SetField(SideTable, storedValue); - SetField(SideTableMark, 1); + uintptr_t storedValue = value >> Offsets::SideTableUnusedLowBits; + assert(storedValue << Offsets::SideTableUnusedLowBits == value); + setField(SideTable, storedValue); + setField(SideTableMark, 1); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setUnownedRefCount(uint32_t value) { assert(!hasSideTable()); - SetField(UnownedRefCount, value); + setField(UnownedRefCount, value); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setIsPinned(bool value) { assert(!hasSideTable()); - SetField(IsPinned, value); + setField(IsPinned, value); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setIsDeiniting(bool value) { assert(!hasSideTable()); - SetField(IsDeiniting, value); + setField(IsDeiniting, value); } LLVM_ATTRIBUTE_ALWAYS_INLINE void setStrongExtraRefCount(uint32_t value) { assert(!hasSideTable()); - SetField(StrongExtraRefCount, value); + setField(StrongExtraRefCount, value); } @@ -438,8 +548,8 @@ class RefCountBitsT { LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT bool incrementStrongExtraRefCount(uint32_t inc) { // This deliberately overflows into the UseSlowRC field. - bits += uint64_t(inc) << StrongExtraRefCountShift; - return (int64_t(bits) >= 0); + bits += BitsType(inc) << Offsets::StrongExtraRefCountShift; + return (SignedBitsType(bits) >= 0); } // FIXME: I don't understand why I can't make clearPinned a template argument @@ -462,8 +572,11 @@ class RefCountBitsT { setUnownedRefCount(getUnownedRefCount() - dec); } -# undef GetField -# undef SetField +# undef getFieldIn +# undef setFieldIn +# undef getField +# undef setField +# undef copyFieldFrom }; typedef RefCountBitsT InlineRefCountBits; @@ -540,6 +653,11 @@ class SideTableRefCountBits : public RefCountBitsT template class RefCounts { std::atomic refCounts; +#if !__LP64__ + // FIXME: hack - something somewhere is assuming a 3-word header on 32-bit + // See also other fixmes marked "small header for 32-bit" + uintptr_t unused __attribute__((unavailable)); +#endif // Out-of-line slow paths. @@ -994,6 +1112,13 @@ static_assert(swift::IsTriviallyConstructible::value, static_assert(std::is_trivially_destructible::value, "InlineRefCounts must be trivially destructible"); +/* FIXME: small header for 32-bit +static_assert(sizeof(InlineRefCounts) == sizeof(uintptr_t), + "InlineRefCounts must be pointer-sized"); +static_assert(alignof(InlineRefCounts) == alignof(uintptr_t), +"InlineRefCounts must be pointer-aligned"); +*/ + class HeapObjectSideTableEntry { // FIXME: does object need to be atomic? From 88882d940a9633ffc8d41c3c32c8dce8b3130fd3 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Fri, 9 Dec 2016 20:02:38 -0800 Subject: [PATCH 20/38] Formatting --- stdlib/public/runtime/HeapObject.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index f5e0be506cd42..fab26041ae30e 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -245,9 +245,8 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_retain_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object) { + if (object) object->refCounts.increment(n); - } } SWIFT_RT_ENTRY_VISIBILITY @@ -261,9 +260,8 @@ SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" void SWIFT_RT_ENTRY_IMPL(swift_nonatomic_retain_n)(HeapObject *object, uint32_t n) SWIFT_CC(RegisterPreservingCC_IMPL) { - if (object) { + if (object) object->refCounts.incrementNonAtomic(n); - } } SWIFT_RT_ENTRY_VISIBILITY @@ -386,9 +384,8 @@ HeapObject *swift::swift_tryPin(HeapObject *object) // Try to set the flag. If this succeeds, the caller will be // responsible for clearing it. - if (object->refCounts.tryIncrementAndPin()) { + if (object->refCounts.tryIncrementAndPin()) return object; - } // If setting the flag failed, it's because it was already set. // Return nil so that the object will be deallocated later. @@ -415,9 +412,8 @@ HeapObject *swift::swift_nonatomic_tryPin(HeapObject *object) // Try to set the flag. If this succeeds, the caller will be // responsible for clearing it. - if (object->refCounts.tryIncrementAndPinNonAtomic()) { + if (object->refCounts.tryIncrementAndPinNonAtomic()) return object; - } // If setting the flag failed, it's because it was already set. // Return nil so that the object will be deallocated later. @@ -451,7 +447,8 @@ bool swift_isDeallocating(HeapObject *object) { SWIFT_RT_ENTRY_IMPL_VISIBILITY extern "C" bool SWIFT_RT_ENTRY_IMPL(swift_isDeallocating)(HeapObject *object) { - if (!object) return false; + if (!object) + return false; return object->refCounts.isDeiniting(); } From 1af09fea3cafc04c9563b4fecec0a0c6b5965a62 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Fri, 9 Dec 2016 20:44:55 -0800 Subject: [PATCH 21/38] Restore optimization of isUniquelyReferenced and isUniquelyReferencedOrPinned. --- stdlib/public/SwiftShims/RefCount.h | 133 ++++++++++++++++++++-------- unittests/runtime/Refcounting.cpp | 40 +++++++++ 2 files changed, 135 insertions(+), 38 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index b833c85bbecbc..682c97721e8c1 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -44,7 +44,8 @@ typedef struct { The strong RC counts strong references to the object. When the strong RC reaches zero the object is deinited, unowned reference reads become errors, - and weak reference reads become nil. + and weak reference reads become nil. The strong RC is stored as an extra + count: when the physical field is 0 the logical value is 1. The unowned RC counts unowned references to the object. The unowned RC also has an extra +1 on behalf of the strong references; this +1 is @@ -250,20 +251,20 @@ struct RefCountBitOffsets; // 32-bit out of line template <> struct RefCountBitOffsets<8> { - static const size_t UnownedRefCountShift = 0; - static const size_t UnownedRefCountBitCount = 32; - static const uint64_t UnownedRefCountMask = maskForField(UnownedRefCount); - - static const size_t IsPinnedShift = shiftAfterField(UnownedRefCount); + static const size_t IsPinnedShift = 0; static const size_t IsPinnedBitCount = 1; static const uint64_t IsPinnedMask = maskForField(IsPinned); - static const size_t IsDeinitingShift = shiftAfterField(IsPinned); + static const size_t UnownedRefCountShift = shiftAfterField(IsPinned); + static const size_t UnownedRefCountBitCount = 31; + static const uint64_t UnownedRefCountMask = maskForField(UnownedRefCount); + + static const size_t IsDeinitingShift = shiftAfterField(UnownedRefCount); static const size_t IsDeinitingBitCount = 1; static const uint64_t IsDeinitingMask = maskForField(IsDeiniting); static const size_t StrongExtraRefCountShift = shiftAfterField(IsDeiniting); - static const size_t StrongExtraRefCountBitCount = 29; + static const size_t StrongExtraRefCountBitCount = 30; static const uint64_t StrongExtraRefCountMask = maskForField(StrongExtraRefCount); static const size_t UseSlowRCShift = shiftAfterField(StrongExtraRefCount); @@ -283,20 +284,20 @@ struct RefCountBitOffsets<8> { // 32-bit inline template <> struct RefCountBitOffsets<4> { - static const size_t UnownedRefCountShift = 0; - static const size_t UnownedRefCountBitCount = 8; - static const uint32_t UnownedRefCountMask = maskForField(UnownedRefCount); - - static const size_t IsPinnedShift = shiftAfterField(UnownedRefCount); + static const size_t IsPinnedShift = 0; static const size_t IsPinnedBitCount = 1; static const uint32_t IsPinnedMask = maskForField(IsPinned); - static const size_t IsDeinitingShift = shiftAfterField(IsPinned); + static const size_t UnownedRefCountShift = shiftAfterField(IsPinned); + static const size_t UnownedRefCountBitCount = 7; + static const uint32_t UnownedRefCountMask = maskForField(UnownedRefCount); + + static const size_t IsDeinitingShift = shiftAfterField(UnownedRefCount); static const size_t IsDeinitingBitCount = 1; static const uint32_t IsDeinitingMask = maskForField(IsDeiniting); static const size_t StrongExtraRefCountShift = shiftAfterField(IsDeiniting); - static const size_t StrongExtraRefCountBitCount = 21; + static const size_t StrongExtraRefCountBitCount = 22; static const uint32_t StrongExtraRefCountMask = maskForField(StrongExtraRefCount); static const size_t UseSlowRCShift = shiftAfterField(StrongExtraRefCount); @@ -328,8 +329,6 @@ struct RefCountBitOffsets<4> { UseSlowRCBitCount == sizeof(bits)*8, "wrong bit count for RefCountBits refcount encoding"); */ -# undef maskForField -# undef shiftAfterField // Basic encoding of refcount and flag data into the object's header. @@ -571,7 +570,74 @@ class RefCountBitsT { void decrementUnownedRefCount(uint32_t dec) { setUnownedRefCount(getUnownedRefCount() - dec); } - + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isUniquelyReferenced() { + static_assert(Offsets::IsPinnedBitCount + + Offsets::UnownedRefCountBitCount + + Offsets::IsDeinitingBitCount + + Offsets::StrongExtraRefCountBitCount + + Offsets::UseSlowRCBitCount == sizeof(bits)*8, + "inspect isUniquelyReferenced after adding fields"); + + // isPinned: don't care + // Unowned: don't care (FIXME: should care and redo initForNotFreeing) + // IsDeiniting: false + // StrongExtra: 0 + // UseSlowRC: false + + // Compiler is clever enough to optimize this. + return + !getUseSlowRC() && !getIsDeiniting() && getStrongExtraRefCount() == 0; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isUniquelyReferencedOrPinned() { + static_assert(Offsets::IsPinnedBitCount + + Offsets::UnownedRefCountBitCount + + Offsets::IsDeinitingBitCount + + Offsets::StrongExtraRefCountBitCount + + Offsets::UseSlowRCBitCount == sizeof(bits)*8, + "inspect isUniquelyReferencedOrPinned after adding fields"); + + // isPinned: don't care + // Unowned: don't care (FIXME: should care and redo initForNotFreeing) + // IsDeiniting: false + // isPinned/StrongExtra: true/any OR false/0 + // UseSlowRC: false + + // Compiler is not clever enough to optimize this. + // return (isUniquelyReferenced() || + // (!getUseSlowRC() && !getIsDeiniting() && getIsPinned())); + + // Bit twiddling solution: + // 1. Define the fields in this order: + // bits that must be zero when not pinned | bits to ignore | IsPinned + // 2. Rotate IsPinned into the sign bit: + // IsPinned | bits that must be zero when not pinned | bits to ignore + // 3. Perform a signed comparison against X = (1 << count of ignored bits). + // IsPinned makes the value negative and thus less than X. + // Zero in the must-be-zero bits makes the value less than X. + // Non-zero and not pinned makes the value greater or equal to X. + + // Count the ignored fields. + constexpr auto ignoredBitsCount = + Offsets::UnownedRefCountBitCount + Offsets::IsDeinitingBitCount; + // Make sure all fields are positioned as expected. + // -1 compensates for the rotation. + static_assert(Offsets::IsPinnedShift == 0, "IsPinned must be the LSB bit"); + static_assert( + shiftAfterField(Offsets::UnownedRefCount)-1 <= ignoredBitsCount && + shiftAfterField(Offsets::IsDeiniting)-1 <= ignoredBitsCount && + Offsets::StrongExtraRefCountShift-1 >= ignoredBitsCount && + Offsets::UseSlowRCShift-1 >= ignoredBitsCount, + "refcount bit layout incorrect for isUniquelyReferencedOrPinned"); + + BitsType X = BitsType(1) << ignoredBitsCount; + BitsType rotatedBits = ((bits >> 1) | (bits << (8*sizeof(bits) - 1))); + return SignedBitsType(rotatedBits) < SignedBitsType(X); + } + # undef getFieldIn # undef setFieldIn # undef getField @@ -579,6 +645,9 @@ class RefCountBitsT { # undef copyFieldFrom }; +# undef maskForField +# undef shiftAfterField + typedef RefCountBitsT InlineRefCountBits; class SideTableRefCountBits : public RefCountBitsT @@ -855,34 +924,22 @@ class RefCounts { return false; // FIXME: implement side table path if useful assert(!bits.getIsDeiniting()); - return bits.getStrongExtraRefCount() == 0; + return bits.isUniquelyReferenced(); } // Return whether the reference count is exactly 1 or the pin flag // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { auto bits = refCounts.load(fake_memory_order_consume); - if (bits.hasSideTable()) - return false; // FIXME: implement side table path if useful + // FIXME: implement side table path if useful + // In the meantime we don't check it here. + // bits.isUniquelyReferencedOrPinned() checks it too, + // and the compiler optimizer does better if this check is not here. + // if (bits.hasSideTable()) + // return false; assert(!bits.getIsDeiniting()); - return (bits.getStrongExtraRefCount() == 0 || bits.getIsPinned()); - - // FIXME: check if generated code is efficient. - // We can't use the old rotate implementation below because - // the strong refcount field is now biased. - - // Rotating right by one sets the sign bit to the pinned bit. After - // rotation, the dealloc flag is the least significant bit followed by the - // reference count. A reference count of two or higher means that our value - // is bigger than 3 if the pinned bit is not set. If the pinned bit is set - // the value is negative. - // Note: Because we are using the sign bit for testing pinnedness it - // is important to do a signed comparison below. - // static_assert(RC_PINNED_FLAG == 1, - // "The pinned flag must be the lowest bit"); - // auto rotateRightByOne = ((value >> 1) | (value << 31)); - // return (int32_t)rotateRightByOne < (int32_t)RC_ONE; + return bits.isUniquelyReferencedOrPinned(); } // Return true if the object has started deiniting. diff --git a/unittests/runtime/Refcounting.cpp b/unittests/runtime/Refcounting.cpp index 363aef94f8c42..e4e43d487d146 100644 --- a/unittests/runtime/Refcounting.cpp +++ b/unittests/runtime/Refcounting.cpp @@ -153,6 +153,46 @@ TEST(RefcountingTest, unowned_retain_release_n) { EXPECT_EQ(1u, value); } +TEST(RefcountingTest, isUniquelyReferenced) { + size_t value = 0; + auto object = allocTestObject(&value, 1); + EXPECT_EQ(0u, value); + EXPECT_TRUE(swift_isUniquelyReferenced_nonNull_native(object)); + + swift_retain(object); + EXPECT_FALSE(swift_isUniquelyReferenced_nonNull_native(object)); + + swift_release(object); + EXPECT_TRUE(swift_isUniquelyReferenced_nonNull_native(object)); + + swift_release(object); + EXPECT_EQ(1u, value); +} + +TEST(RefcountingTest, isUniquelyReferencedOrPinned) { + size_t value = 0; + auto object = allocTestObject(&value, 1); + EXPECT_EQ(0u, value); + // RC 1, unpinned + EXPECT_TRUE(swift_isUniquelyReferencedOrPinned_nonNull_native(object)); + + swift_retain(object); + // RC big, unpinned + EXPECT_FALSE(swift_isUniquelyReferencedOrPinned_nonNull_native(object)); + + auto pinResult = swift_tryPin(object); + // RC big, pinned + EXPECT_TRUE(swift_isUniquelyReferencedOrPinned_nonNull_native(object)); + + swift_release(object); + // RC 1, pinned + EXPECT_TRUE(swift_isUniquelyReferencedOrPinned_nonNull_native(object)); + + swift_unpin(object); + swift_release(object); + EXPECT_EQ(1u, value); +} + ///////////////////////////////////////// // Non-atomic reference counting tests // ///////////////////////////////////////// From 963b9cf16851d8927c86b1d962f55b4d8b369b34 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Tue, 13 Dec 2016 14:45:08 -0800 Subject: [PATCH 22/38] fake_memory_order_consume -> SWIFT_MEMORY_ORDER_CONSUME --- include/swift/Runtime/Metadata.h | 2 +- stdlib/public/SwiftShims/RefCount.h | 50 +++++++++++------------------ stdlib/public/runtime/RefCount.cpp | 6 +--- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/include/swift/Runtime/Metadata.h b/include/swift/Runtime/Metadata.h index 18823fb6fbd48..7438b518f2e0e 100644 --- a/include/swift/Runtime/Metadata.h +++ b/include/swift/Runtime/Metadata.h @@ -1832,7 +1832,7 @@ using ObjCClassWrapperMetadata // is formally UB by C++11 language rules, we should be OK because neither // the processor model nor the optimizer can realistically reorder our uses // of 'consume'. -#if __arm64__ +#if __arm64__ || __arm__ # define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed) #else # define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 682c97721e8c1..e64a09b7e9b3b 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -186,18 +186,6 @@ _swift_release_dealloc(swift::HeapObject *object) namespace swift { -// FIXME: some operations here should be memory_order_consume, -// but (1) the compiler doesn't support consume directly, -// and (2) the compiler implements consume as acquire which -// is unnecessarily slow on some of our CPUs. -// Such operations are written here as fake_memory_order_consume. -// We map them to memory_order_relaxed. This might leave us vulnerable to -// compiler optimizations. In addition, the other dependency-tracking -// annotations that would be required for real memory_order_consume -// are not present. -#define fake_memory_order_consume std::memory_order_relaxed - - // RefCountIsInline: refcount stored in an object // RefCountNotInline: refcount stored in an object's side table entry enum RefCountInlinedness { RefCountNotInline = false, RefCountIsInline = true }; @@ -777,7 +765,7 @@ class RefCounts { // Increment the reference count. void increment(uint32_t inc = 1) { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { newbits = oldbits; @@ -789,7 +777,7 @@ class RefCounts { } void incrementNonAtomic(uint32_t inc = 1) { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); auto newbits = oldbits; bool fast = newbits.incrementStrongExtraRefCount(inc); if (!fast) @@ -807,7 +795,7 @@ class RefCounts { // // Postcondition: the flag is set. bool tryIncrementAndPin() { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { // If the flag is already set, just fail. @@ -826,7 +814,7 @@ class RefCounts { } bool tryIncrementAndPinNonAtomic() { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // If the flag is already set, just fail. if (!bits.hasSideTable() && bits.getIsPinned()) @@ -843,7 +831,7 @@ class RefCounts { // Increment the reference count, unless the object is deiniting. bool tryIncrement() { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { if (!oldbits.hasSideTable() && oldbits.getIsDeiniting()) @@ -894,7 +882,7 @@ class RefCounts { // // Precondition: the reference count must be 1 void decrementFromOneNonAtomic() { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) return bits.getSideTable()->decrementFromOneNonAtomic(); @@ -908,7 +896,7 @@ class RefCounts { // Return the reference count. // Once deinit begins the reference count is undefined. uint32_t getCount() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) return bits.getSideTable()->getCount(); @@ -919,7 +907,7 @@ class RefCounts { // Return whether the reference count is exactly 1. // Once deinit begins the reference count is undefined. bool isUniquelyReferenced() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) return false; // FIXME: implement side table path if useful @@ -930,7 +918,7 @@ class RefCounts { // Return whether the reference count is exactly 1 or the pin flag // is set. Once deinit begins the reference count is undefined. bool isUniquelyReferencedOrPinned() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // FIXME: implement side table path if useful // In the meantime we don't check it here. // bits.isUniquelyReferencedOrPinned() checks it too, @@ -944,7 +932,7 @@ class RefCounts { // Return true if the object has started deiniting. bool isDeiniting() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) return bits.getSideTable()->isDeiniting(); else @@ -959,7 +947,7 @@ class RefCounts { /// unowned reference count is 1 /// The object is assumed to be deiniting with no strong references already. bool canBeFreedNow() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); return (!bits.hasSideTable() && bits.getIsDeiniting() && bits.getStrongExtraRefCount() == 0 && @@ -1024,7 +1012,7 @@ class RefCounts { // the caller because the compiler can optimize this arrangement better. template bool doDecrement(uint32_t dec) { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { @@ -1052,7 +1040,7 @@ class RefCounts { public: // Increment the unowned reference count. void incrementUnowned(uint32_t inc) { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { if (oldbits.hasSideTable()) @@ -1069,7 +1057,7 @@ class RefCounts { // Decrement the unowned reference count. // Return true if the caller should free the object. bool decrementUnownedShouldFree(uint32_t dec) { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; bool performFree; @@ -1097,7 +1085,7 @@ class RefCounts { // Return unowned reference count. // Note that this is not equal to the number of outstanding unowned pointers. uint32_t getUnownedCount() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) return bits.getSideTable()->getUnownedCount(); else @@ -1116,7 +1104,7 @@ class RefCounts { // Increment the weak reference count. void incrementWeak() { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; do { newbits = oldbits; @@ -1128,7 +1116,7 @@ class RefCounts { } bool decrementWeakShouldCleanUp() { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); RefCountBits newbits; bool performFree; @@ -1144,7 +1132,7 @@ class RefCounts { // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. uint32_t getWeakCount() const { - auto bits = refCounts.load(fake_memory_order_consume); + auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); if (bits.hasSideTable()) { return bits.getSideTable()->getWeakCount(); } else { @@ -1309,7 +1297,7 @@ inline bool RefCounts::doDecrementNonAtomic(uint32_t dec) { // Therefore there is no other thread that can be concurrently // manipulating this object's retain counts. - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // Use slow path if we can't guarantee atomicity. if (oldbits.hasSideTable() || oldbits.getUnownedRefCount() != 1) diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index 3b9b28cfdeaf2..1e7e88701f692 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -12,10 +12,6 @@ #include "swift/Runtime/HeapObject.h" -// See note about memory_order_consume in SwiftShims/RefCount.h. -#define fake_memory_order_consume std::memory_order_relaxed - - namespace swift { template @@ -81,7 +77,7 @@ template bool RefCounts::tryIncrementSlow(SideTableRefCou template <> HeapObjectSideTableEntry* RefCounts::allocateSideTable() { - auto oldbits = refCounts.load(fake_memory_order_consume); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // Preflight failures before allocating a new side table. if (oldbits.hasSideTable()) { From de7e517883b6b175bd6aa120764794cbf3efee1f Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 15:40:52 -0800 Subject: [PATCH 23/38] fake_memory_order_consume -> SWIFT_MEMORY_ORDER_CONSUME --- stdlib/public/SwiftShims/RefCount.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index e64a09b7e9b3b..5a2d0c4f44d7d 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -37,6 +37,17 @@ typedef struct { #include "swift/Runtime/Config.h" #include "swift/Runtime/Debug.h" +// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require +// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed' +// is formally UB by C++11 language rules, we should be OK because neither +// the processor model nor the optimizer can realistically reorder our uses +// of 'consume'. +#if __arm64__ || __arm__ +# define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed) +#else +# define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume) +#endif + /* An object conceptually has three refcounts. These refcounts are stored either "inline" in the field following the isa From 596a3a0bf15d1f0d1e0cf57b9b8236e9b9232ea4 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 16:15:38 -0800 Subject: [PATCH 24/38] Undo unnecessary diffs. --- include/swift/Runtime/HeapObject.h | 274 +++++++++++++++++------------ 1 file changed, 158 insertions(+), 116 deletions(-) diff --git a/include/swift/Runtime/HeapObject.h b/include/swift/Runtime/HeapObject.h index 3f3371eca8252..efa4c68fb15b2 100644 --- a/include/swift/Runtime/HeapObject.h +++ b/include/swift/Runtime/HeapObject.h @@ -618,7 +618,6 @@ static inline void swift_unownedTakeAssign(UnownedReference *dest, swift_unownedRelease(oldValue); } - /*****************************************************************************/ /****************************** WEAK REFERENCES ******************************/ /*****************************************************************************/ @@ -691,121 +690,6 @@ extern "C" void swift_weakCopyAssign(WeakReference *dest, WeakReference *src); SWIFT_RUNTIME_EXPORT extern "C" void swift_weakTakeAssign(WeakReference *dest, WeakReference *src); - -/*****************************************************************************/ -/************************** UNKNOWN WEAK REFERENCES **************************/ -/*****************************************************************************/ - -#if SWIFT_OBJC_INTEROP - -/// Initialize a weak reference. -/// -/// \param ref - never null -/// \param value - not necessarily a native Swift object; can be null -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakInit(WeakReference *ref, void *value); - -/// Assign a new value to a weak reference. -/// -/// \param ref - never null -/// \param value - not necessarily a native Swift object; can be null -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakAssign(WeakReference *ref, void *value); - -/// Load a value from a weak reference, much like swift_weakLoadStrong -/// but without requiring the variable to refer to a native Swift object. -/// -/// \param ref - never null -/// \return can be null -SWIFT_RUNTIME_EXPORT -extern "C" void *swift_unknownWeakLoadStrong(WeakReference *ref); - -/// Load a value from a weak reference as if by -/// swift_unknownWeakLoadStrong, but leaving the reference in an -/// uninitialized state. -/// -/// \param ref - never null -/// \return can be null -SWIFT_RUNTIME_EXPORT -extern "C" void *swift_unknownWeakTakeStrong(WeakReference *ref); - -/// Destroy a weak reference variable that might not refer to a native -/// Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakDestroy(WeakReference *object); - -/// Copy-initialize a weak reference variable from one that might not -/// refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakCopyInit(WeakReference *dest, - WeakReference *src); - -/// Take-initialize a weak reference variable from one that might not -/// refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakTakeInit(WeakReference *dest, - WeakReference *src); - -/// Copy-assign a weak reference variable from another when either -/// or both variables might not refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakCopyAssign(WeakReference *dest, - WeakReference *src); - -/// Take-assign a weak reference variable from another when either -/// or both variables might not refer to a native Swift object. -SWIFT_RUNTIME_EXPORT -extern "C" void swift_unknownWeakTakeAssign(WeakReference *dest, - WeakReference *src); - -// SWIFT_OBJC_INTEROP -#else -// not SWIFT_OBJC_INTEROP - -static inline void swift_unknownWeakInit(WeakReference *ref, void *value) { - swift_weakInit(ref, static_cast(value)); -} - -static inline void swift_unknownWeakAssign(WeakReference *ref, void *value) { - swift_weakAssign(ref, static_cast(value)); -} - -static inline void *swift_unknownWeakLoadStrong(WeakReference *ref) { - return static_cast(swift_weakLoadStrong(ref)); -} - -static inline void *swift_unknownWeakTakeStrong(WeakReference *ref) { - return static_cast(swift_weakTakeStrong(ref)); -} - -static inline void swift_unknownWeakDestroy(WeakReference *object) { - swift_weakDestroy(object); -} - -static inline void swift_unknownWeakCopyInit(WeakReference *dest, - WeakReference *src) { - swift_weakCopyInit(dest, src); -} - -static inline void swift_unknownWeakTakeInit(WeakReference *dest, - WeakReference *src) { - swift_weakTakeInit(dest, src); -} - -static inline void swift_unknownWeakCopyAssign(WeakReference *dest, - WeakReference *src) { - swift_weakCopyAssign(dest, src); -} - -static inline void swift_unknownWeakTakeAssign(WeakReference *dest, - WeakReference *src) { - swift_weakTakeAssign(dest, src); -} - -// not SWIFT_OBJC_INTEROP -#endif - - /*****************************************************************************/ /************************* OTHER REFERENCE-COUNTING **************************/ /*****************************************************************************/ @@ -945,6 +829,164 @@ static inline void swift_nonatomic_unknownRelease_n(void *value, int n) #endif /* SWIFT_OBJC_INTEROP */ +/*****************************************************************************/ +/************************** UNKNOWN WEAK REFERENCES **************************/ +/*****************************************************************************/ + +#if SWIFT_OBJC_INTEROP + +/// Initialize a weak reference. +/// +/// \param ref - never null +/// \param value - not necessarily a native Swift object; can be null +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakInit(WeakReference *ref, void *value); + +#else + +static inline void swift_unknownWeakInit(WeakReference *ref, void *value) { + swift_weakInit(ref, static_cast(value)); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Assign a new value to a weak reference. +/// +/// \param ref - never null +/// \param value - not necessarily a native Swift object; can be null +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakAssign(WeakReference *ref, void *value); + +#else + +static inline void swift_unknownWeakAssign(WeakReference *ref, void *value) { + swift_weakAssign(ref, static_cast(value)); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Load a value from a weak reference, much like swift_weakLoadStrong +/// but without requiring the variable to refer to a native Swift object. +/// +/// \param ref - never null +/// \return can be null +SWIFT_RUNTIME_EXPORT +extern "C" void *swift_unknownWeakLoadStrong(WeakReference *ref); + +#else + +static inline void *swift_unknownWeakLoadStrong(WeakReference *ref) { + return static_cast(swift_weakLoadStrong(ref)); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Load a value from a weak reference as if by +/// swift_unknownWeakLoadStrong, but leaving the reference in an +/// uninitialized state. +/// +/// \param ref - never null +/// \return can be null +SWIFT_RUNTIME_EXPORT +extern "C" void *swift_unknownWeakTakeStrong(WeakReference *ref); + +#else + +static inline void *swift_unknownWeakTakeStrong(WeakReference *ref) { + return static_cast(swift_weakTakeStrong(ref)); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Destroy a weak reference variable that might not refer to a native +/// Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakDestroy(WeakReference *object); + +#else + +static inline void swift_unknownWeakDestroy(WeakReference *object) { + swift_weakDestroy(object); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Copy-initialize a weak reference variable from one that might not +/// refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakCopyInit(WeakReference *dest, + WeakReference *src); + +#else + +static inline void swift_unknownWeakCopyInit(WeakReference *dest, + WeakReference *src) { + swift_weakCopyInit(dest, src); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Take-initialize a weak reference variable from one that might not +/// refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakTakeInit(WeakReference *dest, + WeakReference *src); + +#else + +static inline void swift_unknownWeakTakeInit(WeakReference *dest, + WeakReference *src) { + swift_weakTakeInit(dest, src); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Copy-assign a weak reference variable from another when either +/// or both variables might not refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakCopyAssign(WeakReference *dest, + WeakReference *src); + +#else + +static inline void swift_unknownWeakCopyAssign(WeakReference *dest, + WeakReference *src) { + swift_weakCopyAssign(dest, src); +} + +#endif /* SWIFT_OBJC_INTEROP */ + +#if SWIFT_OBJC_INTEROP + +/// Take-assign a weak reference variable from another when either +/// or both variables might not refer to a native Swift object. +SWIFT_RUNTIME_EXPORT +extern "C" void swift_unknownWeakTakeAssign(WeakReference *dest, + WeakReference *src); + +#else + +static inline void swift_unknownWeakTakeAssign(WeakReference *dest, + WeakReference *src) { + swift_weakTakeAssign(dest, src); +} + +#endif /* SWIFT_OBJC_INTEROP */ + /*****************************************************************************/ /************************ UNKNOWN UNOWNED REFERENCES *************************/ /*****************************************************************************/ From 3c94ba1ef449603f78ce9a593ac0b3b95c770ccf Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 16:34:23 -0800 Subject: [PATCH 25/38] fake_memory_order_consume -> SWIFT_MEMORY_ORDER_CONSUME --- include/swift/Runtime/Metadata.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/include/swift/Runtime/Metadata.h b/include/swift/Runtime/Metadata.h index 7438b518f2e0e..8d852694c798e 100644 --- a/include/swift/Runtime/Metadata.h +++ b/include/swift/Runtime/Metadata.h @@ -1827,17 +1827,6 @@ struct TargetObjCClassWrapperMetadata : public TargetMetadata { using ObjCClassWrapperMetadata = TargetObjCClassWrapperMetadata; -// FIXME: Workaround for rdar://problem/18889711. 'Consume' does not require -// a barrier on ARM64, but LLVM doesn't know that. Although 'relaxed' -// is formally UB by C++11 language rules, we should be OK because neither -// the processor model nor the optimizer can realistically reorder our uses -// of 'consume'. -#if __arm64__ || __arm__ -# define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_relaxed) -#else -# define SWIFT_MEMORY_ORDER_CONSUME (std::memory_order_consume) -#endif - /// The structure of metadata for foreign types where the source /// language doesn't provide any sort of more interesting metadata for /// us to use. From ab2e3107d32a4cc62bd8d67c98ebeb5181392f39 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 16:56:12 -0800 Subject: [PATCH 26/38] Remove FIXMEs for canBeFreedNow() optimization, which looks fine. --- stdlib/public/SwiftShims/RefCount.h | 1 - stdlib/public/runtime/HeapObject.cpp | 2 -- 2 files changed, 3 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 5a2d0c4f44d7d..9e367afab0bf1 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -963,7 +963,6 @@ class RefCounts { bits.getIsDeiniting() && bits.getStrongExtraRefCount() == 0 && bits.getUnownedRefCount() == 1); - // FIXME: make sure no-assert build optimizes this } private: diff --git a/stdlib/public/runtime/HeapObject.cpp b/stdlib/public/runtime/HeapObject.cpp index fab26041ae30e..8a8ce5d63e855 100644 --- a/stdlib/public/runtime/HeapObject.cpp +++ b/stdlib/public/runtime/HeapObject.cpp @@ -680,8 +680,6 @@ void swift::swift_deallocObject(HeapObject *object, // Some allocations passed to swift_deallocObject() are not compatible // with swift_unownedRelease() because they do not have ClassMetadata. - // FIXME: reexamine and repair this optimization - if (object->refCounts.canBeFreedNow()) { // object state DEINITING -> DEAD SWIFT_RT_ENTRY_CALL(swift_slowDealloc) From 1f26bb871a56c3ed91def18510b7ed7b38d2f1c9 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 18:25:27 -0800 Subject: [PATCH 27/38] Implement out-of-line path of tryIncrementAndPin(). --- stdlib/public/SwiftShims/RefCount.h | 35 ++++++++++++++++++++--------- stdlib/public/runtime/RefCount.cpp | 34 +++++++++++++++------------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 9e367afab0bf1..7dabd3484519c 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -736,10 +736,10 @@ class RefCounts { void incrementNonAtomicSlow(RefCountBits oldbits, uint32_t inc); LLVM_ATTRIBUTE_NOINLINE - bool tryIncrementAndPinSlow(); + bool tryIncrementAndPinSlow(RefCountBits oldbits); LLVM_ATTRIBUTE_NOINLINE - bool tryIncrementAndPinNonAtomicSlow(); + bool tryIncrementAndPinNonAtomicSlow(RefCountBits); LLVM_ATTRIBUTE_NOINLINE bool tryIncrementSlow(RefCountBits oldbits); @@ -818,25 +818,26 @@ class RefCounts { newbits.setIsPinned(true); bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) - return tryIncrementAndPinSlow(); + return tryIncrementAndPinSlow(oldbits); } while (!refCounts.compare_exchange_weak(oldbits, newbits, std::memory_order_relaxed)); return true; } bool tryIncrementAndPinNonAtomic() { - auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); + auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME); // If the flag is already set, just fail. - if (!bits.hasSideTable() && bits.getIsPinned()) + if (!oldbits.hasSideTable() && oldbits.getIsPinned()) return false; // Try to simultaneously set the flag and increment the reference count. - bits.setIsPinned(true); - bool fast = bits.incrementStrongExtraRefCount(1); + auto newbits = oldbits; + newbits.setIsPinned(true); + bool fast = newbits.incrementStrongExtraRefCount(1); if (!fast) - return tryIncrementAndPinNonAtomicSlow(); - refCounts.store(bits, std::memory_order_relaxed); + return tryIncrementAndPinNonAtomicSlow(oldbits); + refCounts.store(newbits, std::memory_order_relaxed); return true; } @@ -938,7 +939,13 @@ class RefCounts { // return false; assert(!bits.getIsDeiniting()); - return bits.isUniquelyReferencedOrPinned(); + + // bits.isUniquelyReferencedOrPinned() also checks the side table bit + // and this path is optimized better if we don't check it here first. + if (bits.isUniquelyReferencedOrPinned()) return true; + if (!bits.hasSideTable()) + return false; + return bits.getSideTable()->isUniquelyReferencedOrPinned(); } // Return true if the object has started deiniting. @@ -1231,10 +1238,18 @@ class HeapObjectSideTableEntry { return refCounts.tryIncrement(); } + bool tryIncrementAndPin() { + return refCounts.tryIncrementAndPin(); + } + uint32_t getCount() const { return refCounts.getCount(); } + bool isUniquelyReferencedOrPinned() const { + return refCounts.isUniquelyReferencedOrPinned(); + } + // UNOWNED void incrementUnowned(uint32_t inc) { diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp index 1e7e88701f692..920df042660a1 100644 --- a/stdlib/public/runtime/RefCount.cpp +++ b/stdlib/public/runtime/RefCount.cpp @@ -44,22 +44,6 @@ void RefCounts::incrementNonAtomicSlow(RefCountBits oldbits, template void RefCounts::incrementNonAtomicSlow(InlineRefCountBits oldbits, uint32_t n); template void RefCounts::incrementNonAtomicSlow(SideTableRefCountBits oldbits, uint32_t n); -template -bool RefCounts::tryIncrementAndPinSlow() { - abort(); -} -template bool RefCounts::tryIncrementAndPinSlow(); -template bool RefCounts::tryIncrementAndPinSlow(); - -template -bool RefCounts::tryIncrementAndPinNonAtomicSlow() { - abort(); -} -template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); -template bool RefCounts::tryIncrementAndPinNonAtomicSlow(); - - -// SideTableRefCountBits specialization intentionally does not exist. template bool RefCounts::tryIncrementSlow(RefCountBits oldbits) { if (oldbits.hasSideTable()) @@ -70,6 +54,24 @@ bool RefCounts::tryIncrementSlow(RefCountBits oldbits) { template bool RefCounts::tryIncrementSlow(InlineRefCountBits oldbits); template bool RefCounts::tryIncrementSlow(SideTableRefCountBits oldbits); +template +bool RefCounts::tryIncrementAndPinSlow(RefCountBits oldbits) { + if (oldbits.hasSideTable()) + return oldbits.getSideTable()->tryIncrementAndPin(); + else + swift::swift_abortRetainOverflow(); +} +template bool RefCounts::tryIncrementAndPinSlow(InlineRefCountBits oldbits); +template bool RefCounts::tryIncrementAndPinSlow(SideTableRefCountBits oldbits); + +template +bool RefCounts::tryIncrementAndPinNonAtomicSlow(RefCountBits oldbits) { + // No nonatomic implementation provided. + return tryIncrementAndPinSlow(oldbits); +} +template bool RefCounts::tryIncrementAndPinNonAtomicSlow(InlineRefCountBits oldbits); +template bool RefCounts::tryIncrementAndPinNonAtomicSlow(SideTableRefCountBits oldbits); + // Return an object's side table, allocating it if necessary. // Returns null if the object is deiniting. From 5ed05f55c5ce98a43bda5da04caaa3095142f378 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 18:35:05 -0800 Subject: [PATCH 28/38] Fix Linux build. --- stdlib/public/SwiftShims/RefCount.h | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 7dabd3484519c..746035911a8f8 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -29,6 +29,7 @@ typedef struct { #include #include +#include #include #include From dedcbecac9b7067807b8af6d2bf17015cf1db688 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Wed, 14 Dec 2016 19:11:21 -0800 Subject: [PATCH 29/38] Fix Linux build, maybe. --- cmake/modules/AddSwift.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index e8ddbca0ad8f2..425c1324cc116 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -325,7 +325,7 @@ function(_add_variant_link_flags) RESULT_VAR_NAME result) if("${LFLAGS_SDK}" STREQUAL "LINUX") - list(APPEND result "-lpthread" "-ldl") + list(APPEND result "-lpthread" "-ldl" "-latomic") elseif("${LFLAGS_SDK}" STREQUAL "FREEBSD") list(APPEND result "-lpthread") elseif("${LFLAGS_SDK}" STREQUAL "CYGWIN") From 43e4a24e5ec4b38b3edb42cd71ffc7e93dad0425 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 15 Dec 2016 13:54:20 -0800 Subject: [PATCH 30/38] Fix Linux test. --- unittests/runtime/LongTests/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unittests/runtime/LongTests/CMakeLists.txt b/unittests/runtime/LongTests/CMakeLists.txt index 54befa11b67f6..37a7a0e85328f 100644 --- a/unittests/runtime/LongTests/CMakeLists.txt +++ b/unittests/runtime/LongTests/CMakeLists.txt @@ -16,6 +16,11 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND # ${FOUNDATION_LIBRARY} # swiftStdlibUnittest${SWIFT_PRIMARY_VARIANT_SUFFIX} # ) + elseif(SWIFT_HOST_VARIANT STREQUAL "linux") + find_library(ATOMIC_LIBRARY atomic) + list(APPEND PLATFORM_TARGET_LINK_LIBRARIES + ${ATOMIC_LIBRARY} + ) elseif(SWIFT_HOST_VARIANT STREQUAL "freebsd") find_library(EXECINFO_LIBRARY execinfo) list(APPEND PLATFORM_TARGET_LINK_LIBRARIES From 7adaffa507bc0b918d1d627d17b8007beb027c96 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 15 Dec 2016 14:30:44 -0800 Subject: [PATCH 31/38] Try to fix Linux tests. --- unittests/runtime/LongTests/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unittests/runtime/LongTests/CMakeLists.txt b/unittests/runtime/LongTests/CMakeLists.txt index 37a7a0e85328f..5c28a79f07813 100644 --- a/unittests/runtime/LongTests/CMakeLists.txt +++ b/unittests/runtime/LongTests/CMakeLists.txt @@ -17,9 +17,8 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND # swiftStdlibUnittest${SWIFT_PRIMARY_VARIANT_SUFFIX} # ) elseif(SWIFT_HOST_VARIANT STREQUAL "linux") - find_library(ATOMIC_LIBRARY atomic) list(APPEND PLATFORM_TARGET_LINK_LIBRARIES - ${ATOMIC_LIBRARY} + "atomic" ) elseif(SWIFT_HOST_VARIANT STREQUAL "freebsd") find_library(EXECINFO_LIBRARY execinfo) From 7c2c0cdb77d1f35e34c00628aa2f288cfbe7e5d5 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 15 Dec 2016 14:51:59 -0800 Subject: [PATCH 32/38] Fix Linux tests. --- unittests/runtime/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittests/runtime/CMakeLists.txt b/unittests/runtime/CMakeLists.txt index 797bbf0a6809f..347746e6faa46 100644 --- a/unittests/runtime/CMakeLists.txt +++ b/unittests/runtime/CMakeLists.txt @@ -22,6 +22,8 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND ${FOUNDATION_LIBRARY} swiftStdlibUnittest${SWIFT_PRIMARY_VARIANT_SUFFIX} ) + elseif(SWIFT_HOST_VARIANT STREQUAL "linux") + list(APPEND PLATFORM_TARGET_LINK_LIBRARIES "atomic") elseif(SWIFT_HOST_VARIANT STREQUAL "freebsd") find_library(EXECINFO_LIBRARY execinfo) list(APPEND PLATFORM_TARGET_LINK_LIBRARIES From c85353043a9ea4421e7124f87372f8b9b1589ca2 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 15 Dec 2016 15:26:40 -0800 Subject: [PATCH 33/38] Fix `swiftc -static-stdlib` on Linux. --- utils/gen-static-stdlib-link-args | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/gen-static-stdlib-link-args b/utils/gen-static-stdlib-link-args index af717ea470019..fae2bb088874d 100755 --- a/utils/gen-static-stdlib-link-args +++ b/utils/gen-static-stdlib-link-args @@ -33,6 +33,7 @@ fi cat >$OUTPUTFILE < Date: Fri, 16 Dec 2016 14:53:54 -0800 Subject: [PATCH 34/38] Add -latomic to -static-executable. --- utils/static-executable-args.lnk | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/static-executable-args.lnk b/utils/static-executable-args.lnk index 6987433fc6185..ff408601185c4 100644 --- a/utils/static-executable-args.lnk +++ b/utils/static-executable-args.lnk @@ -8,6 +8,7 @@ -Xlinker --defsym=__import_pthread_key_create=pthread_key_create -lpthread +-latomic -licui18n -licuuc -licudata From 1b5b8c461408455d6d08ae97842a17c592f8f653 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Thu, 9 Feb 2017 01:40:45 -0800 Subject: [PATCH 35/38] LLVM_ATTRIBUTE_UNUSED_RESULT has become LLVM_NODISCARD. --- stdlib/public/SwiftShims/RefCount.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 7c2c6fd3f1bba..7b59508777409 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -383,7 +383,7 @@ class RefCountBitsT { // (for example, because UseSlowRC is set // or because the refcount is now zero and should deinit). template - LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool doDecrementStrongExtraRefCount(uint32_t dec) { #ifndef NDEBUG if (!hasSideTable()) { @@ -544,7 +544,7 @@ class RefCountBitsT { // Returns true if the increment is a fast-path result. // Returns false if the increment should fall back to some slow path // (for example, because UseSlowRC is set or because the refcount overflowed). - LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool incrementStrongExtraRefCount(uint32_t inc) { // This deliberately overflows into the UseSlowRC field. bits += BitsType(inc) << Offsets::StrongExtraRefCountShift; @@ -553,7 +553,7 @@ class RefCountBitsT { // FIXME: I don't understand why I can't make clearPinned a template argument // (compiler balks at calls from class RefCounts that way) - LLVM_ATTRIBUTE_ALWAYS_INLINE LLVM_ATTRIBUTE_UNUSED_RESULT + LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool decrementStrongExtraRefCount(uint32_t dec, bool clearPinned = false) { if (clearPinned) return doDecrementStrongExtraRefCount(dec); @@ -1275,7 +1275,7 @@ class HeapObjectSideTableEntry { // WEAK - LLVM_ATTRIBUTE_UNUSED_RESULT + LLVM_NODISCARD HeapObjectSideTableEntry* incrementWeak() { // incrementWeak need not be atomic w.r.t. concurrent deinit initiation. // The client can't actually get a reference to the object without From 24d1fc2f52316732a9d1de19e18aa7f107d21c15 Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Fri, 17 Feb 2017 15:07:51 -0800 Subject: [PATCH 36/38] Add descriptors of weak reference bits for lldb. --- include/swift/ABI/System.h | 36 ++++++++++++++++++++ stdlib/public/runtime/WeakReference.h | 49 ++++++++++++++++++++------- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/include/swift/ABI/System.h b/include/swift/ABI/System.h index 3bad3622c9877..bd7e84d95c334 100644 --- a/include/swift/ABI/System.h +++ b/include/swift/ABI/System.h @@ -56,16 +56,36 @@ /// ``pointer & SWIFT_ABI_XXX_OBJC_RESERVED_BITS_MASK == 0 && /// pointer & SWIFT_ABI_XXX_SWIFT_SPARE_BITS_MASK != 0``. +// Weak references use a marker to tell when they are controlled by +// the ObjC runtime and when they are controlled by the Swift runtime. +// Non-ObjC platforms don't use this marker. +#define SWIFT_ABI_DEFAULT_OBJC_WEAK_REFERENCE_MARKER_MASK 0 +#define SWIFT_ABI_DEFAULT_OBJC_WEAK_REFERENCE_MARKER_VALUE 0 + /*********************************** i386 *************************************/ // Heap objects are pointer-aligned, so the low two bits are unused. #define SWIFT_ABI_I386_SWIFT_SPARE_BITS_MASK 0x00000003U +// ObjC weak reference discriminator is the LSB. +#define SWIFT_ABI_I386_OBJC_WEAK_REFERENCE_MARKER_MASK \ + (SWIFT_ABI_DEFAULT_OBJC_RESERVED_BITS_MASK | \ + 1< Date: Fri, 17 Feb 2017 17:25:15 -0800 Subject: [PATCH 37/38] Don't static_assert ObjC bits on non-ObjC platforms. --- stdlib/public/runtime/WeakReference.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stdlib/public/runtime/WeakReference.h b/stdlib/public/runtime/WeakReference.h index b78611f63cec7..aa8f4f41d2000 100644 --- a/stdlib/public/runtime/WeakReference.h +++ b/stdlib/public/runtime/WeakReference.h @@ -100,13 +100,15 @@ class WeakReferenceBits { static_assert((NativeMarkerMask & heap_object_abi::SwiftSpareBitsMask) == NativeMarkerMask, "native marker mask must fall within Swift spare bits"); +#if SWIFT_OBJC_INTEROP static_assert((NativeMarkerMask & heap_object_abi::ObjCReservedBitsMask) == heap_object_abi::ObjCReservedBitsMask, "native marker mask must contain all ObjC tagged pointer bits"); static_assert((NativeMarkerValue & heap_object_abi::ObjCReservedBitsMask) == 0, "native marker value must not interfere with ObjC bits"); - +#endif + uintptr_t bits; public: From 3910e4336f4019f38ca321c60a1ebd1980c7d7bf Mon Sep 17 00:00:00 2001 From: Greg Parker Date: Fri, 24 Feb 2017 02:05:48 -0800 Subject: [PATCH 38/38] Fix LongRefcounting test. * Don't verify death by testing for symbol names. * Fix max allowed retain count. --- .../runtime/LongTests/LongRefcounting.cpp | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/unittests/runtime/LongTests/LongRefcounting.cpp b/unittests/runtime/LongTests/LongRefcounting.cpp index 22ea079d286de..82a1df63c465e 100644 --- a/unittests/runtime/LongTests/LongRefcounting.cpp +++ b/unittests/runtime/LongTests/LongRefcounting.cpp @@ -62,7 +62,8 @@ static SWIFT_CC(swift) void deinitTestObject(SWIFT_CONTEXT HeapObject *_object) // URC load crashes // URC increment OK // URC decrement OK - ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + ASSERT_DEATH(swift_unownedCheck(object), + "attempted to read an unowned reference"); swift_unownedRetain(object); swift_unownedRetain(object); swift_unownedRelease(object); @@ -136,8 +137,9 @@ static void releaseALot(TestObject *object, size_t &deallocated, } } -// 32-3 bits of extra retain count, plus 1 for the implicit retain -const uint64_t maxRC = 1ULL << (32 - 3); +// Maximum legal retain count. +// 32-2 bits of extra retain count, plus 1 for the implicit retain. +const uint64_t maxRC = 1ULL << (32 - 2); TEST(LongRefcountingTest, retain_max) { size_t deallocated = 0; @@ -159,7 +161,8 @@ TEST(LongRefcountingTest, retain_overflow_DeathTest) { // RC is 1. Retain to maxRC, then retain again and verify overflow error. retainALot(object, deallocated, maxRC - 1); EXPECT_EQ(0u, deallocated); - ASSERT_DEATH(swift_retain(object), "swift_abortRetainOverflow"); + ASSERT_DEATH(swift_retain(object), + "object was retained too many times"); } TEST(LongRefcountingTest, nonatomic_retain_max) { @@ -182,7 +185,8 @@ TEST(LongRefcountingTest, nonatomic_retain_overflow_DeathTest) { // RC is 1. Retain to maxRC, then retain again and verify overflow error. retainALot(object, deallocated, maxRC - 1); EXPECT_EQ(0u, deallocated); - ASSERT_DEATH(swift_nonatomic_retain(object), "swift_abortRetainOverflow"); + ASSERT_DEATH(swift_nonatomic_retain(object), + "object was retained too many times"); } @@ -305,7 +309,8 @@ TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_no_side_DeathTest) { // URC load crashes // URC increment can't happen // URC decrement OK - ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + ASSERT_DEATH(swift_unownedCheck(object), + "attempted to read an unowned reference"); swift_unownedRelease(object); EXPECT_ALLOCATED(object); @@ -513,7 +518,8 @@ TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_with_side_DeathTest) // URC load crashes // URC increment can't happen // URC decrement OK - ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + ASSERT_DEATH(swift_unownedCheck(object), + "attempted to read an unowned reference"); swift_unownedRelease(object); EXPECT_ALLOCATED(object); EXPECT_ALLOCATED(side); @@ -752,7 +758,8 @@ TEST(LongRefcountingTest, lifecycle_live_deiniting_deinited_freed_with_side_Deat // URC load crashes // URC increment can't happen // URC decrement OK - ASSERT_DEATH(swift_unownedCheck(object), "swift_abortRetainUnowned"); + ASSERT_DEATH(swift_unownedCheck(object), + "attempted to read an unowned reference"); swift_unownedRelease(object); EXPECT_ALLOCATED(object); EXPECT_ALLOCATED(side);