[stdlib] Create a UTF8 StringStorage

Add a CharacterView to _FixedFormatUnicode

Implement RangeReplaceable inits to override defaults that would always assert
for non-empty inputs (you can't grow a _BoundedBuffer beyond its capacity and
the default capacity is zero).

Author: Dave Abrahams

stdlib/public/core/AnyUnicode.swift

@@ -50,7 +50,26 @@ public protocol _FixedFormatUnicode : _AnyUnicode {
   // where Iterator.Element == UInt16
 
   /// An FCC-normalized view of the string
-  var fccNormalizedUTF16 : FCCNormalizedUTF16View { get }
+  var fccNormalizedUTF16: FCCNormalizedUTF16View { get }
+
+  associatedtype CharacterView : BidirectionalCollection
+  // where Iterator.Element == Character
+
+  var characters: CharacterView { get }
+}
+
+/// Default views
+public extension _FixedFormatUnicode
+where
+  CodeUnits.Iterator.Element  == Encoding.EncodedScalar.Iterator.Element,
+  CodeUnits.Iterator.Element : UnsignedInteger,
+  CodeUnits.SubSequence : RandomAccessCollection,
+  CodeUnits.SubSequence.Index == CodeUnits.Index,
+  CodeUnits.SubSequence.SubSequence == CodeUnits.SubSequence,
+  CodeUnits.SubSequence.Iterator.Element == CodeUnits.Iterator.Element {
+  var characters: UnicodeStorage<CodeUnits,Encoding>.CharacterView {
+    return UnicodeStorage(codeUnits).characters
+  }
 }
 
 public extension _FixedFormatUnicode {
@@ -108,12 +127,12 @@ public extension _FixedFormatUnicode where Encoding == Latin1 {
 
 public extension _FixedFormatUnicode
 where Encoding == Latin1, CodeUnits.Iterator.Element : UnsignedInteger {
-  var rawUTF16 : LazyMapRandomAccessCollection<CodeUnits, UInt16> {
+  var rawUTF16: LazyMapRandomAccessCollection<CodeUnits, UInt16> {
     return fccNormalizedUTF16
   }
 
   /// An FCC-normalized view of the string
-  var fccNormalizedUTF16 : LazyMapRandomAccessCollection<CodeUnits, UInt16> {
+  var fccNormalizedUTF16: LazyMapRandomAccessCollection<CodeUnits, UInt16> {
     return codeUnits.lazy.map { numericCast($0) }
   }
 }

stdlib/public/core/BoundedBufferReference.swift

@@ -117,6 +117,19 @@ extension _BoundedBufferReference {
 
 /// Fulfills the RangeReplaceableCollection requirements
 extension _BoundedBufferReference {
+  public init<S : Sequence>(_ elements: S)
+    where S.Iterator.Element == Iterator.Element {
+    self.init(Array(elements))
+  }
+  
+  public init<C : Collection>(_ elements: C)
+    where C.Iterator.Element == Iterator.Element {
+    self.init(_uninitializedCount: numericCast(elements.count))
+    withUnsafeMutableBufferPointer {
+      elements._copyCompleteContents(initializing: $0)
+    }
+  }
+  
   public func replaceSubrange<C>(
     _ target: Range<Int>,
     with newValues: C

stdlib/public/core/StringStorage.swift

@@ -379,3 +379,101 @@ extension _Latin1StringStorage : _FixedFormatUnicode {
   }
 }
 
+//===--- UTF-8 String Storage ---------------------------------------------===//
+public struct UTF8StringHeader : _BoundedBufferHeader {
+  public var count: UInt32
+  public var capacity: UInt32
+  
+  public init(count: Int, capacity: Int) {
+    self.count = numericCast(count)
+    self.capacity = numericCast(capacity)
+  }
+}
+
+public final class _UTF8StringStorage
+// FIXME: we might want our own header type
+  : _StringStorageBase<UTF8StringHeader, UInt8>
+  , _NSStringCore // Ensures that we implement essential NSString methods.  
+{
+  // WORKAROUND: helping type inference along will be unnecessary someday
+  public typealias _Element = UInt8
+  public typealias Iterator = IndexingIterator<_UTF8StringStorage>
+  
+  /// Returns a pointer to contiguously-stored UTF-16 code units
+  /// comprising the whole string, or NULL if such storage isn't
+  /// available.
+  ///
+  /// WARNING: don't use this method from Swift code; ARC may end the
+  /// lifetime of self before you get a chance to use the result.
+  @objc
+  public func _fastCharacterContents() -> UnsafeMutablePointer<UInt16>? {
+    return nil
+  }
+
+  /// Returns a pointer to contiguously-stored code units in the
+  /// system encoding comprising the whole string, or NULL if such
+  /// storage isn't available.
+  ///
+  // WARNING: don't use this method from Swift code; ARC may end the lifetime of
+  // self before you get a chance to use the result.
+  // WARNING: Before you implement this as anything other than “return nil,”
+  // see https://github.com/apple/swift/pull/3151#issuecomment-285583557
+  @objc
+  public func _fastCStringContents(
+    _ nullTerminationRequired: Int8
+  ) -> UnsafePointer<CChar>? {
+    return nil
+  }
+  
+  // WORKAROUND: rdar://31047127 prevents us from hoisting this into
+  // _StringStorageBase
+  @nonobjc
+  public override var _baseAddress: UnsafeMutablePointer<UTF8.CodeUnit> {
+    return UnsafeMutablePointer(
+      Builtin.projectTailElems(self, Element.self))
+  }
+
+  @nonobjc
+  public var isKnownASCII = false
+  @nonobjc
+  public var isKnownLatin1 = false
+  @nonobjc
+  public var isKnownValidEncoding = false
+  @nonobjc
+  public var isKnownFCCNormalized = false
+  @nonobjc
+  public var isKnownFCDForm = false
+  @nonobjc
+  public var isKnownNFDNormalized = false
+  @nonobjc
+  public var isKnownNFCNormalized = false
+}
+
+extension _UTF8StringStorage : _BoundedBufferReference {
+  @nonobjc
+  public static func _emptyInstance() -> _UTF8StringStorage {
+    return _UTF8StringStorage(uninitializedWithMinimumCapacity: 0)
+  }
+}
+
+extension _UTF8StringStorage : _FixedFormatUnicode {
+  public typealias Encoding = UTF8
+  
+  // WORKAROUND: helping type inference along will be unnecessary someday
+  public typealias CodeUnits = _UTF8StringStorage
+  
+  public typealias RawUTF16View = UnicodeStorage<
+    CodeUnits, Encoding
+  >.TranscodedView<UTF16>
+
+  public typealias FCCNormalizedUTF16View = UnicodeStorage<
+    CodeUnits, Encoding
+  >.FCCNormalizedUTF16View
+  
+  public var codeUnits: CodeUnits { return self }
+  public var rawUTF16: RawUTF16View { return RawUTF16View(self) }
+  public var fccNormalizedUTF16: FCCNormalizedUTF16View {
+    return UnicodeStorage(self, UTF8.self).fccNormalizedUTF16
+  }
+}
+

test/Prototypes/AnyUnicode.swift

@@ -517,6 +517,55 @@ extension AnyUnicode {
   }
 }
 
+// Work around name collision ambiguity
+extension _FixedFormatUnicode {
+  internal var _codeUnits: CodeUnits { return codeUnits }
+  internal var _characters: CharacterView { return characters }
+  internal var _fccNormalizedUTF16: FCCNormalizedUTF16View {
+    return fccNormalizedUTF16
+  }
+}
+
+extension AnyUnicode
+where Self : _FixedFormatUnicode,
+Self.CodeUnits : RandomAccessCollection,
+Self.CodeUnits.Iterator.Element : UnsignedInteger,
+Self.RawUTF16View : BidirectionalCollection,
+Self.RawUTF16View.Iterator.Element == UTF16.CodeUnit,
+Self.FCCNormalizedUTF16View.Iterator.Element : UnsignedInteger,
+Self.CodeUnits.Index == Self.CodeUnits.SubSequence.Index, 
+Self.CodeUnits.SubSequence : RandomAccessCollection, 
+Self.CodeUnits.SubSequence == Self.CodeUnits.SubSequence.SubSequence, 
+Self.CodeUnits.Iterator.Element == Self.CodeUnits.SubSequence.Iterator.Element, 
+Self.CodeUnits.SubSequence.Iterator.Element == Self.Encoding.EncodedScalar.Iterator.Element,
+Self.CharacterView.Iterator.Element == Character,
+Self.CharacterView.Index : SignedInteger
+{
+  var codeUnits: AnyCodeUnits {
+    return AnyCodeUnits(self._codeUnits)
+  }
+  
+  var rawUTF16: AnyUTF16 { return AnyUTF16(self.rawUTF16 as RawUTF16View) }
+  // FIXME: this could be more efficient for encodings such as Latin1
+  var utf32: AnyUnicodeBidirectionalUInt32 {
+      return AnyUnicodeBidirectionalUInt32(
+      UnicodeStorage(
+        _codeUnits, Encoding.self
+      ).transcoded(to: UTF32.self)
+    )
+  }
+  var fccNormalizedUTF16: AnyUTF16 {
+    return AnyUTF16(fccNormalizedUTF16 as FCCNormalizedUTF16View)
+  }
+  // FIXME: Could be more efficient generally
+  var extendedASCII: AnyUnicodeBidirectionalUInt32 {
+    return utf32
+  }
+  var characters: AnyCharacters {
+    return AnyCharacters(_characters)
+  }
+}
+
 struct AnyRandomAccessUnsignedIntegers<
   Base: RandomAccessCollection, Element_ : UnsignedInteger
 > : RandomAccessCollection
@@ -585,9 +634,9 @@ extension UTF16CompatibleStringContents : _FixedFormatUnicode {
   var fccNormalizedUTF16: FCCNormalizedUTF16View {
     switch self {
     case .utf16(let storage):
-      return AnyUTF16(storage.fccNormalizedUTF16)
+      return storage.fccNormalizedUTF16
     case .latin1(let storage):
-      return AnyUTF16(storage.fccNormalizedUTF16)
+      return storage.fccNormalizedUTF16
     }
   }
 
@@ -728,6 +777,14 @@ case latin1(_Latin1StringStorage)
 case any(AnyUnicodeBox)
 }
 
+extension _UTF16StringStorage : AnyUnicode {
+  
+}
+
+extension _Latin1StringStorage : AnyUnicode {
+  
+}
+
 extension AnyStringContents : AnyUnicode {
   var encoding: AnyUnicodeEncoding.Type {
     switch self {
@@ -788,9 +845,9 @@ extension AnyStringContents : AnyUnicode {
   var fccNormalizedUTF16: AnyUTF16 {
     switch self {
     case .utf16(let storage):
-      return AnyUTF16(storage.fccNormalizedUTF16)
+      return storage.fccNormalizedUTF16
     case .latin1(let storage):
-      return AnyUTF16(storage.fccNormalizedUTF16)
+      return storage.fccNormalizedUTF16
     case .any(let base):
       return base.fccNormalizedUTF16
     }
@@ -883,6 +940,18 @@ extension AnyStringContents : AnyUnicode {
       return base.isKnownNFCNormalized
     }
   }
+
+  init<T: AnyUnicode>(_ x: T) {
+    if let s = x as? _Latin1StringStorage {
+      self = .latin1(s)
+    }
+    else if let s = x as? _UTF16StringStorage {
+      self = .utf16(s)
+    }
+    else {
+      self = .any(AnyUnicodeBox(wrapping: x))
+    }
+  }
 }
 
 print(MemoryLayout<UTF16CompatibleStringContents>.size)
@@ -895,4 +964,16 @@ suite.test("basics") {
   expectTrue(x.elementsEqual(y))
 }
 
+suite.test("AnyStringContents") {
+  let sample = "abcdefghijklmnopqrstuvwxyz\n"
+  + "🇸🇸🇬🇱🇱🇸🇩🇯🇺🇸\n"
+  + "Σὲ 👥🥓γνωρίζω ἀπὸ τὴν κόψη χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!\n"
+  + "Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,\n"
+  + "გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო\n"
+  + "Зарегистрируйтесь сейчас на Десятую Международную Конференцию по\n"
+  + "  ๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช  พระปกเกศกองบู๊กู้ขึ้นใหม่\n"
+  + "ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ"
+
+  var s = AnyStringContents(_UTF16StringStorage(sample.utf16))
+}
 runAllTests()