[DNM] Parse /.../ regex literals

include/swift/AST/DiagnosticEngine.h

@@ -848,6 +848,7 @@ namespace swift {
     friend class DiagnosticTransaction;
     friend class CompoundDiagnosticTransaction;
     friend class DiagnosticStateRAII;
+    friend class DiagnosticQueue;
 
   public:
     explicit DiagnosticEngine(SourceManager &SourceMgr)
@@ -1137,10 +1138,16 @@ namespace swift {
     /// Send \c diag to all diagnostic consumers.
     void emitDiagnostic(const Diagnostic &diag);
 
+    /// Handle a new diagnostic, which will either be emitted, or added to an
+    /// active transaction.
+    void handleDiagnostic(Diagnostic diag);
+
     /// Send all tentative diagnostics to all diagnostic consumers and
     /// delete them.
     void emitTentativeDiagnostics();
 
+    void forwardTentativeDiagnosticsTo(DiagnosticEngine &targetEngine);
+
   public:
     DiagnosticKind declaredDiagnosticKindFor(const DiagID id);
 
@@ -1333,6 +1340,50 @@ namespace swift {
     }
   };
 
+  class DiagnosticQueue final {
+    /// The underlying diagnostic engine that the diagnostics will be emitted
+    /// by.
+    DiagnosticEngine &UnderlyingEngine;
+
+    /// A temporary engine used to queue diagnostics.
+    DiagnosticEngine QueueEngine;
+
+  public:
+    DiagnosticQueue(const DiagnosticQueue &) = delete;
+    DiagnosticQueue &operator=(const DiagnosticQueue &) = delete;
+
+    explicit DiagnosticQueue(DiagnosticEngine &engine)
+        : UnderlyingEngine(engine), QueueEngine(engine.SourceMgr) {
+      // Open a transaction to avoid emitting any diagnostics.
+      QueueEngine.TransactionCount++;
+    }
+
+    /// Retrieve the engine which may be used to enqueue diagnostics.
+    DiagnosticEngine &getDiags() { return QueueEngine; }
+
+    /// Retrieve the underlying engine which will receive the diagnostics.
+    DiagnosticEngine &getUnderlyingDiags() { return UnderlyingEngine; }
+
+    /// Clear this queue and erase all diagnostics recorded.
+    void clear() {
+      assert(QueueEngine.TransactionCount == 1 &&
+             "Haven't closed outstanding DiagnosticTransactions");
+      QueueEngine.TentativeDiagnostics.clear();
+    }
+
+    /// Emit all the diagnostics recorded by this queue.
+    void emit() {
+      assert(QueueEngine.TransactionCount == 1 &&
+             "Haven't closed outstanding DiagnosticTransactions");
+      QueueEngine.forwardTentativeDiagnosticsTo(UnderlyingEngine);
+    }
+
+    ~DiagnosticQueue() {
+      emit();
+      QueueEngine.TransactionCount--;
+    }
+  };
+
   inline void
   DiagnosticEngine::diagnoseWithNotes(InFlightDiagnostic parentDiag,
                                       llvm::function_ref<void(void)> builder) {

include/swift/AST/DiagnosticsParse.def

@@ -94,6 +94,9 @@ ERROR(forbidden_extended_escaping_string,none,
 ERROR(regex_literal_parsing_error,none,
       "%0", (StringRef))
 
+ERROR(prefix_slash_not_allowed,none,
+      "prefix slash not allowed", ())
+
 //------------------------------------------------------------------------------
 // MARK: Lexer diagnostics
 //------------------------------------------------------------------------------

include/swift/Parse/Lexer.h

@@ -75,7 +75,10 @@ class Lexer {
   const LangOptions &LangOpts;
   const SourceManager &SourceMgr;
   const unsigned BufferID;
-  DiagnosticEngine *Diags;
+
+  /// A queue of diagnostics to emit when a token is consumed. We want to queue
+  /// them, as the parser may backtrack and re-lex a token.
+  Optional<DiagnosticQueue> DiagQueue;
 
   using State = LexerState;
 
@@ -154,6 +157,19 @@ class Lexer {
 
   void initialize(unsigned Offset, unsigned EndOffset);
 
+  /// Retrieve the diagnostic engine for emitting diagnostics for the current
+  /// token.
+  DiagnosticEngine *getTokenDiags() {
+    return DiagQueue ? &DiagQueue->getDiags() : nullptr;
+  }
+
+  /// Retrieve the underlying diagnostic engine we emit diagnostics to. Note
+  /// this should only be used for diagnostics not concerned with the current
+  /// token.
+  DiagnosticEngine *getUnderlyingDiags() {
+    return DiagQueue ? &DiagQueue->getUnderlyingDiags() : nullptr;
+  }
+
 public:
   /// Create a normal lexer that scans the whole source buffer.
   ///
@@ -209,6 +225,10 @@ class Lexer {
       LeadingTriviaResult = LeadingTrivia;
       TrailingTriviaResult = TrailingTrivia;
     }
+    // Emit any diagnostics recorded for this token.
+    if (DiagQueue)
+      DiagQueue->emit();
+
     if (Result.isNot(tok::eof))
       lexImpl();
   }
@@ -298,12 +318,12 @@ class Lexer {
   void restoreState(State S, bool enableDiagnostics = false) {
     assert(S.isValid());
     CurPtr = getBufferPtrForSourceLoc(S.Loc);
-    // Don't reemit diagnostics while readvancing the lexer.
-    llvm::SaveAndRestore<DiagnosticEngine*>
-      D(Diags, enableDiagnostics ? Diags : nullptr);
-
     lexImpl();
 
+    // Don't re-emit diagnostics from readvancing the lexer.
+    if (DiagQueue && !enableDiagnostics)
+      DiagQueue->clear();
+
     // Restore Trivia.
     if (TriviaRetention == TriviaRetentionMode::WithTrivia)
       LeadingTrivia = S.LeadingTrivia;
@@ -505,7 +525,7 @@ class Lexer {
 
   void getStringLiteralSegments(const Token &Str,
                                 SmallVectorImpl<StringSegment> &Segments) {
-    return getStringLiteralSegments(Str, Segments, Diags);
+    return getStringLiteralSegments(Str, Segments, getTokenDiags());
   }
 
   static SourceLoc getSourceLoc(const char *Loc) {
@@ -531,6 +551,9 @@ class Lexer {
     void operator=(const SILBodyRAII&) = delete;
   };
 
+  /// Attempt to re-lex a regex literal with forward slashes `/.../`.
+  bool tryLexAsForwardSlashRegexLiteral(State S);
+
 private:
   /// Nul character meaning kind.
   enum class NulCharacterKind {

include/swift/Parse/Parser.h

@@ -559,6 +559,11 @@ class Parser {
     return f(backtrackScope);
   }
 
+  /// Discard the current token. This will avoid interface hashing or updating
+  /// the previous loc. Only should be used if you've completely re-lexed
+  /// a different token at that position.
+  SourceLoc discardToken();
+
   /// Consume a token that we created on the fly to correct the original token
   /// stream from lexer.
   void consumeExtraToken(Token K);
@@ -1745,8 +1750,11 @@ class Parser {
   ParserResult<Expr>
   parseExprPoundCodeCompletion(Optional<StmtKind> ParentKind);
 
+  UnresolvedDeclRefExpr *makeExprOperator(Token opToken);
   UnresolvedDeclRefExpr *parseExprOperator();
 
+  void tryLexRegexLiteral();
+
   void validateCollectionElement(ParserResult<Expr> element);
 
   //===--------------------------------------------------------------------===//

lib/AST/DiagnosticEngine.cpp

@@ -1040,15 +1040,19 @@ DiagnosticBehavior DiagnosticState::determineBehavior(const Diagnostic &diag) {
 
 void DiagnosticEngine::flushActiveDiagnostic() {
   assert(ActiveDiagnostic && "No active diagnostic to flush");
+  handleDiagnostic(std::move(*ActiveDiagnostic));
+  ActiveDiagnostic.reset();
+}
+
+void DiagnosticEngine::handleDiagnostic(Diagnostic diag) {
   if (TransactionCount == 0) {
-    emitDiagnostic(*ActiveDiagnostic);
+    emitDiagnostic(diag);
     WrappedDiagnostics.clear();
     WrappedDiagnosticArgs.clear();
   } else {
-    onTentativeDiagnosticFlush(*ActiveDiagnostic);
-    TentativeDiagnostics.emplace_back(std::move(*ActiveDiagnostic));
+    onTentativeDiagnosticFlush(diag);
+    TentativeDiagnostics.emplace_back(std::move(diag));
   }
-  ActiveDiagnostic.reset();
 }
 
 void DiagnosticEngine::emitTentativeDiagnostics() {
@@ -1060,6 +1064,16 @@ void DiagnosticEngine::emitTentativeDiagnostics() {
   WrappedDiagnosticArgs.clear();
 }
 
+void DiagnosticEngine::forwardTentativeDiagnosticsTo(
+    DiagnosticEngine &targetEngine) {
+  for (auto &diag : TentativeDiagnostics) {
+    targetEngine.handleDiagnostic(diag);
+  }
+  TentativeDiagnostics.clear();
+  WrappedDiagnostics.clear();
+  WrappedDiagnosticArgs.clear();
+}
+
 /// Returns the access level of the least accessible PrettyPrintedDeclarations
 /// buffer that \p decl should appear in.
 ///