Implement Swift serialization and deserialization of Clang types.

As part of this, we have to change the type export rules to
prevent `@convention(c)` function types from being used in
exported interfaces if they aren't serializable.  This is a
more conservative version of the original rule I had, which
was to import such function-pointer types as opaque pointers.
That rule would've completely prevented importing function-pointer
types defined in bridging headers and so simply doesn't work,
so we're left trying to catch the unsupportable cases
retroactively.  This has the unfortunate consequence that we
can't necessarily serialize the internal state of the compiler,
but that was already true due to normal type uses of aggregate
types from bridging headers; if we can teach the compiler to
reliably serialize such types, we should be able to use the
same mechanisms for function types.

This PR doesn't flip the switch to use Clang function types
by default, so many of the clang-function-type-serialization
FIXMEs are still in place.

Author: rjmccall

include/swift/AST/ASTContext.h

@@ -595,6 +595,10 @@ class ASTContext final {
                        const FunctionType::ExtInfo incompleteExtInfo,
                        FunctionTypeRepresentation trueRep);
 
+  /// Get the Swift declaration that a Clang declaration was exported from,
+  /// if applicable.
+  const Decl *getSwiftDeclForExportedClangDecl(const clang::Decl *decl);
+
   /// Determine whether the given Swift type is representable in a
   /// given foreign language.
   ForeignRepresentationInfo

include/swift/AST/ClangModuleLoader.h

@@ -14,10 +14,12 @@
 #define SWIFT_AST_CLANG_MODULE_LOADER_H
 
 #include "swift/AST/ModuleLoader.h"
+#include "swift/Basic/TaggedUnion.h"
 
 namespace clang {
 class ASTContext;
 class CompilerInstance;
+class Decl;
 class Preprocessor;
 class Sema;
 class TargetInfo;
@@ -26,6 +28,7 @@ class Type;
 
 namespace swift {
 
+class Decl;
 class DeclContext;
 class VisibleDeclConsumer;
 
@@ -40,6 +43,69 @@ enum class ClangTypeKind {
   ObjCProtocol,
 };
 
+/// A path for serializing a declaration.
+class StableSerializationPath {
+public:
+  struct ExternalPath {
+    enum ComponentKind {
+      /// A named record type (but not a template specialization)
+      Record,
+
+      /// A named enum type
+      Enum,
+
+      /// A C++ namespace
+      Namespace,
+
+      /// A typedef
+      Typedef,
+
+      /// A typedef's anonymous tag declaration.  Identifier is empty.
+      TypedefAnonDecl,
+
+      /// An Objective-C interface.
+      ObjCInterface,
+
+      /// An Objective-C protocol.
+      ObjCProtocol,
+    };
+
+    static bool requiresIdentifier(ComponentKind kind) {
+      return kind != TypedefAnonDecl;
+    }
+
+    SmallVector<std::pair<ComponentKind, Identifier>, 2> Path;
+
+    void add(ComponentKind kind, Identifier name) {
+      Path.push_back({kind, name});
+    }
+  };
+private:
+  TaggedUnion<void, const Decl *, ExternalPath> Union;
+
+public:
+  StableSerializationPath() {}
+  StableSerializationPath(const Decl *d) : Union(d) {}
+  StableSerializationPath(ExternalPath ext) : Union(ext) {}
+
+  explicit operator bool() const { return !Union.empty(); }
+
+  bool isSwiftDecl() const { return Union.isa<const Decl*>(); }
+  const Decl *getSwiftDecl() const {
+    assert(isSwiftDecl());
+    return Union.get<const Decl*>();
+  }
+
+  bool isExternalPath() const { return Union.isa<ExternalPath>(); }
+  const ExternalPath &getExternalPath() const {
+    assert(isExternalPath());
+    return Union.get<ExternalPath>();
+  }
+
+  SWIFT_DEBUG_DUMP;
+  void dump(raw_ostream &os) const;
+};
+
 class ClangModuleLoader : public ModuleLoader {
 private:
   virtual void anchor();
@@ -111,6 +177,41 @@ class ClangModuleLoader : public ModuleLoader {
   /// Print the Clang type.
   virtual void printClangType(const clang::Type *type,
                               llvm::raw_ostream &os) const = 0;
+
+  /// Try to find a stable serialization path for the given declaration,
+  /// if there is one.
+  virtual StableSerializationPath
+  findStableSerializationPath(const clang::Decl *decl) const = 0;
+
+  /// Try to resolve a stable serialization path down to the original
+  /// declaration.
+  virtual const clang::Decl *
+  resolveStableSerializationPath(const StableSerializationPath &path) const = 0;
+
+  /// Determine whether the given type is serializable.
+  ///
+  /// If \c checkCanonical is true, checks the canonical type,
+  /// not the given possibly-sugared type.  In general:
+  ///  - non-canonical representations should be preserving the
+  ///    sugared type even if it isn't serializable, since that
+  ///    maintains greater source fidelity;
+  ///  - semantic checks need to be checking the serializability
+  ///    of the canonical type, since it's always potentially
+  ///    necessary to serialize that (e.g. in SIL); and
+  ///  - serializers can try to serialize the sugared type to
+  ///    maintain source fidelity and just fall back on the canonical
+  ///    type if that's not possible.
+  ///
+  /// The expectation here is that this predicate is meaningful
+  /// independent of the actual form of serialization: the types
+  /// that we can't reliably binary-serialize without an absolute
+  /// Clang AST cross-reference are the same types that won't
+  /// reliably round-trip through a textual format.  At the very
+  /// least, it's probably best to use conservative predicates
+  /// that work both ways so that language behavior doesn't differ
+  /// based on subtleties like the target module interface format.
+  virtual bool isSerializable(const clang::Type *type,
+                              bool checkCanonical) const = 0;
 };
 
 } // namespace swift

include/swift/AST/DiagnosticsSema.def

@@ -2660,6 +2660,10 @@ ERROR(assoc_conformance_from_implementation_only_module,none,
       "cannot use conformance of %0 to %1 in associated type %3 (inferred as "
       "%4); %2 has been imported as implementation-only",
       (Type, DeclName, Identifier, Type, Type))
+ERROR(unexportable_clang_function_type,none,
+      "cannot export the underlying C type of the function type %0; "
+      "it may use anonymous types or types defined outside of a module",
+      (Type))
 
 WARNING(warn_implementation_only_conflict,none,
         "%0 inconsistently imported as implementation-only",

include/swift/ClangImporter/ClangImporter.h

@@ -422,6 +422,16 @@ class ClangImporter final : public ClangModuleLoader {
                                             SourceLoc loc) const override;
   void printClangType(const clang::Type *type,
                       llvm::raw_ostream &os) const override;
+
+  StableSerializationPath
+  findStableSerializationPath(const clang::Decl *decl) const override;
+
+  const clang::Decl *
+  resolveStableSerializationPath(
+                            const StableSerializationPath &path) const override;
+
+  bool isSerializable(const clang::Type *type,
+                      bool checkCanonical) const override;
 };
 
 ImportDecl *createImportDecl(ASTContext &Ctx, DeclContext *DC, ClangNode ClangN,

include/swift/ClangImporter/SwiftAbstractBasicReader.h

@@ -0,0 +1,95 @@
+//===- SwiftAbstractBasicReader.h - Clang serialization adapter -*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides an intermediate CRTP class which implements most of
+// Clang's AbstractBasicReader interface, paralleling the behavior defined
+// in SwiftAbstractBasicWriter.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_CLANGIMPORTER_SWIFTABSTRACTBASICREADER_H
+#define SWIFT_CLANGIMPORTER_SWIFTABSTRACTBASICREADER_H
+
+#include "clang/AST/AbstractTypeReader.h"
+
+// This include is required to instantiate the template code in
+// AbstractBasicReader.h, i.e. it's a workaround to an include-what-you-use
+// violation.
+#include "clang/AST/DeclObjC.h"
+
+namespace swift {
+
+/// An implementation of Clang's AbstractBasicReader interface for a Swift
+/// datastream-based reader.  This is paired with the AbstractBasicWriter
+/// implementation in SwiftAbstractBasicWriter.h.  Note that the general
+/// expectation is that the types and declarations involved will have passed
+/// a serializability check when this is used for actual deserialization.
+///
+/// The subclass must implement:
+///   uint64_t readUInt64();
+///   clang::IdentifierInfo *readIdentifier();
+///   clang::Stmt *readStmtRef();
+///   clang::Decl *readDeclRef();
+template <class Impl>
+class DataStreamBasicReader
+       : public clang::serialization::DataStreamBasicReader<Impl> {
+  using super = clang::serialization::DataStreamBasicReader<Impl>;
+public:
+  using super::asImpl;
+  using super::getASTContext;
+
+  DataStreamBasicReader(clang::ASTContext &ctx) : super(ctx) {}
+
+  /// Perform all the calls necessary to write out the given type.
+  clang::QualType readTypeRef() {
+    auto kind = clang::Type::TypeClass(asImpl().readUInt64());
+    return clang::serialization::AbstractTypeReader<Impl>(asImpl()).read(kind);
+  }
+
+  bool readBool() {
+    return asImpl().readUInt64() != 0;
+  }
+
+  uint32_t readUInt32() {
+    return uint32_t(asImpl().readUInt64());
+  }
+
+  clang::Selector readSelector() {
+    uint64_t numArgsPlusOne = asImpl().readUInt64();
+
+    // The null case.
+    if (numArgsPlusOne == 0)
+      return clang::Selector();
+
+    unsigned numArgs = unsigned(numArgsPlusOne - 1);
+    SmallVector<clang::IdentifierInfo *, 4> chunks;
+    for (unsigned i = 0, e = std::max(numArgs, 1U); i != e; ++i)
+      chunks.push_back(asImpl().readIdentifier());
+
+    return getASTContext().Selectors.getSelector(numArgs, chunks.data());
+  }
+
+  clang::SourceLocation readSourceLocation() {
+    // Always read null.
+    return clang::SourceLocation();
+  }
+
+  clang::QualType readQualType() {
+    clang::Qualifiers quals = asImpl().readQualifiers();
+    clang::QualType type = asImpl().readTypeRef();
+    return getASTContext().getQualifiedType(type, quals);
+  }
+};
+
+}
+
+#endif

include/swift/ClangImporter/SwiftAbstractBasicWriter.h

@@ -0,0 +1,90 @@
+//===- SwiftAbstractBasicWriter.h - Clang serialization adapter -*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides an intermediate CRTP class which implements most of
+// Clang's AbstractBasicWriter interface, allowing largely the same logic
+// to be used for both the importer's "can this be serialized" checks and
+// the serializer's actual serialization logic.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_CLANGIMPORTER_SWIFTABSTRACTBASICWRITER_H
+#define SWIFT_CLANGIMPORTER_SWIFTABSTRACTBASICWRITER_H
+
+#include "clang/AST/AbstractTypeWriter.h"
+
+namespace swift {
+
+/// An implementation of Clang's AbstractBasicWriter interface for a Swift
+/// datastream-based reader.  This is paired with the AbstractBasicReader
+/// implementation in SwiftAbstractBasicReader.h.  Note that the general
+/// expectation is that the types and declarations involved will have passed
+/// a serializability check when this is used for actual serialization.
+/// The code in this class is also used when implementing that
+/// serializability check and so must be a little more cautious.
+///
+/// The subclass must implement:
+///   void writeUInt64(uint64_t value);
+///   void writeIdentifier(const clang::IdentifierInfo *ident);
+///   void writeStmtRef(const clang::Stmt *stmt);
+///   void writeDeclRef(const clang::Decl *decl);
+template <class Impl>
+class DataStreamBasicWriter
+       : public clang::serialization::DataStreamBasicWriter<Impl> {
+  using super = clang::serialization::DataStreamBasicWriter<Impl>;
+public:
+  using super::asImpl;
+
+  /// Perform all the calls necessary to write out the given type.
+  void writeTypeRef(const clang::Type *type) {
+    asImpl().writeUInt64(uint64_t(type->getTypeClass()));
+    clang::serialization::AbstractTypeWriter<Impl>(asImpl()).write(type);
+  }
+
+  void writeBool(bool value) {
+    asImpl().writeUInt64(uint64_t(value));
+  }
+
+  void writeUInt32(uint32_t value) {
+    asImpl().writeUInt64(uint64_t(value));
+  }
+
+  void writeSelector(clang::Selector selector) {
+    if (selector.isNull()) {
+      asImpl().writeUInt64(0);
+      return;
+    }
+
+    asImpl().writeUInt64(selector.getNumArgs() + 1);
+    for (unsigned i = 0, e = std::max(selector.getNumArgs(), 1U); i != e; ++i)
+      asImpl().writeIdentifier(selector.getIdentifierInfoForSlot(i));
+  }
+
+  void writeSourceLocation(clang::SourceLocation loc) {
+    // DataStreamBasicReader will always read null; the serializability
+    // check overrides this to complain about non-null source locations.
+  }
+
+  void writeQualType(clang::QualType type) {
+    assert(!type.isNull());
+
+    auto split = type.split();
+    asImpl().writeQualifiers(split.Quals);
+
+    // Just recursively visit the given type.
+    asImpl().writeTypeRef(split.Ty);
+  }
+};
+
+}
+
+#endif

lib/AST/ASTContext.cpp

@@ -4391,6 +4391,18 @@ ASTContext::getClangFunctionType(ArrayRef<AnyFunctionType::Param> params,
   return impl.Converter.getValue().getFunctionType(params, resultTy, trueRep);
 }
 
+const Decl *
+ASTContext::getSwiftDeclForExportedClangDecl(const clang::Decl *decl) {
+  auto &impl = getImpl();
+
+  // If we haven't exported anything yet, this must not be how we found
+  // this declaration.
+  if (!impl.Converter) return nullptr;
+
+  return impl.Converter->getSwiftDeclForExportedClangDecl(decl);
+}
+
+
 CanGenericSignature ASTContext::getSingleGenericParameterSignature() const {
   if (auto theSig = getImpl().SingleGenericParameterSignature)
     return theSig;