The Great Desugaring

A major source of headaches and probably the source ldc-developers#1 of mapping failures for sophisticated C++ libraries has been the restrictions of D template instantiation scope. Those restrictions are totally necessary and work great in D, and Clang adding "sugar" to its types made it possible to comply with them... most of the time.

One particular wall was hit while trying to map the MSVC standard library. For type_traits's conjunction<_Is_character<char>, _Is_character<char>>, the base class in Clang's AST is:

    TemplateSpecializationType 0xdfaed30 '_Conjunction<struct std::_Is_character<char>, struct std::_Is_character<char> >' sugar _Conjunction
        | -TemplateArgument type 'struct std::_Is_character<char>':'struct std::_Is_character<char>'
        | SubstTemplateTypeParmType 0xdda9f20 'struct std::_Is_character<char>' sugar
            | -TemplateTypeParmType 0xa7ce0f0 '_Traits' dependent contains_unexpanded_pack depth 0 index 0 pack
            | `-TemplateTypeParm 0xa7ce0c0 '_Traits'
            `-RecordType 0xa9079d0 'struct std::_Is_character<char>'
                `-ClassTemplateSpecialization 0xa907930 '_Is_character'
        | -TemplateArgument type 'struct std::_Is_character<char>' : 'struct std::_Is_character<char>'
        | SubstTemplateTypeParmType 0xdda9f20 'struct std::_Is_character<char>' sugar (...same as above....)
        `-RecordType 0xdfaed10 'struct std::_Conjunction<struct std::_Is_character<char>, struct std::_Is_character<char> >'
        `-ClassTemplateSpecialization 0xdfaec78 '_Conjunction'

This is one of those rare cases where Clang has lost pack information and no simple/good heuristic to guess when two or more arguments come from the same pack appears to be possible. Hence either Clang needs to be modified to preserve pack info, or a complicated check/heuristic is needed, or we've got to stop trying to stick to DMD's way.

The third option was chosen because there's actually no good reason to comply with those restrictions. Reflection doesn't get improved, and while instantiation from C++ modules, every referenced symbol is from C++ modules.

So from now on:
 - Every type gets desugared before being mapped in non-dependent contexts.
 - C++ modules now have access to the "C++ global namespace" during name lookups, through a special type of global import.

This allows to discard the C++ symbol collector and substitution complicated hack when mapping template arguments deduced by Sema.

Author: Syniurge

dmd2/cpp/cppdeclaration.cpp

@@ -233,14 +233,6 @@ void DtorDeclaration::semantic(Scope *sc)
     ::DtorDeclaration::semantic(sc);
 }
 
-// Cheat and use the C++ "global scope", we can do it safely in specific cases
-Scope *globalScope(::Module *m)
-{
-    auto sc = Scope::createGlobal(m);
-    sc = sc->push(cpp::Module::rootPackage);
-    return sc;
-}
-
 void DeclReferencer::Traverse(Loc loc, Scope *sc, clang::Stmt *S)
 {
     this->loc = loc;
@@ -490,12 +482,11 @@ void FuncDeclaration::semantic3reference(::FuncDeclaration *fd, Scope *sc)
     const clang::FunctionDecl *Def;
     if (!FD->isInvalidDecl() && FD->hasBody(Def))
     {
-        auto globalSc = globalScope(sc->instantiatingModule());
-        declReferencer.Traverse(fd->loc, globalSc, Def->getBody());
+        declReferencer.Traverse(fd->loc, sc, Def->getBody());
 
         if (auto Ctor = dyn_cast<clang::CXXConstructorDecl>(FD))
             for (auto& Init: Ctor->inits())
-                declReferencer.Traverse(fd->loc, globalSc, Init->getInit());
+                declReferencer.Traverse(fd->loc, sc, Init->getInit());
     }
 
     fd->semanticRun = PASSsemantic3done;

dmd2/cpp/cppdeclaration.h

@@ -208,7 +208,6 @@ class DeclReferencer : public clang::RecursiveASTVisitor<DeclReferencer>
     DeclReferencer() : expmap(mapper)
     {
         mapper.addImplicitDecls = false;
-        mapper.cppPrefix = false;
     }
 
     void Traverse(Loc loc, Scope *sc, clang::Stmt *S);
@@ -221,7 +220,6 @@ class DeclReferencer : public clang::RecursiveASTVisitor<DeclReferencer>
 };
 
 extern DeclReferencer declReferencer;
-Scope *globalScope(::Module *m);
 
 const clang::Decl *getCanonicalDecl(const clang::Decl *D); // the only difference with D->getCanonicalDecl() is that if the canonical decl is an out-of-ilne friend' decl and the actual decl is declared, this returns the latter instead of the former
 bool isPolymorphic(const clang::RecordDecl *D);

dmd2/cpp/cppexpression.cpp

@@ -478,9 +478,8 @@ Expression* ExprMapper::fromExpression(const clang::Expr *E, bool interpret)  //
     }
     else if (auto SNTTP = dyn_cast<clang::SubstNonTypeTemplateParmExpr>(E))
     {
-        if (/*!interpret || */SNTTP->isValueDependent()) // NOTE/FIXME(?): this doesn't work with packs, which get added n times if they have n elements, and Clang doesn't store pack info in the SNTTP
-            e = fromExpressionNonTypeTemplateParm(loc,
-                                SNTTP->getParameter());
+        if (SNTTP->isValueDependent())
+            e = fromExpressionNonTypeTemplateParm(loc, SNTTP->getParameter());
         else
             e = fromExpression(SNTTP->getReplacement());
     }

dmd2/cpp/cppimport.cpp

@@ -16,10 +16,10 @@ namespace cpp
 Import::Import(Loc loc, Identifiers *packages, Identifier *id, Identifier *aliasId, int isstatic)
     : ::Import(loc, packages, id, aliasId, isstatic)
 {
-    // add "_cpp" as leftmost package to avoid name clashes
+    // add "§cpp" as leftmost package to avoid name clashes
     if (!this->packages)
         this->packages = new Identifiers;
-    this->packages->shift(Identifier::idPool("_cpp"));  // any better idea ?
+    this->packages->shift(Identifier::idPool(u8"§cpp"));
 
     if (!aliasId)
         setSymIdent();
@@ -63,4 +63,19 @@ void Modmap::semantic(Scope* sc)
 {
 }
 
+GlobalImport::GlobalImport(Loc loc)
+    : ::Import(loc, nullptr, Identifier::idPool(u8"§cpp"), nullptr, 0)
+{
+    this->pkg = cpp::Module::rootPackage;
+}
+
+void GlobalImport::load(Scope *sc)
+{
+    if (loaded)
+        return;
+
+    sc->scopesym->importScope(pkg, Prot(PROTprivate));
+    loaded = true;
+}
+
 }

dmd2/cpp/cppimport.h

@@ -22,8 +22,8 @@ class Import : public ::Import
 
     Import(Loc loc, Identifiers *packages, Identifier *id, Identifier *aliasId, int isstatic);
 
-    ::Module *loadModule(Loc loc, Identifiers *packages, Identifier *id);
-    void load(Scope *sc) override; // HACK
+    ::Module *loadModule(Loc loc, Identifiers *packages, Identifier *id) override;
+    void load(Scope *sc) override;
 };
 
 struct Modmap : public ::Modmap
@@ -33,8 +33,20 @@ struct Modmap : public ::Modmap
 
     Modmap(Loc loc, StringExp *arg);
 
-    void importAll(Scope *sc);
-    void semantic(Scope *sc);
+    void importAll(Scope *sc) override;
+    void semantic(Scope *sc) override;
+};
+
+// Special import for lookups from C++ modules
+class GlobalImport : public ::Import
+{
+public:
+    CALYPSO_LANGPLUGIN
+
+    bool loaded = false;
+
+    GlobalImport(Loc loc);
+    void load(Scope *sc) override;
 };
 
 }

dmd2/cpp/cppmodule.cpp

@@ -54,7 +54,7 @@ Modules Module::amodules;
 
 void Module::init()
 {
-    rootPackage = new Package(Identifier::idPool("_cpp"));
+    rootPackage = new Package(Identifier::idPool(u8"§cpp"));
     rootPackage->symtab = new DsymbolTable;
 
     modules->insert(rootPackage);
@@ -121,6 +121,8 @@ void Module::addPreambule()
             ::Import *im = new ::Import(Loc(), packages, Identifier::idPool("core"), nullptr, true);
             members->shift(im);
         }
+        // the C++ special import for global namespace lookups
+        members->shift(new GlobalImport(Loc()));
         { // object
             ::Import *im = new ::Import(Loc(), nullptr, Id::object, nullptr, true);
             members->shift(im);

dmd2/cpp/cpptemplate.cpp

@@ -54,100 +54,6 @@ TemplateDeclaration::TemplateDeclaration(const TemplateDeclaration &o)
 
 IMPLEMENT_syntaxCopy(TemplateDeclaration, TempOrSpec)
 
-// HACK-ish unfortunately.. but arg deduction isn't trivial. Can't think of a simpler way.
-// TODO: remove this, use cppdeclaration.cpp Referencer approach instead w/ the global scope
-struct CppSymCollector
-{
-    Dsymbols *substsyms;
-    CppSymCollector(Dsymbols *substsyms)
-        : substsyms(substsyms) {}
-
-    inline void addIfCPP(Dsymbol *s)
-    {
-        if (isCPP(s))
-            substsyms->push(s);
-
-        if (auto ti = s->parent->isTemplateInstance())
-            if (isCPP(ti))
-                collect(ti->tiargs);
-    }
-
-    void collect(Dsymbol *s)
-    {
-        addIfCPP(s);
-    }
-
-    void collect(Type *t)
-    {
-        switch (t->ty)
-        {
-            case Tstruct:
-                addIfCPP(static_cast<TypeStruct*>(t)->sym);
-                break;
-            case Tclass:
-                addIfCPP(static_cast<TypeClass*>(t)->sym);
-                break;
-            case Tenum:
-                addIfCPP(static_cast<TypeEnum*>(t)->sym);
-                break;
-            case Tarray:
-            case Tsarray:
-            case Tpointer:
-            case Treference:
-                collect(static_cast<TypeNext*>(t)->next);
-                break;
-            case Tfunction:
-            {
-                auto tf = static_cast<TypeFunction*>(t);
-                collect(tf->next);
-                for (auto p: *tf->parameters)
-                    collect(p->type);
-                break;
-            }
-            case Tident:
-            case Tinstance:
-            default:
-//                 ::warning(Loc(), "Collecting C++ symbols unhandled for type %s:\"%s\"",
-//                           t->kind(), t->toChars());
-                break;
-        }
-    }
-
-    void collect(Expression *e)
-    {
-        // TODO DotIdExp ...
-    }
-
-    void collect(Tuple *tup)
-    {
-        collect(&tup->objects);
-    }
-
-    void collect(Objects *tiargs)
-    {
-        for (auto o: *tiargs)
-        {
-            Type *ta = isType(o);
-            Expression *ea = isExpression(o);
-            Dsymbol *sa = isDsymbol(o);
-            Tuple *tupa = isTuple(o);
-
-            if (ta) collect(ta);
-            else if (ea) collect(ea);
-            else if (sa) collect(sa);
-            else { assert(tupa); collect(tupa); }
-        }
-    }
-};
-
-static Dsymbols *collectSymbols(Objects *tiargs)
-{
-    auto substsyms = new Dsymbols;
-    CppSymCollector(substsyms).collect(tiargs);
-
-    return substsyms;
-}
-
 static void fillTemplateArgumentListInfo(Loc loc, Scope *sc, clang::TemplateArgumentListInfo& Args,
                                 Objects *tiargs, const clang::RedeclarableTemplateDecl *Temp,
                                 TypeMapper& tymap, ExprMapper& expmap)
@@ -416,8 +322,6 @@ MATCH TemplateDeclaration::matchWithInstance(Scope *sc, ::TemplateInstance *ti,
 
     TypeMapper tymap;
     tymap.addImplicitDecls = false;
-    tymap.substsyms = collectSymbols(dedtypes);
-    tymap.desugar = true;
 
     auto InstArgs = (isForeignInstance(ti) ? getTemplateInstantiationArgs(Inst)
                                                             : getTemplateArgs(Inst))->asArray();
@@ -610,7 +514,6 @@ TemplateInstUnion TemplateDeclaration::getClangInst(Scope* sc, ::TemplateInstanc
     TypeMapper tymap;
     ExprMapper expmap(tymap);
     tymap.addImplicitDecls = false;
-    tymap.cppPrefix = false;
 
     auto Temp = const_cast<clang::RedeclarableTemplateDecl*>
                                 (getDefinition(getPrimaryTemplate(), false));
@@ -772,7 +675,6 @@ bool TemplateInstance::semanticTiargs(Scope* sc)
         TypeMapper tymap;
         ExprMapper expmap(tymap);
         tymap.addImplicitDecls = false;
-        tymap.cppPrefix = false;
 
         auto Args = getTemplateArgs(Inst)->asArray();
         auto Arg = Args.begin();
@@ -781,8 +683,6 @@ bool TemplateInstance::semanticTiargs(Scope* sc)
         SpecValue spec(tymap);
         getIdentifierOrNull(Temp, &spec);
 
-        auto sc2 = globalScope(sc->instantiatingModule());
-
         for (size_t i = spec ? 1 : 0; i < tiargs->dim; Arg++, Param++)
         {
             auto NTTPD = dyn_cast<clang::NonTypeTemplateParmDecl>(*Param);
@@ -795,7 +695,7 @@ bool TemplateInstance::semanticTiargs(Scope* sc)
             for (auto arg: *a) {
                 if (auto e = isExpression(arg)) {
                     assert(isExpression((*tiargs)[i]));
-                    (*tiargs)[i] = e->semantic(sc2);
+                    (*tiargs)[i] = e->semantic(sc);
                 }
                 i++;
             }
@@ -861,7 +761,6 @@ void TemplateInstance::correctTiargs()
 
         TypeMapper tymap;
         tymap.addImplicitDecls = false;
-        tymap.substsyms = collectSymbols(tiargs);
 
         primTiargs = tiargs;
         tiargs = TypeMapper::FromType(tymap, loc).fromTemplateArguments(Args.begin(), Args.end(),