Integrate the assert_writeln_magic in the preprocessor

Author: wilzbach

ddoc/.gitignore

@@ -0,0 +1,2 @@
+/libddoc_preprocessor.a
+/ddoc_preprocessor

ddoc/dub.sdl

@@ -0,0 +1,4 @@
+name "ddoc_preprocessor"
+description "Preprocessos source code before running Ddoc over it"
+dependency "libdparse" version="0.7.2-alpha.4"
+targetType "executable"

ddoc/dub.selections.json

@@ -0,0 +1,6 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"libdparse": "0.7.2-alpha.4"
+	}
+}

ddoc/source/assert_writeln_magic.d

@@ -192,87 +192,6 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
     visitor.visit(m);
 }
 
-void parseFile(string fileName, string destFile)
-{
-    import std.array : uninitializedArray;
-
-    auto inFile = File(fileName);
-    if (inFile.size == 0)
-        warningf("%s is empty", inFile.name);
-
-    ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
-    if (sourceCode.length == 0)
-        return;
-
-    inFile.rawRead(sourceCode);
-    auto fl = FileLines(fileName, destFile);
-    auto visitor = new TestVisitor!(typeof(fl))(fl);
-    parseString(sourceCode, fileName, visitor);
-    delete visitor;
-}
-
-// Modify a path under oldBase to a new path with the same subpath under newBase.
-// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar`
-string rebasePath(string path, string oldBase, string newBase)
-{
-    import std.path : absolutePath, buildPath, relativePath;
-    return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath));
-}
-
-version(unittest) { void main(){} } else
-void main(string[] args)
-{
-    import std.file;
-    import std.getopt;
-    import std.path;
-
-    string inputDir, outputDir;
-    string[] ignoredFiles;
-
-    auto helpInfo = getopt(args, config.required,
-            "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
-            "outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir,
-            "ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles);
-
-    if (helpInfo.helpWanted)
-    {
-        return defaultGetoptPrinter(`assert_writeln_magic
-Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls.
-`, helpInfo.options);
-    }
-
-    inputDir = inputDir.asNormalizedPath.array;
-
-    DirEntry[] files;
-
-    // inputDir as default output directory
-    if (!outputDir.length)
-        outputDir = inputDir;
-
-    if (inputDir.isFile)
-    {
-        files = [DirEntry(inputDir)];
-        inputDir = "";
-    }
-    else
-    {
-        files = dirEntries(inputDir, SpanMode.depth).filter!(
-                a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
-    }
-
-    foreach (file; files)
-    {
-        if (!ignoredFiles.any!(x => file.name.canFind(x)))
-        {
-            // single files
-            if (inputDir.length == 0)
-                parseFile(file.name, outputDir);
-            else
-                parseFile(file.name, file.name.rebasePath(inputDir, outputDir));
-        }
-    }
-}
-
 /**
 A simple line-based in-memory representation of a file.
  - will automatically write all changes when the object is destructed
@@ -284,47 +203,26 @@ struct FileLines
 
     string[] lines;
     string destFile;
-    bool overwriteInputFile;
     bool hasWrittenChanges;
 
-    this(string inputFile, string destFile)
+    this(string inputText)
     {
-        stderr.writefln("%s -> %s", inputFile, destFile);
-        this.overwriteInputFile = inputFile == destFile;
-        this.destFile = destFile;
-        lines = File(inputFile).byLineCopy.array;
-
-        destFile.dirName.mkdirRecurse;
-    }
-
-    // dumps all changes
-    ~this()
-    {
-        if (overwriteInputFile)
-        {
-            if (hasWrittenChanges)
-            {
-                auto tmpFile = File(destFile ~ ".tmp", "w");
-                writeLinesToFile(tmpFile);
-                tmpFile.close;
-                tmpFile.name.rename(destFile);
-            }
-        }
-        else
-        {
-            writeLinesToFile(File(destFile, "w"));
-        }
+        lines = inputText.split("\n");
     }
 
     // writes all changes to a random, temporary file
-    void writeLinesToFile(File outFile) {
+    auto buildLines() {
+        auto app = appender!string;
         // dump file
         foreach (line; lines)
-            outFile.writeln(line);
+        {
+            app ~= line;
+            app ~= "\n";
+        }
         // within the docs we automatically inject std.stdio (hence we need to do the same here)
         // writeln needs to be @nogc, @safe, pure and nothrow (we just fake it)
-        outFile.writeln("// \nprivate void writeln(T)(T l) { }");
-        outFile.flush;
+        app ~= "// \nprivate void writeln(T)(T l) { }";
+        return app.data;
     }
 
     string opIndex(size_t i) { return lines[i]; }

ddoc/source/preprocessor.d

@@ -46,23 +46,40 @@ All unknown options are passed to the compiler.
     assert(pos >= 0, "An input file (.d or .dd) must be provided");
     auto inputFile = args[pos];
     auto text = inputFile.readText;
-    // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
-    args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
+
+    // for now only non-package modules are supported
+    if (!inputFile.endsWith("package.d"))
+        // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
+        args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
 
     // transform and extend the ddoc page
     text = genGrammar(text);
     text = genHeader(text);
     text = genChangelogVersion(inputFile, text);
+    text = assertWriteln(inputFile, text);
 
     // inject custom, "dynamic" macros
-    text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    if (inputFile.endsWith(".dd"))
+        text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    else if (inputFile.endsWith(".d"))
+        text ~= "/**\nMacros:\n\nSRC_FILENAME=%s\n*/".format(inputFile.buildNormalizedPath);
     return compile(text, args);
 }
 
 auto compile(R)(R buffer, string[] arguments)
 {
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
+
+    // writing to a temporary file might be needed to support package.d modules
+    //import std.file : write, tempDir;
+    //import std.uuid : randomUUID;
+    //auto fileName = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ ".d");
+    //std.file.write(fileName, buffer);
+    //writeln(fileName);
+    //scope(exit) fileName.remove;
+    //args = args.replace("-", fileName);
+
     foreach (arg; ["-c", "-o-", "-"])
     {
         if (!args.canFind(arg))
@@ -72,6 +89,11 @@ auto compile(R)(R buffer, string[] arguments)
     pipes.stdin.write(buffer);
     pipes.stdin.close;
     return wait(pipes.pid);
+    //import std.process : execute;
+    //auto ret = execute(args);
+    //if (ret.status != 0)
+        //stderr.writeln(ret.output);
+    //return ret.status;
 }
 
 // replaces the content of a DDoc macro call
@@ -277,3 +299,13 @@ auto genChangelogVersion(string fileName, string fileText)
     }
     return fileText;
 }
+
+auto assertWriteln(string fileName, string fileText)
+{
+    import assert_writeln_magic;
+    auto fl = FileLines(fileText);
+    auto visitor = new TestVisitor!(typeof(fl))(fl);
+    parseString(cast(ubyte[]) fileText.representation, fileName, visitor);
+    delete visitor;
+    return fl.buildLines;
+}

posix.mak

@@ -282,7 +282,7 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \
 # Ddoc binaries
 ################################################################################
 
-DDOC_BIN:=$G/ddoc
+DDOC_BIN:=$G/ddoc_preprocessor
 
 ################################################################################
 # Resources
@@ -947,8 +947,9 @@ $G/contributors_list.ddoc:  | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
 #   other needed files (e.g. posix.mak) get copied over
 ################################################################################
 
-$(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD)
-	$(STABLE_DMD) -g -of$@ $<
+$(DDOC_BIN): ddoc/source/preprocessor.d ddoc/source/assert_writeln_magic.d | $(STABLE_DMD)
+	$(STABLE_DMD_BIN_ROOT)/dub build --compiler=$(STABLE_DMD) --root=ddoc && \
+		mv ddoc/ddoc_preprocessor $@
 
 ################################################################################
 # Build and render the DMD man page