Integrate the assert_writeln_magic in the preprocessor

Author: wilzbach

ddoc/.gitignore

@@ -0,0 +1,3 @@
+/libddoc_preprocessor.a
+/ddoc_preprocessor
+/ddoc_preprocessor-test-application

ddoc/dub.sdl

@@ -0,0 +1,4 @@
+name "ddoc_preprocessor"
+description "Preprocesses source code before running Ddoc over it"
+dependency "libdparse" version="0.7.2-alpha.4"
+targetType "executable"

ddoc/dub.selections.json

@@ -0,0 +1,6 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"libdparse": "0.7.2-alpha.4"
+	}
+}

ddoc/source/assert_writeln_magic.d

@@ -192,87 +192,6 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
     visitor.visit(m);
 }
 
-void parseFile(string fileName, string destFile)
-{
-    import std.array : uninitializedArray;
-
-    auto inFile = File(fileName);
-    if (inFile.size == 0)
-        warningf("%s is empty", inFile.name);
-
-    ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
-    if (sourceCode.length == 0)
-        return;
-
-    inFile.rawRead(sourceCode);
-    auto fl = FileLines(fileName, destFile);
-    auto visitor = new TestVisitor!(typeof(fl))(fl);
-    parseString(sourceCode, fileName, visitor);
-    delete visitor;
-}
-
-// Modify a path under oldBase to a new path with the same subpath under newBase.
-// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar`
-string rebasePath(string path, string oldBase, string newBase)
-{
-    import std.path : absolutePath, buildPath, relativePath;
-    return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath));
-}
-
-version(unittest) { void main(){} } else
-void main(string[] args)
-{
-    import std.file;
-    import std.getopt;
-    import std.path;
-
-    string inputDir, outputDir;
-    string[] ignoredFiles;
-
-    auto helpInfo = getopt(args, config.required,
-            "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
-            "outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir,
-            "ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles);
-
-    if (helpInfo.helpWanted)
-    {
-        return defaultGetoptPrinter(`assert_writeln_magic
-Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls.
-`, helpInfo.options);
-    }
-
-    inputDir = inputDir.asNormalizedPath.array;
-
-    DirEntry[] files;
-
-    // inputDir as default output directory
-    if (!outputDir.length)
-        outputDir = inputDir;
-
-    if (inputDir.isFile)
-    {
-        files = [DirEntry(inputDir)];
-        inputDir = "";
-    }
-    else
-    {
-        files = dirEntries(inputDir, SpanMode.depth).filter!(
-                a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
-    }
-
-    foreach (file; files)
-    {
-        if (!ignoredFiles.any!(x => file.name.canFind(x)))
-        {
-            // single files
-            if (inputDir.length == 0)
-                parseFile(file.name, outputDir);
-            else
-                parseFile(file.name, file.name.rebasePath(inputDir, outputDir));
-        }
-    }
-}
-
 /**
 A simple line-based in-memory representation of a file.
  - will automatically write all changes when the object is destructed
@@ -284,47 +203,26 @@ struct FileLines
 
     string[] lines;
     string destFile;
-    bool overwriteInputFile;
     bool hasWrittenChanges;
 
-    this(string inputFile, string destFile)
+    this(string inputText)
     {
-        stderr.writefln("%s -> %s", inputFile, destFile);
-        this.overwriteInputFile = inputFile == destFile;
-        this.destFile = destFile;
-        lines = File(inputFile).byLineCopy.array;
-
-        destFile.dirName.mkdirRecurse;
-    }
-
-    // dumps all changes
-    ~this()
-    {
-        if (overwriteInputFile)
-        {
-            if (hasWrittenChanges)
-            {
-                auto tmpFile = File(destFile ~ ".tmp", "w");
-                writeLinesToFile(tmpFile);
-                tmpFile.close;
-                tmpFile.name.rename(destFile);
-            }
-        }
-        else
-        {
-            writeLinesToFile(File(destFile, "w"));
-        }
+        lines = inputText.split("\n");
     }
 
     // writes all changes to a random, temporary file
-    void writeLinesToFile(File outFile) {
+    auto buildLines() {
+        auto app = appender!string;
         // dump file
         foreach (line; lines)
-            outFile.writeln(line);
+        {
+            app ~= line;
+            app ~= "\n";
+        }
         // within the docs we automatically inject std.stdio (hence we need to do the same here)
         // writeln needs to be @nogc, @safe, pure and nothrow (we just fake it)
-        outFile.writeln("// \nprivate void writeln(T)(T l) { }");
-        outFile.flush;
+        app ~= "// \nprivate void writeln(T)(T l) { }";
+        return app.data;
     }
 
     string opIndex(size_t i) { return lines[i]; }

ddoc/source/preprocessor.d

@@ -22,10 +22,11 @@ struct Config
 {
     string dmdBinPath = "dmd";
     string outputFile;
-    string cwd = __FILE_FULL_PATH__.dirName;
+    string cwd = __FILE_FULL_PATH__.dirName.dirName.dirName;
 }
 Config config;
 
+version(unittest) {} else
 int main(string[] rootArgs)
 {
     import std.getopt;
@@ -46,32 +47,53 @@ All unknown options are passed to the compiler.
     assert(pos >= 0, "An input file (.d or .dd) must be provided");
     auto inputFile = args[pos];
     auto text = inputFile.readText;
-    // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
-    args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
+
+    // for now only non-package modules are supported
+    if (!inputFile.endsWith("package.d", "index.d"))
+        // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
+        args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
 
     // transform and extend the ddoc page
     text = genGrammar(text);
     text = genHeader(text);
     text = genChangelogVersion(inputFile, text);
 
+    if (inputFile.endsWith(".d"))
+        text = assertWriteln(inputFile, text);
+
     // inject custom, "dynamic" macros
-    text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    if (inputFile.endsWith(".dd", "index.d"))
+        text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
+    else if (inputFile.endsWith(".d"))
+        text ~= "/**\nMacros:\n\nSRC_FILENAME=%s\n*/".format(inputFile.buildNormalizedPath);
     return compile(text, args);
 }
 
 auto compile(R)(R buffer, string[] arguments)
 {
+    import core.time : usecs;
+    import core.thread : Thread;
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
-    foreach (arg; ["-c", "-o-", "-"])
+
+    import std.file : write, tempDir;
+    import std.uuid : randomUUID;
+    auto fileName = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", "") ~ ".d");
+    std.file.write(fileName, buffer);
+    scope(exit) fileName.remove;
+    args = args.replace("-", fileName);
+
+    foreach (arg; ["-c", "-o-"])
     {
         if (!args.canFind(arg))
             args ~= arg;
     }
     auto pipes = pipeProcess(args, Redirect.stdin);
-    pipes.stdin.write(buffer);
-    pipes.stdin.close;
-    return wait(pipes.pid);
+    import std.process : execute;
+    auto ret = execute(args);
+    if (ret.status != 0)
+        stderr.writeln(ret.output);
+    return ret.status;
 }
 
 // replaces the content of a DDoc macro call
@@ -277,3 +299,13 @@ auto genChangelogVersion(string fileName, string fileText)
     }
     return fileText;
 }
+
+auto assertWriteln(string fileName, string fileText)
+{
+    import assert_writeln_magic;
+    auto fl = FileLines(fileText);
+    auto visitor = new TestVisitor!(typeof(fl))(fl);
+    parseString(cast(ubyte[]) fileText.representation, fileName, visitor);
+    delete visitor;
+    return fl.buildLines;
+}

posix.mak

@@ -134,6 +134,9 @@
 #  before actually running Ddoc.
 #  Currently this is used for:
 #    - TOC
+#    - dynamic TOC generation
+#    - GRAMMAR overview generation
+#    - CHANGELOG menu generation
 #    - assert -> writeln magic
 PWD=$(shell pwd)
 MAKEFILE=$(firstword $(MAKEFILE_LIST))
@@ -282,7 +285,7 @@ DDOC_VARS_PRERELEASE_VERBATIM=$(DDOC_VARS_PRERELEASE) \
 # Ddoc binaries
 ################################################################################
 
-DDOC_BIN:=$G/ddoc
+DDOC_BIN:=$G/ddoc_preprocessor
 
 ################################################################################
 # Resources
@@ -669,23 +672,24 @@ $W/phobos-prerelease/object.verbatim : $(DMD) $G/changelog/next-version
 ################################################################################
 
 .PHONY: phobos-prerelease
-phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD)
+phobos-prerelease : ${PHOBOS_FILES} druntime-target $(STD_DDOC_PRERELEASE) $(DDOC_BIN) $(DMD) \
+					$G/changelog/next-version
 	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_PRERELEASE_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
 
 phobos-release : ${PHOBOS_FILES} druntime-target $(STD_DDOC_RELEASE) $(DDOC_BIN) $(DMD)
 	$(MAKE) --directory=$(PHOBOS_DIR) -f posix.mak html $(DDOC_VARS_RELEASE_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD))"
 
-phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD)
+phobos-latest : ${PHOBOS_LATEST_FILES} druntime-latest-target $(STD_DDOC_LATEST) $(DDOC_BIN) $(DMD_LATEST)
 	$(MAKE) --directory=$(PHOBOS_LATEST_DIR) -f posix.mak html $(DDOC_VARS_LATEST_HTML) \
-		DMD="$(abspath $(DDOC_BIN)) --compiler=$(DMD)"
+		DMD="$(abspath $(DDOC_BIN)) --compiler=$(abspath $(DMD_LATEST))"
 
 phobos-prerelease-verbatim : ${PHOBOS_FILES} druntime-target \
 		$W/phobos-prerelease/index.verbatim
 $W/phobos-prerelease/index.verbatim : verbatim.ddoc \
 		$W/phobos-prerelease/object.verbatim \
-		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version
+		$W/phobos-prerelease/mars.verbatim $G/changelog/next-version $(DMD)
 	${MAKE} --directory=${PHOBOS_DIR} -f posix.mak html $(DDOC_VARS_PRERELEASE_VERBATIM) \
 	  DOC_OUTPUT_DIR=$W/phobos-prerelease-verbatim
 	$(call CHANGE_SUFFIX,html,verbatim,$W/phobos-prerelease-verbatim)
@@ -838,7 +842,7 @@ test_dspec: dspec_tester.d $(DMD) $(PHOBOS_LIB)
 	@echo "Test the D Language specification"
 	$(DMD) -run $< --compiler=$(DMD)
 
-test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all
+test: test_dspec test/next_version.sh all
 	@echo "Searching for trailing whitespace"
 	@grep -n '[[:blank:]]$$' $$(find . -type f -name "*.dd" | grep -v .generated) ; test $$? -eq 1
 	@echo "Searching for tabs"
@@ -847,8 +851,8 @@ test: $(ASSERT_WRITELN_BIN)_test test_dspec test/next_version.sh all
 	@grep -n "UNDEFINED MACRO" $$(find $W -type f -name "*.html" -not -path "$W/phobos/*") ; test $$? -eq 1
 	@echo "Searching for undefined ddoc"
 	@grep -rn '[$$](' $$(find $W/phobos-prerelease -type f -name "*.html") ; test $$? -eq 1
-	@echo "Executing assert_writeln_magic tests"
-	$<
+	@echo "Executing ddoc_preprocessor tests"
+	dub -C ddoc test
 	@echo "Executing next_version tests"
 	test/next_version.sh
 
@@ -933,22 +937,12 @@ $G/contributors_list.ddoc:  | $(STABLE_RDMD) $(TOOLS_DIR) $(INSTALLER_DIR)
 # Custom DDoc wrapper
 # ------------------
 #
-# This allows extending Ddoc files dynamically on-the-fly.
-# It is currently only used for the specification pages
-#
-# It does:
-# - dynamic TOC generation
-# - GRAMMAR overview generation
-# - CHANGELOG menu generation
-# - Assert -> writeln magic (https://dlang.org/blog/2017/03/08/editable-and-runnable-doc-examples-on-dlang-org)
-#  - This transforms assert(a == b) to writeln(a); // b
-#  - It creates a copy of Phobos to apply the transformations
-#  - All "d" files are piped through the transformator,
-#   other needed files (e.g. posix.mak) get copied over
+# This allows extending Ddoc files and D source code files dynamically on-the-fly.
 ################################################################################
 
-$(DDOC_BIN): ddoc_preprocessor.d | $(STABLE_DMD)
-	$(STABLE_DMD) -g -of$@ $<
+$(DDOC_BIN): ddoc/source/preprocessor.d ddoc/source/assert_writeln_magic.d | $(STABLE_DMD)
+	$(STABLE_DMD_BIN_ROOT)/dub build --compiler=$(STABLE_DMD) --root=ddoc && \
+		mv ddoc/ddoc_preprocessor $@
 
 ################################################################################
 # Build and render the DMD man page