[WIP] RIIR HtmlDocCk

fmease · fmease · commit 0e2954600540 · 2024-05-30T20:31:40.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -31,6 +31,7 @@ members = [
   "src/tools/miri/cargo-miri",
   "src/tools/rustdoc-themes",
   "src/tools/unicode-table-generator",
+  "src/tools/htmldocck",
   "src/tools/jsondocck",
   "src/tools/jsondoclint",
   "src/tools/llvm-bitcode-linker",
diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs
@@ -311,7 +311,8 @@ lint_any!(
     CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata";
     Compiletest, "src/tools/compiletest", "compiletest";
     CoverageDump, "src/tools/coverage-dump", "coverage-dump";
-    Jsondocck, "src/tools/jsondocck", "jsondocck";
+    HtmldocCk, "src/tools/htmldocck", "htmldocck";
+    JsondocCk, "src/tools/jsondocck", "jsondocck";
     Jsondoclint, "src/tools/jsondoclint", "jsondoclint";
     LintDocs, "src/tools/lint-docs", "lint-docs";
     LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker";
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
@@ -1766,13 +1766,18 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
         }
 
+        if mode == "rustdoc" {
+            // Use the beta compiler for htmldocck.
+            let compiler = compiler.with_stage(0);
+            cmd.arg("--htmldocck-path").arg(builder.ensure(tool::HtmlDocCk { compiler, target }));
+        }
+
         if mode == "rustdoc-json" {
-            // Use the beta compiler for jsondocck
-            let json_compiler = compiler.with_stage(0);
-            cmd.arg("--jsondocck-path")
-                .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }));
+            // Use the beta compiler for jsondocck.
+            let compiler = compiler.with_stage(0);
+            cmd.arg("--jsondocck-path").arg(builder.ensure(tool::JsonDocCk { compiler, target }));
             cmd.arg("--jsondoclint-path")
-                .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
+                .arg(builder.ensure(tool::JsonDocLint { compiler, target }));
         }
 
         if mode == "coverage-map" {
diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs
@@ -303,6 +303,7 @@ bootstrap_tool!(
     RustInstaller, "src/tools/rust-installer", "rust-installer";
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
     LintDocs, "src/tools/lint-docs", "lint-docs";
+    HtmlDocCk, "src/tools/htmldocck", "htmldocck";
     JsonDocCk, "src/tools/jsondocck", "jsondocck";
     JsonDocLint, "src/tools/jsondoclint", "jsondoclint";
     HtmlChecker, "src/tools/html-checker", "html-checker";
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs
@@ -764,7 +764,8 @@ impl<'a> Builder<'a> {
                 clippy::CollectLicenseMetadata,
                 clippy::Compiletest,
                 clippy::CoverageDump,
-                clippy::Jsondocck,
+                clippy::HtmldocCk,
+                clippy::JsondocCk,
                 clippy::Jsondoclint,
                 clippy::LintDocs,
                 clippy::LlvmBitcodeLinker,
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
@@ -193,9 +193,12 @@ pub struct Config {
     /// The coverage-dump executable.
     pub coverage_dump_path: Option<PathBuf>,
 
-    /// The Python executable to use for LLDB and htmldocck.
+    /// The Python executable to use for LLDB.
     pub python: String,
 
+    /// The htmldocck executable.
+    pub htmldocck_path: Option<String>,
+
     /// The jsondocck executable.
     pub jsondocck_path: Option<String>,
 
diff --git a/src/tools/compiletest/src/header/tests.rs b/src/tools/compiletest/src/header/tests.rs
@@ -131,6 +131,7 @@ impl ConfigBuilder {
             "--compile-lib-path=",
             "--run-lib-path=",
             "--python=",
+            // FIXME(fmease): Do we need to set htmldocck-path to "", too?
             "--jsondocck-path=",
             "--src-base=",
             "--build-base=",
diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs
@@ -48,7 +48,9 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
         .optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
         .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
+        // FIXME(fmease): fix docs here
         .reqopt("", "python", "path to python to use for doc tests", "PATH")
+        .optopt("", "htmldocck-path", "path to htmldocck to use for doc tests", "PATH")
         .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
         .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
         .optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM")
@@ -235,6 +237,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
         coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
         python: matches.opt_str("python").unwrap(),
+        htmldocck_path: matches.opt_str("htmldocck-path"),
         jsondocck_path: matches.opt_str("jsondocck-path"),
         jsondoclint_path: matches.opt_str("jsondoclint-path"),
         valgrind_path: matches.opt_str("valgrind-path"),
@@ -617,6 +620,7 @@ fn common_inputs_stamp(config: &Config) -> Stamp {
 
     if let Some(ref rustdoc_path) = config.rustdoc_path {
         stamp.add_path(&rustdoc_path);
+        // FIXME(fmease): Remove this one once the rewrite is completed.
         stamp.add_path(&rust_src_dir.join("src/etc/htmldocck.py"));
     }
 
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
@@ -3030,9 +3030,16 @@ impl<'test> TestCx<'test> {
         if self.props.check_test_line_numbers_match {
             self.check_rustdoc_test_option(proc_res);
         } else {
-            let root = self.config.find_rust_src_root().unwrap();
-            let mut cmd = Command::new(&self.config.python);
-            cmd.arg(root.join("src/etc/htmldocck.py")).arg(&out_dir).arg(&self.testpaths.file);
+            // FIXME(fmease): Temporary commented out code:
+            // FIXME(fmease): I don't like this unwrap!
+            let mut cmd = Command::new(self.config.htmldocck_path.as_ref().unwrap());
+            cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file);
+
+            // let root = self.config.find_rust_src_root().unwrap();
+            // let mut cmd = Command::new(&self.config.python);
+            // cmd.arg(root.join("src/etc/htmldocck.py"));
+            // cmd.arg(&out_dir).arg(&self.testpaths.file);
+
             if self.config.bless {
                 cmd.arg("--bless");
             }
diff --git a/src/tools/htmldocck/Cargo.toml b/src/tools/htmldocck/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "htmldocck"
+version = "0.1.0"
+description = "A test framework for rustdoc's HTML backend"
+edition = "2021"
+
+[dependencies]
+getopts = "0.2"
+regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
+shlex = "1.3.0"
+unicode-width = "0.1.4"
diff --git a/src/tools/htmldocck/src/cache.rs b/src/tools/htmldocck/src/cache.rs
@@ -0,0 +1,69 @@
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    path::Path,
+};
+
+use crate::error::DiagCtxt;
+
+pub(crate) struct Cache<'a> {
+    root: &'a Path,
+    // FIXME: `&'a str`s
+    files: HashMap<String, String>,
+    // FIXME: `&'a str`, comment what this is for -- `-`
+    last_path: Option<String>,
+}
+
+impl<'a> Cache<'a> {
+    pub(crate) fn new(root: &'a Path) -> Self {
+        Self { root, files: HashMap::new(), last_path: None }
+    }
+
+    // FIXME: check file vs. dir (`@has <PATH>` vs. `@has-dir <PATH>`)
+    /// Check if the path points to an existing entity.
+    pub(crate) fn has(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<bool, ()> {
+        // FIXME: should we use `try_exists` over `exists` instead? matters the most for `@!has <PATH>`.
+        let path = self.resolve(path, dcx)?;
+
+        Ok(self.files.contains_key(&path) || Path::new(&path).exists())
+    }
+
+    /// Load the contents of the given path.
+    pub(crate) fn load(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<&str, ()> {
+        let path = self.resolve(path, dcx)?;
+
+        Ok(match self.files.entry(path) {
+            Entry::Occupied(entry) => entry.into_mut(),
+            Entry::Vacant(entry) => {
+                // FIXME: better message, location
+                let data =
+                    std::fs::read_to_string(self.root.join(entry.key())).map_err(|error| {
+                        dcx.emit(&format!("failed to read file: {error}"), None, None)
+                    })?;
+                entry.insert(data)
+            }
+        })
+    }
+
+    // FIXME: &str -> &str if possible
+    fn resolve(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<String, ()> {
+        if path == "-" {
+            // FIXME: no cloning
+            return self
+                .last_path
+                .clone()
+                // FIXME better diag, location
+                .ok_or_else(|| {
+                    dcx.emit(
+                        "attempt to use `-` ('previous path') in the very first command",
+                        None,
+                        None,
+                    )
+                });
+        }
+
+        // While we could normalize the `path` at this point by
+        // using `std::path::absolute`, it's likely not worth it.
+        self.last_path = Some(path.clone());
+        Ok(path)
+    }
+}
diff --git a/src/tools/htmldocck/src/check.rs b/src/tools/htmldocck/src/check.rs
@@ -0,0 +1,165 @@
+use crate::cache::Cache;
+use crate::error::{DiagCtxt, Source};
+use crate::{Command, CommandKind};
+
+impl Command<'_> {
+    // FIXME: implement all checks!
+    // FIXME: move regex parsing etc. into the parser maybe
+    pub(crate) fn run(self, cache: &mut Cache<'_>, dcx: &mut DiagCtxt) -> Result<(), ()> {
+        let result = self.kind.run(cache, self.source.clone(), dcx)?;
+
+        if result == self.negated {
+            // FIXME: better diag
+            dcx.emit("check failed", self.source, None);
+            return Err(());
+        }
+
+        Ok(())
+    }
+}
+
+impl CommandKind {
+    fn run(
+        self,
+        cache: &mut Cache<'_>,
+        source: Source<'_>,
+        dcx: &mut DiagCtxt,
+    ) -> Result<bool, ()> {
+        Ok(match self {
+            Self::HasFile { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a file
+            Self::HasDir { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a directory
+            Self::Has { path, xpath, text } => {
+                let _data = cache.load(path, dcx)?;
+                _ = xpath;
+                _ = text;
+                true // FIXME
+            }
+            Self::HasRaw { path, text } => {
+                let data = cache.load(path, dcx)?;
+
+                if text.is_empty() {
+                    // fast path
+                    return Ok(true);
+                }
+
+                let text = channel_url::instantiate(&text, dcx)?;
+                let text = text.replace(|c: char| c.is_ascii_whitespace(), " ");
+                let data = data.replace(|c: char| c.is_ascii_whitespace(), " ");
+
+                data.contains(&text)
+            }
+            Self::Matches { path, xpath, pattern } => {
+                let _data = cache.load(path, dcx)?;
+                _ = xpath;
+
+                let Ok(_pattern) =
+                    regex::RegexBuilder::new(&pattern).unicode(true).build().map_err(|error| {
+                        // FIXME: better error message and location
+                        // FIXME: Use `regex_syntax` directly. Its error type exposes the
+                        // underlying span which we can then translate/offset.
+                        _ = error;
+                        dcx.emit(&format!("malformed regex"), Some(source), None)
+                    })
+                else {
+                    return Err(());
+                };
+
+                true // FIXME
+            }
+            Self::MatchesRaw { path, pattern } => {
+                let data = cache.load(path, dcx)?;
+                let pattern = channel_url::instantiate(&pattern, dcx)?;
+
+                if pattern.is_empty() {
+                    // fast path
+                    return Ok(true);
+                }
+
+                let Ok(pattern) =
+                    regex::RegexBuilder::new(&pattern).unicode(true).build().map_err(|error| {
+                        // FIXME: better error message and location
+                        // FIXME: Use `regex_syntax` directly. Its error type exposes the
+                        // underlying span which we can then translate/offset.
+                        _ = error;
+                        dcx.emit(&format!("malformed regex"), Some(source), None)
+                    })
+                else {
+                    return Err(());
+                };
+
+                pattern.is_match(data)
+            }
+            Self::Count { path, xpath, text, count } => {
+                let _data = cache.load(path, dcx)?;
+                _ = xpath;
+                _ = text;
+                _ = count;
+                true // FIXME
+            }
+            Self::Files { path, files } => {
+                let _data = cache.load(path, dcx)?;
+                _ = files;
+                true // FIXME
+            }
+            Self::Snapshot { name, path, xpath } => {
+                let _data = cache.load(path, dcx)?;
+                _ = name;
+                _ = path;
+                _ = xpath;
+                true // FIXME
+            }
+        })
+    }
+}
+
+mod channel_url {
+    use std::{borrow::Cow, sync::OnceLock};
+
+    use crate::error::DiagCtxt;
+
+    const PLACEHOLDER: &str = "{{channel}}";
+
+    pub(super) fn instantiate<'a>(input: &'a str, dcx: &mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
+        let Some(channel_url) = channel_url(dcx)? else { return Ok(input.into()) };
+        Ok(input.replace(PLACEHOLDER, channel_url).into())
+    }
+
+    #[allow(dead_code)] // FIXME
+    pub(super) fn anonymize<'a>(input: &'a str, dcx: &'_ mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
+        let Some(channel_url) = channel_url(dcx)? else { return Ok(input.into()) };
+        Ok(input.replace(channel_url, PLACEHOLDER).into())
+    }
+
+    fn channel_url(dcx: &mut DiagCtxt) -> Result<Option<&'static str>, ()> {
+        static CHANNEL_URL: OnceLock<Option<String>> = OnceLock::new();
+
+        // FIXME: Use `get_or_try_init` here (instead of `get`→`set`→`get`) if/once stabilized (on beta).
+
+        if let Some(channel_url) = CHANNEL_URL.get() {
+            return Ok(channel_url.as_deref());
+        }
+
+        const KEY: &str = "DOC_RUST_LANG_ORG_CHANNEL";
+
+        let channel_url = match std::env::var(KEY) {
+            Ok(url) => Some(url),
+            // FIXME: should we make the channel mandatory instead?
+            Err(std::env::VarError::NotPresent) => None,
+            Err(std::env::VarError::NotUnicode(var)) => {
+                // FIXME: better diag
+                // FIXME: Use `OsStr::display` (instead of `to_string_lossy`) if/once stabilized (on beta).
+                dcx.emit(
+                    &format!("env var `{KEY}` is not valid UTF-8: `{}`", var.to_string_lossy()),
+                    None,
+                    None,
+                );
+                return Err(());
+            }
+        };
+
+        // unwrap: The static item is locally scoped and no other thread tries to initialize it.
+        CHANNEL_URL.set(channel_url).unwrap();
+        // unwrap: Initialized above.
+        Ok(CHANNEL_URL.get().unwrap().as_deref())
+    }
+}
diff --git a/src/tools/htmldocck/src/config.rs b/src/tools/htmldocck/src/config.rs
diff --git a/src/tools/htmldocck/src/error.rs b/src/tools/htmldocck/src/error.rs
diff --git a/src/tools/htmldocck/src/main.rs b/src/tools/htmldocck/src/main.rs
diff --git a/src/tools/htmldocck/src/parse.rs b/src/tools/htmldocck/src/parse.rs
diff --git a/triagebot.toml b/triagebot.toml
