From 4b14573d50ef13e869db6f997873b1051d2a2c34 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 11 May 2018 00:02:05 +0200 Subject: [PATCH 1/2] Add minification process --- src/Cargo.lock | 20 ++++++++++++----- src/librustdoc/Cargo.toml | 1 + src/librustdoc/html/render.rs | 36 +++++++++++++++++++++--------- src/librustdoc/html/static/main.js | 4 ++-- src/librustdoc/lib.rs | 10 ++++++++- 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 21c35458398b7..1746621661c75 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -233,7 +233,7 @@ dependencies = [ "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "shell-escape 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -417,7 +417,7 @@ dependencies = [ "serde 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1169,7 +1169,7 @@ dependencies = [ "serde_derive 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "toml-query 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1187,6 +1187,14 @@ name = "memoffset" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "minifier" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "miniz-sys" version = "0.1.10" @@ -2222,6 +2230,7 @@ dependencies = [ name = "rustdoc" version = "0.0.0" dependencies = [ + "minifier 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2654,7 +2663,7 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3099,6 +3108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum mdbook 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "90b5a8d7e341ceee5db3882a06078d42661ddcfa2b3687319cc5da76ec4e782f" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum minifier 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "26f3e36a4db1981b16567e4abfd6ddc3641bc9b950bdc868701f656bf9b74bdd" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" "checksum miow 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9224c91f82b3c47cf53dcf78dfaa20d6888fbcc5d272d5f2fcdf8a697f3c987d" "checksum nibble_vec 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c8d77f3db4bce033f4d04db08079b2ef1c3d02b44e86f25d08886fafa7756ffa" @@ -3197,7 +3207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum syntex_syntax 0.52.0 (registry+https://github.com/rust-lang/crates.io-index)" = "76a302e717e348aa372ff577791c3832395650073b8d8432f8b3cb170b34afde" "checksum tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)" = "6af6b94659f9a571bf769a5b71f54079393585ee0bfdd71b691be22d7d6b1d18" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -"checksum tempfile 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8cddbd26c5686ece823b507f304c8f188daef548b4cb753512d929ce478a093c" +"checksum tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "47776f63b85777d984a50ce49d6b9e58826b6a3766a449fc95bc66cd5663c15b" "checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" "checksum term 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e6b677dd1e8214ea1ef4297f85dbcbed8e8cdddb561040cc998ca2551c37561" diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 3a376bb9affba..96a2194eeeefd 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -10,3 +10,4 @@ path = "lib.rs" [dependencies] pulldown-cmark = { version = "0.1.2", default-features = false } tempdir = "0.3" +minifier = "0.0.11" diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 21de2db1dfe74..f945534c409d8 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -76,6 +76,8 @@ use html::item_type::ItemType; use html::markdown::{self, Markdown, MarkdownHtml, MarkdownSummaryLine}; use html::{highlight, layout}; +use minifier; + /// A pair of name and its optional document. pub type NameDoc = (String, Option); @@ -509,7 +511,8 @@ pub fn run(mut krate: clean::Crate, css_file_extension: Option, renderinfo: RenderInfo, sort_modules_alphabetically: bool, - themes: Vec) -> Result<(), Error> { + themes: Vec, + enable_minification: bool) -> Result<(), Error> { let src_root = match krate.src { FileName::Real(ref p) => match p.parent() { Some(p) => p.to_path_buf(), @@ -661,7 +664,7 @@ pub fn run(mut krate: clean::Crate, CACHE_KEY.with(|v| *v.borrow_mut() = cache.clone()); CURRENT_LOCATION_KEY.with(|s| s.borrow_mut().clear()); - write_shared(&cx, &krate, &*cache, index)?; + write_shared(&cx, &krate, &*cache, index, enable_minification)?; // And finally render the whole crate's documentation cx.krate(krate) @@ -740,7 +743,8 @@ fn build_index(krate: &clean::Crate, cache: &mut Cache) -> String { fn write_shared(cx: &Context, krate: &clean::Crate, cache: &Cache, - search_index: String) -> Result<(), Error> { + search_index: String, + enable_minification: bool) -> Result<(), Error> { // Write out the shared files. Note that these are shared among all rustdoc // docs placed in the output directory, so this needs to be a synchronized // operation with respect to all other rustdocs running around. @@ -814,16 +818,20 @@ themePicker.onclick = function() {{ .join(",")).as_bytes(), )?; - write(cx.dst.join(&format!("main{}.js", cx.shared.resource_suffix)), - include_bytes!("static/main.js"))?; - write(cx.dst.join(&format!("settings{}.js", cx.shared.resource_suffix)), - include_bytes!("static/settings.js"))?; + write_minify(cx.dst.join(&format!("main{}.js", cx.shared.resource_suffix)), + include_str!("static/main.js"), + enable_minification)?; + write_minify(cx.dst.join(&format!("settings{}.js", cx.shared.resource_suffix)), + include_str!("static/settings.js"), + enable_minification)?; { let mut data = format!("var resourcesSuffix = \"{}\";\n", - cx.shared.resource_suffix).into_bytes(); - data.extend_from_slice(include_bytes!("static/storage.js")); - write(cx.dst.join(&format!("storage{}.js", cx.shared.resource_suffix)), &data)?; + cx.shared.resource_suffix); + data.push_str(include_str!("static/storage.js")); + write_minify(cx.dst.join(&format!("storage{}.js", cx.shared.resource_suffix)), + &data, + enable_minification)?; } if let Some(ref css) = cx.shared.css_file_extension { @@ -1020,6 +1028,14 @@ fn write(dst: PathBuf, contents: &[u8]) -> Result<(), Error> { Ok(try_err!(fs::write(&dst, contents), &dst)) } +fn write_minify(dst: PathBuf, contents: &str, enable_minification: bool) -> Result<(), Error> { + if enable_minification { + write(dst, minifier::js::minify(contents).as_bytes()) + } else { + write(dst, contents.as_bytes()) + } +} + /// Takes a path to a source file and cleans the path to it. This canonicalizes /// things like ".." to components which preserve the "top down" hierarchy of a /// static HTML tree. Each component in the cleaned path will be passed as an diff --git a/src/librustdoc/html/static/main.js b/src/librustdoc/html/static/main.js index 8569abeb09cc1..2e7a1c68579a9 100644 --- a/src/librustdoc/html/static/main.js +++ b/src/librustdoc/html/static/main.js @@ -196,7 +196,7 @@ onEach(e.getElementsByTagName('span'), function(i_e) { removeClass(i_e, 'line-highlighted'); }); - }) + }); for (i = from; i <= to; ++i) { addClass(document.getElementById(i), 'line-highlighted'); } @@ -1944,7 +1944,7 @@ hasClass(next.nextElementSibling, 'docblock')))) { insertAfter(toggle.cloneNode(true), e.childNodes[e.childNodes.length - 1]); } - } + }; onEach(document.getElementsByClassName('method'), func); onEach(document.getElementsByClassName('impl'), func); onEach(document.getElementsByClassName('impl-items'), function(e) { diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 059d41698953d..e79f67db130db 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -46,6 +46,7 @@ extern crate test as testing; extern crate rustc_errors as errors; extern crate pulldown_cmark; extern crate tempdir; +extern crate minifier; extern crate serialize as rustc_serialize; // used by deriving @@ -297,6 +298,11 @@ pub fn opts() -> Vec { "How errors and other messages are produced", "human|json|short") }), + unstable("disable-minification", |o| { + o.optflag("", + "disable-minification", + "Disable minification applied on JS files") + }), ] } @@ -478,6 +484,7 @@ pub fn main_args(args: &[String]) -> isize { let linker = matches.opt_str("linker").map(PathBuf::from); let sort_modules_alphabetically = !matches.opt_present("sort-modules-by-appearance"); let resource_suffix = matches.opt_str("resource-suffix"); + let enable_minification = !matches.opt_present("disable-minification"); let edition = matches.opt_str("edition").unwrap_or("2015".to_string()); let edition = match edition.parse() { @@ -521,7 +528,8 @@ pub fn main_args(args: &[String]) -> isize { css_file_extension, renderinfo, sort_modules_alphabetically, - themes) + themes, + enable_minification) .expect("failed to generate documentation"); 0 } From e2db0a5630a39fc1725cae1e804120cff75dc1cb Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 12 May 2018 00:45:41 +0200 Subject: [PATCH 2/2] Update js tester tool --- src/tools/rustdoc-js/tester.js | 169 ++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 47 deletions(-) diff --git a/src/tools/rustdoc-js/tester.js b/src/tools/rustdoc-js/tester.js index 25f7a2d1294c5..1fc0d649f4f48 100644 --- a/src/tools/rustdoc-js/tester.js +++ b/src/tools/rustdoc-js/tester.js @@ -12,73 +12,148 @@ const fs = require('fs'); const TEST_FOLDER = 'src/test/rustdoc-js/'; +function getNextStep(content, pos, stop) { + while (pos < content.length && content[pos] !== stop && + (content[pos] === ' ' || content[pos] === '\t' || content[pos] === '\n')) { + pos += 1; + } + if (pos >= content.length) { + return null; + } + if (content[pos] !== stop) { + return pos * -1; + } + return pos; +} + // Stupid function extractor based on indent. function extractFunction(content, functionName) { - var x = content.split('\n'); - var in_func = false; var indent = 0; - var lines = []; - - for (var i = 0; i < x.length; ++i) { - if (in_func === false) { - var splitter = "function " + functionName + "("; - if (x[i].trim().startsWith(splitter)) { - in_func = true; - indent = x[i].split(splitter)[0].length; - lines.push(x[i]); - } - } else { - lines.push(x[i]); - if (x[i].trim() === "}" && x[i].split("}")[0].length === indent) { - return lines.join("\n"); + var splitter = "function " + functionName + "("; + + while (true) { + var start = content.indexOf(splitter); + if (start === -1) { + break; + } + var pos = start; + while (pos < content.length && content[pos] !== ')') { + pos += 1; + } + if (pos >= content.length) { + break; + } + pos = getNextStep(content, pos + 1, '{'); + if (pos === null) { + break; + } else if (pos < 0) { + content = content.slice(-pos); + continue; + } + while (pos < content.length) { + if (content[pos] === '"' || content[pos] === "'") { + var stop = content[pos]; + var is_escaped = false; + do { + if (content[pos] === '\\') { + pos += 2; + } else { + pos += 1; + } + } while (pos < content.length && + (content[pos] !== stop || content[pos - 1] === '\\')); + } else if (content[pos] === '{') { + indent += 1; + } else if (content[pos] === '}') { + indent -= 1; + if (indent === 0) { + return content.slice(start, pos + 1); + } } + pos += 1; } + content = content.slice(start + 1); } return null; } // Stupid function extractor for array. function extractArrayVariable(content, arrayName) { - var x = content.split('\n'); - var found_var = false; - var lines = []; - - for (var i = 0; i < x.length; ++i) { - if (found_var === false) { - var splitter = "var " + arrayName + " = ["; - if (x[i].trim().startsWith(splitter)) { - found_var = true; - i -= 1; - } - } else { - lines.push(x[i]); - if (x[i].endsWith('];')) { - return lines.join("\n"); + var splitter = "var " + arrayName; + while (true) { + var start = content.indexOf(splitter); + if (start === -1) { + break; + } + var pos = getNextStep(content, start, '='); + if (pos === null) { + break; + } else if (pos < 0) { + content = content.slice(-pos); + continue; + } + pos = getNextStep(content, pos, '['); + if (pos === null) { + break; + } else if (pos < 0) { + content = content.slice(-pos); + continue; + } + while (pos < content.length) { + if (content[pos] === '"' || content[pos] === "'") { + var stop = content[pos]; + do { + if (content[pos] === '\\') { + pos += 2; + } else { + pos += 1; + } + } while (pos < content.length && + (content[pos] !== stop || content[pos - 1] === '\\')); + } else if (content[pos] === ']' && + pos + 1 < content.length && + content[pos + 1] === ';') { + return content.slice(start, pos + 2); } + pos += 1; } + content = content.slice(start + 1); } return null; } // Stupid function extractor for variable. function extractVariable(content, varName) { - var x = content.split('\n'); - var found_var = false; - var lines = []; - - for (var i = 0; i < x.length; ++i) { - if (found_var === false) { - var splitter = "var " + varName + " = "; - if (x[i].trim().startsWith(splitter)) { - found_var = true; - i -= 1; - } - } else { - lines.push(x[i]); - if (x[i].endsWith(';')) { - return lines.join("\n"); + var splitter = "var " + varName; + while (true) { + var start = content.indexOf(splitter); + if (start === -1) { + break; + } + var pos = getNextStep(content, start, '='); + if (pos === null) { + break; + } else if (pos < 0) { + content = content.slice(-pos); + continue; + } + while (pos < content.length) { + if (content[pos] === '"' || content[pos] === "'") { + var stop = content[pos]; + do { + if (content[pos] === '\\') { + pos += 2; + } else { + pos += 1; + } + } while (pos < content.length && + (content[pos] !== stop || content[pos - 1] === '\\')); + } else if (content[pos] === ';') { + return content.slice(start, pos + 1); } + pos += 1; } + content = content.slice(start + 1); } return null; } @@ -101,7 +176,7 @@ function loadThings(thingsToLoad, kindOfLoad, funcToCall, fileContent) { for (var i = 0; i < thingsToLoad.length; ++i) { var tmp = funcToCall(fileContent, thingsToLoad[i]); if (tmp === null) { - console.error('enable to find ' + kindOfLoad + ' "' + thingsToLoad[i] + '"'); + console.error('unable to find ' + kindOfLoad + ' "' + thingsToLoad[i] + '"'); process.exit(1); } content += tmp;