Rollup merge of #47250 - GuillaumeGomez:test-rustdoc-js, r=Mark-Simulacrum

Test rustdoc js

Add tests for the rustdoc search. It was heavily required because of all the recent breaking changes that happened while I went through improvements in doc search (add search in/for generic search for example).

src/bootstrap/builder.rs

@@ -254,7 +254,9 @@ fn get_step_descriptions(kind: Kind) -> Vec<StepDescription> {
             Kind::Test => describe!(check::Tidy, check::Bootstrap, check::DefaultCompiletest,
                 check::HostCompiletest, check::Crate, check::CrateLibrustc, check::Rustdoc,
                 check::Linkcheck, check::Cargotest, check::Cargo, check::Rls, check::Docs,
-                check::ErrorIndex, check::Distcheck, check::Rustfmt, check::Miri, check::Clippy),
+                check::ErrorIndex, check::Distcheck, check::Rustfmt, check::Miri, check::Clippy,
+                check::RustdocJS),
+
             Kind::Bench => describe!(check::Crate, check::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
                 doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,
@@ -443,7 +445,8 @@ pub fn cargo(&self,
         let out_dir = self.stage_out(compiler, mode);
         cargo.env("CARGO_TARGET_DIR", out_dir)
              .arg(cmd)
-             .arg("--target").arg(target);
+             .arg("--target")
+             .arg(target);
 
         // If we were invoked from `make` then that's already got a jobserver
         // set up for us so no need to tell Cargo about jobs all over again.

src/bootstrap/check.rs

@@ -424,6 +424,43 @@ fn path_for_cargo(builder: &Builder, compiler: Compiler) -> OsString {
     env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocJS {
+    pub host: Interned<String>,
+    pub target: Interned<String>,
+}
+
+impl Step for RustdocJS {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/test/rustdoc-js")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(RustdocJS {
+            host: run.host,
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) {
+        if let Some(ref nodejs) = builder.config.nodejs {
+            let mut command = Command::new(nodejs);
+            command.args(&["src/tools/rustdoc-js/tester.js", &*self.host]);
+            builder.ensure(::doc::Std {
+                target: self.target,
+                stage: builder.top_stage,
+            });
+            builder.run(&mut command);
+        } else {
+            println!("No nodejs found, skipping \"src/test/rustdoc-js\" tests");
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Tidy {
     host: Interned<String>,

src/bootstrap/doc.rs

@@ -419,8 +419,8 @@ fn run(self, builder: &Builder) {
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Std {
-    stage: u32,
-    target: Interned<String>,
+    pub stage: u32,
+    pub target: Interned<String>,
 }
 
 impl Step for Std {

src/librustdoc/html/static/main.js

@@ -353,35 +353,33 @@
      * This code is an unmodified version of the code written by Marco de Wit
      * and was found at http://stackoverflow.com/a/18514751/745719
      */
-    var levenshtein = (function() {
-        var row2 = [];
-        return function(s1, s2) {
-            if (s1 === s2) {
-                return 0;
+    var levenshtein_row2 = [];
+    function levenshtein(s1, s2) {
+        if (s1 === s2) {
+            return 0;
+        }
+        var s1_len = s1.length, s2_len = s2.length;
+        if (s1_len && s2_len) {
+            var i1 = 0, i2 = 0, a, b, c, c2, row = levenshtein_row2;
+            while (i1 < s1_len) {
+                row[i1] = ++i1;
             }
-            var s1_len = s1.length, s2_len = s2.length;
-            if (s1_len && s2_len) {
-                var i1 = 0, i2 = 0, a, b, c, c2, row = row2;
-                while (i1 < s1_len) {
-                    row[i1] = ++i1;
-                }
-                while (i2 < s2_len) {
-                    c2 = s2.charCodeAt(i2);
-                    a = i2;
-                    ++i2;
-                    b = i2;
-                    for (i1 = 0; i1 < s1_len; ++i1) {
-                        c = a + (s1.charCodeAt(i1) !== c2 ? 1 : 0);
-                        a = row[i1];
-                        b = b < a ? (b < c ? b + 1 : c) : (a < c ? a + 1 : c);
-                        row[i1] = b;
-                    }
+            while (i2 < s2_len) {
+                c2 = s2.charCodeAt(i2);
+                a = i2;
+                ++i2;
+                b = i2;
+                for (i1 = 0; i1 < s1_len; ++i1) {
+                    c = a + (s1.charCodeAt(i1) !== c2 ? 1 : 0);
+                    a = row[i1];
+                    b = b < a ? (b < c ? b + 1 : c) : (a < c ? a + 1 : c);
+                    row[i1] = b;
                 }
-                return b;
             }
-            return s1_len + s2_len;
-        };
-    })();
+            return b;
+        }
+        return s1_len + s2_len;
+    }
 
     function initSearch(rawSearchIndex) {
         var currentResults, index, searchIndex;
@@ -400,12 +398,20 @@
         /**
          * Executes the query and builds an index of results
          * @param  {[Object]} query     [The user query]
-         * @param  {[type]} max         [The maximum results returned]
          * @param  {[type]} searchWords [The list of search words to query
          *                               against]
          * @return {[type]}             [A search index of results]
          */
-        function execQuery(query, max, searchWords) {
+        function execQuery(query, searchWords) {
+            function itemTypeFromName(typename) {
+                for (var i = 0; i < itemTypes.length; ++i) {
+                    if (itemTypes[i] === typename) {
+                        return i;
+                    }
+                }
+                return -1;
+            }
+
             var valLower = query.query.toLowerCase(),
                 val = valLower,
                 typeFilter = itemTypeFromName(query.type),
@@ -1021,9 +1027,8 @@
             return true;
         }
 
-        function getQuery() {
-            var matches, type, query, raw =
-                document.getElementsByClassName('search-input')[0].value;
+        function getQuery(raw) {
+            var matches, type, query;
             query = raw;
 
             matches = query.match(/^(fn|mod|struct|enum|trait|type|const|macro)\s*:\s*/i);
@@ -1227,7 +1232,7 @@
         }
 
         function showResults(results) {
-            var output, query = getQuery();
+            var output, query = getQuery(document.getElementsByClassName('search-input')[0].value);
 
             currentResults = query.id;
             output = '<h1>Results for ' + escape(query.query) +
@@ -1271,7 +1276,7 @@
                 resultIndex;
             var params = getQueryStringParams();
 
-            query = getQuery();
+            query = getQuery(document.getElementsByClassName('search-input')[0].value);
             if (e) {
                 e.preventDefault();
             }
@@ -1293,19 +1298,10 @@
                 }
             }
 
-            results = execQuery(query, 20000, index);
+            results = execQuery(query, index);
             showResults(results);
         }
 
-        function itemTypeFromName(typename) {
-            for (var i = 0; i < itemTypes.length; ++i) {
-                if (itemTypes[i] === typename) {
-                    return i;
-                }
-            }
-            return -1;
-        }
-
         function buildIndex(rawSearchIndex) {
             searchIndex = [];
             var searchWords = [];

src/test/rustdoc-js/basic.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'String';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::string', 'name': 'String' },
+        { 'path': 'std::ffi', 'name': 'OsString' },
+        { 'path': 'std::ffi', 'name': 'CString' },
+    ],
+    'in_args': [
+        { 'path': 'std::str', 'name': 'eq' },
+    ],
+    'returned': [
+        { 'path': 'std::string::String', 'name': 'add' },
+    ],
+};

src/test/rustdoc-js/enum-option.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'enum:Option';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::option', 'name': 'Option' },
+    ],
+};

src/test/rustdoc-js/fn-forget.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'fn:forget';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::mem', 'name': 'forget' },
+        { 'path': 'std::fmt', 'name': 'format' },
+    ],
+};

src/test/rustdoc-js/from_u.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'from_u';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::char', 'name': 'from_u32' },
+        { 'path': 'std::str', 'name': 'from_utf8' },
+        { 'path': 'std::string::String', 'name': 'from_utf8' },
+        { 'path': 'std::boxed::Box', 'name': 'from_unique' },
+        { 'path': 'std::i32', 'name': 'from_unsigned' },
+        { 'path': 'std::i128', 'name': 'from_unsigned' },
+    ],
+};

src/test/rustdoc-js/macro-print.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'macro:print';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std', 'name': 'print' },
+        { 'path': 'std', 'name': 'eprint' },
+        { 'path': 'std', 'name': 'println' },
+        { 'path': 'std', 'name': 'eprintln' },
+    ],
+};

src/test/rustdoc-js/string-from_ut.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'String::from_ut';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::string::String', 'name': 'from_utf8' },
+        { 'path': 'std::string::String', 'name': 'from_utf8' },
+        { 'path': 'std::string::String', 'name': 'from_utf8_lossy' },
+        { 'path': 'std::string::String', 'name': 'from_utf16_lossy' },
+        { 'path': 'std::string::String', 'name': 'from_utf8_unchecked' },
+    ],
+};

src/test/rustdoc-js/struct-vec.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const QUERY = 'struct:Vec';
+
+const EXPECTED = {
+    'others': [
+        { 'path': 'std::vec', 'name': 'Vec' },
+        { 'path': 'std::collections', 'name': 'VecDeque' },
+        { 'path': 'alloc::raw_vec', 'name': 'RawVec' },
+    ],
+};

src/tools/rustdoc-js/tester.js

+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+const fs = require('fs');
+
+const TEST_FOLDER = 'src/test/rustdoc-js/';
+
+// Stupid function extractor based on indent.
+function extractFunction(content, functionName) {
+    var x = content.split('\n');
+    var in_func = false;
+    var indent = 0;
+    var lines = [];
+
+    for (var i = 0; i < x.length; ++i) {
+        if (in_func === false) {
+            var splitter = "function " + functionName + "(";
+            if (x[i].trim().startsWith(splitter)) {
+                in_func = true;
+                indent = x[i].split(splitter)[0].length;
+                lines.push(x[i]);
+            }
+        } else {
+            lines.push(x[i]);
+            if (x[i].trim() === "}" && x[i].split("}")[0].length === indent) {
+                return lines.join("\n");
+            }
+        }
+    }
+    return null;
+}
+
+// Stupid function extractor for array.
+function extractArrayVariable(content, arrayName) {
+    var x = content.split('\n');
+    var found_var = false;
+    var lines = [];
+
+    for (var i = 0; i < x.length; ++i) {
+        if (found_var === false) {
+            var splitter = "var " + arrayName + " = [";
+            if (x[i].trim().startsWith(splitter)) {
+                found_var = true;
+                i -= 1;
+            }
+        } else {
+            lines.push(x[i]);
+            if (x[i].endsWith('];')) {
+                return lines.join("\n");
+            }
+        }
+    }
+    return null;
+}
+
+// Stupid function extractor for variable.
+function extractVariable(content, varName) {
+    var x = content.split('\n');
+    var found_var = false;
+    var lines = [];
+
+    for (var i = 0; i < x.length; ++i) {
+        if (found_var === false) {
+            var splitter = "var " + varName + " = ";
+            if (x[i].trim().startsWith(splitter)) {
+                found_var = true;
+                i -= 1;
+            }
+        } else {
+            lines.push(x[i]);
+            if (x[i].endsWith(';')) {
+                return lines.join("\n");
+            }
+        }
+    }
+    return null;
+}
+
+function loadContent(content) {
+    var Module = module.constructor;
+    var m = new Module();
+    m._compile(content, "tmp.js");
+    return m.exports;
+}
+
+function readFile(filePath) {
+    return fs.readFileSync(filePath, 'utf8');
+}
+
+function loadThings(thingsToLoad, kindOfLoad, funcToCall, fileContent) {
+    var content = '';
+    for (var i = 0; i < thingsToLoad.length; ++i) {
+        var tmp = funcToCall(fileContent, thingsToLoad[i]);
+        if (tmp === null) {
+            console.error('enable to find ' + kindOfLoad + ' "' + thingsToLoad[i] + '"');
+            process.exit(1);
+        }
+        content += tmp;
+        content += 'exports.' + thingsToLoad[i] + ' = ' + thingsToLoad[i] + ';';
+    }
+    return content;
+}
+
+function lookForEntry(entry, data) {
+    for (var i = 0; i < data.length; ++i) {
+        var allGood = true;
+        for (var key in entry) {
+            if (!entry.hasOwnProperty(key)) {
+                continue;
+            }
+            var value = data[i][key];
+            // To make our life easier, if there is a "parent" type, we add it to the path.
+            if (key === 'path' && data[i]['parent'] !== undefined) {
+                if (value.length > 0) {
+                    value += '::' + data[i]['parent']['name'];
+                } else {
+                    value = data[i]['parent']['name'];
+                }
+            }
+            if (value !== entry[key]) {
+                allGood = false;
+                break;
+            }
+        }
+        if (allGood === true) {
+            return true;
+        }
+    }
+    return false;
+}
+
+function main(argv) {
+    if (argv.length !== 3) {
+        console.error("Expected toolchain to check as argument (for example 'x86_64-apple-darwin'");
+        return 1;
+    }
+    var toolchain = argv[2];
+
+    var mainJs = readFile("build/" + toolchain + "/doc/main.js");
+    var searchIndex = readFile("build/" + toolchain + "/doc/search-index.js").split("\n");
+    if (searchIndex[searchIndex.length - 1].length === 0) {
+        searchIndex.pop();
+    }
+    searchIndex.pop();
+    searchIndex = loadContent(searchIndex.join("\n") + '\nexports.searchIndex = searchIndex;');
+    finalJS = "";
+
+    var arraysToLoad = ["itemTypes"];
+    var variablesToLoad = ["MAX_LEV_DISTANCE", "MAX_RESULTS", "TY_PRIMITIVE", "levenshtein_row2"];
+    // execQuery first parameter is built in getQuery (which takes in the search input).
+    // execQuery last parameter is built in buildIndex.
+    // buildIndex requires the hashmap from search-index.
+    var functionsToLoad = ["levenshtein", "validateResult", "getQuery", "buildIndex", "execQuery"];
+
+    finalJS += 'window = { "currentCrate": "std" };\n';
+    finalJS += loadThings(arraysToLoad, 'array', extractArrayVariable, mainJs);
+    finalJS += loadThings(variablesToLoad, 'variable', extractVariable, mainJs);
+    finalJS += loadThings(functionsToLoad, 'function', extractFunction, mainJs);
+
+    var loaded = loadContent(finalJS);
+    var index = loaded.buildIndex(searchIndex.searchIndex);
+
+    var errors = 0;
+
+    fs.readdirSync(TEST_FOLDER).forEach(function(file) {
+        var loadedFile = loadContent(readFile(TEST_FOLDER + file) +
+                               'exports.QUERY = QUERY;exports.EXPECTED = EXPECTED;');
+        const expected = loadedFile.EXPECTED;
+        const query = loadedFile.QUERY;
+        var results = loaded.execQuery(loaded.getQuery(query), index);
+        process.stdout.write('Checking "' + file + '" ... ');
+        var error_text = [];
+        for (var key in expected) {
+            if (!expected.hasOwnProperty(key)) {
+                continue;
+            }
+            if (!results.hasOwnProperty(key)) {
+                error_text.push('==> Unknown key "' + key + '"');
+                break;
+            }
+            var entry = expected[key];
+            var found = false;
+            for (var i = 0; i < entry.length; ++i) {
+                if (lookForEntry(entry[i], results[key]) === true) {
+                    found = true;
+                } else {
+                    error_text.push("==> Result not found in '" + key + "': '" +
+                                    JSON.stringify(entry[i]) + "'");
+                }
+            }
+        }
+        if (error_text.length !== 0) {
+            errors += 1;
+            console.error("FAILED");
+            console.error(error_text.join("\n"));
+        } else {
+            console.log("OK");
+        }
+    });
+    return errors;
+}
+
+process.exit(main(process.argv));