compiletest: Add support for running rustdoc tests

Add a new test directory called 'rustdoc' where all files inside are documented
and run against the `htmldocck` script to have assertions about the output.

configure

@@ -1108,6 +1108,7 @@ do
     make_dir $h/test/debuginfo-gdb
     make_dir $h/test/debuginfo-lldb
     make_dir $h/test/codegen
+    make_dir $h/test/rustdoc
 done
 
 # Configure submodules

mk/tests.mk

@@ -304,6 +304,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
     check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-cfail-full-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-rmake-exec \
+	check-stage$(1)-T-$(2)-H-$(3)-rustdocck-exec \
         check-stage$(1)-T-$(2)-H-$(3)-crates-exec \
         check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-bench-exec \
@@ -471,6 +472,7 @@ DEBUGINFO_GDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 DEBUGINFO_LLDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 CODEGEN_RS := $(wildcard $(S)src/test/codegen/*.rs)
 CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
+RUSTDOCCK_RS := $(wildcard $(S)src/test/rustdocck/*.rs)
 
 # perf tests are the same as bench tests only they run under
 # a performance monitor.
@@ -489,6 +491,7 @@ PRETTY_TESTS := $(PRETTY_RS)
 DEBUGINFO_GDB_TESTS := $(DEBUGINFO_GDB_RS)
 DEBUGINFO_LLDB_TESTS := $(DEBUGINFO_LLDB_RS)
 CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
+RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
 
 CTEST_SRC_BASE_rpass = run-pass
 CTEST_BUILD_BASE_rpass = run-pass
@@ -550,6 +553,11 @@ CTEST_BUILD_BASE_codegen = codegen
 CTEST_MODE_codegen = codegen
 CTEST_RUNTOOL_codegen = $(CTEST_RUNTOOL)
 
+CTEST_SRC_BASE_rustdocck = rustdoc
+CTEST_BUILD_BASE_rustdocck = rustdoc
+CTEST_MODE_rustdocck = rustdoc
+CTEST_RUNTOOL_rustdocck = $(CTEST_RUNTOOL)
+
 # CTEST_DISABLE_$(TEST_GROUP), if set, will cause the test group to be
 # disabled and the associated message to be printed as a warning
 # during attempts to run those tests.
@@ -618,12 +626,14 @@ CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) := \
 		--compile-lib-path $$(HLIB$(1)_H_$(3)) \
         --run-lib-path $$(TLIB$(1)_T_$(2)_H_$(3)) \
         --rustc-path $$(HBIN$(1)_H_$(3))/rustc$$(X_$(3)) \
+        --rustdoc-path $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
         --clang-path $(if $(CFG_CLANG),$(CFG_CLANG),clang) \
         --llvm-bin-path $(CFG_LLVM_INST_DIR_$(CFG_BUILD))/bin \
         --aux-base $$(S)src/test/auxiliary/ \
         --stage-id stage$(1)-$(2) \
         --target $(2) \
         --host $(3) \
+	--python $$(CFG_PYTHON) \
         --gdb-version="$(CFG_GDB_VERSION)" \
         --lldb-version="$(CFG_LLDB_VERSION)" \
         --android-cross-path=$(CFG_ANDROID_CROSS_PATH) \
@@ -660,6 +670,9 @@ CTEST_DEPS_debuginfo-lldb_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_LLDB_TESTS) \
                                                $(S)src/etc/lldb_batchmode.py \
                                                $(S)src/etc/lldb_rust_formatters.py
 CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
+CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
+        $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
+	$(S)src/etc/htmldocck.py
 
 endef
 
@@ -722,7 +735,8 @@ endif
 
 endef
 
-CTEST_NAMES = rpass rpass-valgrind rpass-full cfail-full rfail cfail pfail bench perf debuginfo-gdb debuginfo-lldb codegen
+CTEST_NAMES = rpass rpass-valgrind rpass-full cfail-full rfail cfail pfail \
+	bench perf debuginfo-gdb debuginfo-lldb codegen rustdocck
 
 $(foreach host,$(CFG_HOST), \
  $(eval $(foreach target,$(CFG_TARGET), \
@@ -890,6 +904,7 @@ TEST_GROUPS = \
 	bench \
 	perf \
 	rmake \
+	rustdocck \
 	debuginfo-gdb \
 	debuginfo-lldb \
 	codegen \

src/compiletest/common.rs

@@ -23,7 +23,8 @@ pub enum Mode {
     Pretty,
     DebugInfoGdb,
     DebugInfoLldb,
-    Codegen
+    Codegen,
+    Rustdoc,
 }
 
 impl FromStr for Mode {
@@ -39,6 +40,7 @@ fn from_str(s: &str) -> Result<Mode, ()> {
           "debuginfo-lldb" => Ok(DebugInfoLldb),
           "debuginfo-gdb" => Ok(DebugInfoGdb),
           "codegen" => Ok(Codegen),
+          "rustdoc" => Ok(Rustdoc),
           _ => Err(()),
         }
     }
@@ -56,6 +58,7 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
             DebugInfoGdb => "debuginfo-gdb",
             DebugInfoLldb => "debuginfo-lldb",
             Codegen => "codegen",
+            Rustdoc => "rustdoc",
         }, f)
     }
 }
@@ -71,6 +74,12 @@ pub struct Config {
     // The rustc executable
     pub rustc_path: PathBuf,
 
+    // The rustdoc executable
+    pub rustdoc_path: PathBuf,
+
+    // The python executable
+    pub python: String,
+
     // The clang executable
     pub clang_path: Option<PathBuf>,
 

src/compiletest/compiletest.rs

@@ -60,6 +60,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
         vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
           reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
           reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
+          reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
+          reqopt("", "python", "path to python to use for doc tests", "PATH"),
           optopt("", "clang-path", "path to  executable for codegen tests", "PATH"),
           optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
           optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
@@ -128,6 +130,8 @@ fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
         compile_lib_path: matches.opt_str("compile-lib-path").unwrap(),
         run_lib_path: matches.opt_str("run-lib-path").unwrap(),
         rustc_path: opt_path(matches, "rustc-path"),
+        rustdoc_path: opt_path(matches, "rustdoc-path"),
+        python: matches.opt_str("python").unwrap(),
         clang_path: matches.opt_str("clang-path").map(|s| PathBuf::from(&s)),
         valgrind_path: matches.opt_str("valgrind-path"),
         force_valgrind: matches.opt_present("force-valgrind"),
@@ -168,6 +172,7 @@ pub fn log_config(config: &Config) {
     logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
     logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
     logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
+    logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
     logv(c, format!("src_base: {:?}", config.src_base.display()));
     logv(c, format!("build_base: {:?}", config.build_base.display()));
     logv(c, format!("stage_id: {}", config.stage_id));

src/compiletest/runtest.rs

@@ -12,7 +12,7 @@
 
 use common::Config;
 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
-use common::{Codegen, DebugInfoLldb, DebugInfoGdb};
+use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc};
 use errors;
 use header::TestProps;
 use header;
@@ -57,15 +57,16 @@ pub fn run_metrics(config: Config, testfile: &Path, mm: &mut MetricMap) {
     let props = header::load_props(&testfile);
     debug!("loaded props");
     match config.mode {
-      CompileFail => run_cfail_test(&config, &props, &testfile),
-      ParseFail => run_cfail_test(&config, &props, &testfile),
-      RunFail => run_rfail_test(&config, &props, &testfile),
-      RunPass => run_rpass_test(&config, &props, &testfile),
-      RunPassValgrind => run_valgrind_test(&config, &props, &testfile),
-      Pretty => run_pretty_test(&config, &props, &testfile),
-      DebugInfoGdb => run_debuginfo_gdb_test(&config, &props, &testfile),
-      DebugInfoLldb => run_debuginfo_lldb_test(&config, &props, &testfile),
-      Codegen => run_codegen_test(&config, &props, &testfile, mm),
+        CompileFail => run_cfail_test(&config, &props, &testfile),
+        ParseFail => run_cfail_test(&config, &props, &testfile),
+        RunFail => run_rfail_test(&config, &props, &testfile),
+        RunPass => run_rpass_test(&config, &props, &testfile),
+        RunPassValgrind => run_valgrind_test(&config, &props, &testfile),
+        Pretty => run_pretty_test(&config, &props, &testfile),
+        DebugInfoGdb => run_debuginfo_gdb_test(&config, &props, &testfile),
+        DebugInfoLldb => run_debuginfo_lldb_test(&config, &props, &testfile),
+        Codegen => run_codegen_test(&config, &props, &testfile, mm),
+        Rustdoc => run_rustdoc_test(&config, &props, &testfile),
     }
 }
 
@@ -725,32 +726,37 @@ fn run_lldb(config: &Config,
                 -> ProcRes {
         // Prepare the lldb_batchmode which executes the debugger script
         let lldb_script_path = rust_src_root.join("src/etc/lldb_batchmode.py");
+        cmd2proces(config,
+                   test_executable,
+                   Command::new(&config.python)
+                           .arg(&lldb_script_path)
+                           .arg(test_executable)
+                           .arg(debugger_script)
+                           .env("PYTHONPATH",
+                                config.lldb_python_dir.as_ref().unwrap()))
+    }
+}
 
-        let mut cmd = Command::new("python");
-        cmd.arg(&lldb_script_path)
-           .arg(test_executable)
-           .arg(debugger_script)
-           .env("PYTHONPATH", config.lldb_python_dir.as_ref().unwrap());
-
-        let (status, out, err) = match cmd.output() {
-            Ok(Output { status, stdout, stderr }) => {
-                (status,
-                 String::from_utf8(stdout).unwrap(),
-                 String::from_utf8(stderr).unwrap())
-            },
-            Err(e) => {
-                fatal(&format!("Failed to setup Python process for \
-                                LLDB script: {}", e))
-            }
-        };
+fn cmd2proces(config: &Config, test_executable: &Path, cmd: &mut Command)
+              -> ProcRes {
+    let (status, out, err) = match cmd.output() {
+        Ok(Output { status, stdout, stderr }) => {
+            (status,
+             String::from_utf8(stdout).unwrap(),
+             String::from_utf8(stderr).unwrap())
+        },
+        Err(e) => {
+            fatal(&format!("Failed to setup Python process for \
+                            LLDB script: {}", e))
+        }
+    };
 
-        dump_output(config, test_executable, &out, &err);
-        return ProcRes {
-            status: Status::Normal(status),
-            stdout: out,
-            stderr: err,
-            cmdline: format!("{:?}", cmd)
-        };
+    dump_output(config, test_executable, &out, &err);
+    ProcRes {
+        status: Status::Normal(status),
+        stdout: out,
+        stderr: err,
+        cmdline: format!("{:?}", cmd)
     }
 }
 
@@ -1157,6 +1163,24 @@ fn compile_test_(config: &Config, props: &TestProps,
     compose_and_run_compiler(config, props, testfile, args, None)
 }
 
+fn document(config: &Config, props: &TestProps,
+            testfile: &Path, extra_args: &[String]) -> (ProcRes, PathBuf) {
+    let aux_dir = aux_output_dir_name(config, testfile);
+    let out_dir = output_base_name(config, testfile);
+    ensure_dir(&out_dir);
+    let mut args = vec!["-L".to_string(),
+                        aux_dir.to_str().unwrap().to_string(),
+                        "-o".to_string(),
+                        out_dir.to_str().unwrap().to_string(),
+                        testfile.to_str().unwrap().to_string()];
+    args.extend(extra_args.iter().cloned());
+    let args = ProcArgs {
+        prog: config.rustdoc_path.to_str().unwrap().to_string(),
+        args: args,
+    };
+    (compose_and_run_compiler(config, props, testfile, args, None), out_dir)
+}
+
 fn exec_compiled_test(config: &Config, props: &TestProps,
                       testfile: &Path) -> ProcRes {
 
@@ -1181,20 +1205,17 @@ fn exec_compiled_test(config: &Config, props: &TestProps,
     }
 }
 
-fn compose_and_run_compiler(
-    config: &Config,
-    props: &TestProps,
-    testfile: &Path,
-    args: ProcArgs,
-    input: Option<String>) -> ProcRes {
-
+fn compose_and_run_compiler(config: &Config, props: &TestProps,
+                            testfile: &Path, args: ProcArgs,
+                            input: Option<String>) -> ProcRes {
     if !props.aux_builds.is_empty() {
         ensure_dir(&aux_output_dir_name(config, testfile));
     }
 
     let aux_dir = aux_output_dir_name(config, testfile);
     // FIXME (#9639): This needs to handle non-utf8 paths
-    let extra_link_args = vec!("-L".to_string(), aux_dir.to_str().unwrap().to_string());
+    let extra_link_args = vec!["-L".to_string(),
+                               aux_dir.to_str().unwrap().to_string()];
 
     for rel_ab in &props.aux_builds {
         let abs_ab = config.aux_base.join(rel_ab);
@@ -1330,8 +1351,8 @@ fn make_exe_name(config: &Config, testfile: &Path) -> PathBuf {
     f
 }
 
-fn make_run_args(config: &Config, props: &TestProps, testfile: &Path) ->
-   ProcArgs {
+fn make_run_args(config: &Config, props: &TestProps, testfile: &Path)
+                 -> ProcArgs {
     // If we've got another tool to run under (valgrind),
     // then split apart its command
     let mut args = split_maybe_args(&config.runtool);
@@ -1797,3 +1818,21 @@ fn charset() -> &'static str {
         "UTF-8"
     }
 }
+
+fn run_rustdoc_test(config: &Config, props: &TestProps, testfile: &Path) {
+    let (proc_res, out_dir) = document(config, props, testfile, &[]);
+    if !proc_res.status.success() {
+        fatal_proc_rec("rustdoc failed!", &proc_res);
+    }
+    let root = find_rust_src_root(config).unwrap();
+
+    let res = cmd2proces(config,
+                         testfile,
+                         Command::new(&config.python)
+                                 .arg(root.join("src/etc/htmldocck.py"))
+                                 .arg(out_dir)
+                                 .arg(testfile));
+    if !res.status.success() {
+        fatal_proc_rec("htmldocck failed!", &res);
+    }
+}