rustdoc: run on plain Markdown files.

This theoretically gives rustdoc the ability to render our guides, tutorial and manual (not in practice, since the files themselves need to be adjusted slightly to use Sundown-compatible functionality). Fixes #11392.

rustdoc: run on plain Markdown files.
This theoretically gives rustdoc the ability to render our guides, tutorial and manual (not in practice, since the files themselves need to be adjusted slightly to use Sundown-compatible functionality). Fixes #11392.
69b8ef80 · Huon Wilson · e959c879 · 69b8ef80 · 69b8ef80 · 69b8ef80
5 changed file
--- a/src/doc/rustdoc.md
+++ b/src/doc/rustdoc.md
@@ -181,3 +181,28 @@ rustdoc will implicitly add `extern crate <crate>;` where `<crate>` is the name
 the crate being tested to the top of each code example. This means that rustdoc
 must be able to find a compiled version of the library crate being tested. Extra
 search paths may be added via the `-L` flag to `rustdoc`.
+
+# Standalone Markdown files
+
+As well as Rust crates, rustdoc supports rendering pure Markdown files
+into HTML and testing the code snippets from them. A Markdown file is
+detected by a `.md` or `.markdown` extension.
+
+There are 4 options to modify the output that Rustdoc creates.
+- `--markdown-css PATH`: adds a `<link rel="stylesheet">` tag pointing to `PATH`.
+- `--markdown-in-header FILE`: includes the contents of `FILE` at the
+  end of the `<head>...</head>` section.
+- `--markdown-before-content FILE`: includes the contents of `FILE`
+  directly after `<body>`, before the rendered content (including the
+  title).
+- `--markdown-after-content FILE`: includes the contents of `FILE`
+  directly before `</body>`, after all the rendered content.
+
+All of these can be specified multiple times, and they are output in
+the order in which they are specified. The first line of the file must
+be the title, prefixed with `%` (e.g. this page has `% Rust
+Documentation` on the first line).
+
+Like with a Rust crate, the `--test` argument will run the code
+examples to check they compile, and obeys any `--test-args` flags. The
+tests are named after the last `#` heading.
--- a/src/librustdoc/html/markdown.rs
+++ b/src/librustdoc/html/markdown.rs
@@ -28,7 +28,6 @@

 use std::cast;
 use std::fmt;
-use std::intrinsics;
 use std::io;
 use std::libc;
 use std::local_data;
@@ -258,7 +257,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
            };
            if ignore { return }
            vec::raw::buf_as_slice((*text).data, (*text).size as uint, |text| {
-                let tests: &mut ::test::Collector = intrinsics::transmute(opaque);
+                let tests = &mut *(opaque as *mut ::test::Collector);
                let text = str::from_utf8(text).unwrap();
                let mut lines = text.lines().map(|l| stripped_filtered_line(l).unwrap_or(l));
                let text = lines.to_owned_vec().connect("\n");
@@ -266,6 +265,19 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
            })
        }
    }
+    extern fn header(_ob: *buf, text: *buf, level: libc::c_int, opaque: *libc::c_void) {
+        unsafe {
+            let tests = &mut *(opaque as *mut ::test::Collector);
+            if text.is_null() {
+                tests.register_header("", level as u32);
+            } else {
+                vec::raw::buf_as_slice((*text).data, (*text).size as uint, |text| {
+                    let text = str::from_utf8(text).unwrap();
+                    tests.register_header(text, level as u32);
+                })
+            }
+        }
+    }

    unsafe {
        let ob = bufnew(OUTPUT_UNIT);
@@ -276,7 +288,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
            blockcode: Some(block),
            blockquote: None,
            blockhtml: None,
-            header: None,
+            header: Some(header),
            other: mem::init()
        };


--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -14,7 +14,7 @@
 #[crate_type = "dylib"];
 #[crate_type = "rlib"];

-#[feature(globs, struct_variant, managed_boxes)];
+#[feature(globs, struct_variant, managed_boxes, macro_rules)];

 extern crate syntax;
 extern crate rustc;
@@ -26,6 +26,7 @@
 extern crate testing = "test";
 extern crate time;

+use std::cell::RefCell;
 use std::local_data;
 use std::io;
 use std::io::{File, MemWriter};
@@ -44,6 +45,7 @@ pub mod html {
    pub mod markdown;
    pub mod render;
 }
+pub mod markdown;
 pub mod passes;
 pub mod plugins;
 pub mod visit_ast;
@@ -105,6 +107,19 @@ pub fn opts() -> ~[getopts::OptGroup] {
        optflag("", "test", "run code examples as tests"),
        optmulti("", "test-args", "arguments to pass to the test runner",
                 "ARGS"),
+        optmulti("", "markdown-css", "CSS files to include via <link> in a rendered Markdown file",
+                 "FILES"),
+        optmulti("", "markdown-in-header",
+                 "files to include inline in the <head> section of a rendered Markdown file",
+                 "FILES"),
+        optmulti("", "markdown-before-content",
+                 "files to include inline between <body> and the content of a rendered \
+                 Markdown file",
+                 "FILES"),
+        optmulti("", "markdown-after-content",
+                 "files to include inline between the content and </body> of a rendered \
+                 Markdown file",
+                 "FILES"),
    ]
 }

@@ -137,8 +152,24 @@ pub fn main_args(args: &[~str]) -> int {
    }
    let input = matches.free[0].as_slice();

-    if matches.opt_present("test") {
-        return test::run(input, &matches);
+    let libs = matches.opt_strs("L").map(|s| Path::new(s.as_slice()));
+    let libs = @RefCell::new(libs.move_iter().collect());
+
+    let test_args = matches.opt_strs("test-args");
+    let test_args = test_args.iter().flat_map(|s| s.words()).map(|s| s.to_owned()).to_owned_vec();
+
+    let should_test = matches.opt_present("test");
+    let markdown_input = input.ends_with(".md") || input.ends_with(".markdown");
+
+    let output = matches.opt_str("o").map(|s| Path::new(s));
+
+    match (should_test, markdown_input) {
+        (true, true) => return markdown::test(input, libs, test_args),
+        (true, false) => return test::run(input, libs, test_args),
+
+        (false, true) => return markdown::render(input, output.unwrap_or(Path::new("doc")),
+                                                 &matches),
+        (false, false) => {}
    }

    if matches.opt_strs("passes") == ~[~"list"] {
@@ -163,7 +194,6 @@ pub fn main_args(args: &[~str]) -> int {

    info!("going to format");
    let started = time::precise_time_ns();
-    let output = matches.opt_str("o").map(|s| Path::new(s));
    match matches.opt_str("w") {
        Some(~"html") | None => {
            match html::render::run(krate, output.unwrap_or(Path::new("doc"))) {

--- a/src/librustdoc/markdown.rs
+++ b/src/librustdoc/markdown.rs
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::{str, io};
+use std::cell::RefCell;
+use std::vec_ng::Vec;
+
+use collections::HashSet;
+
+use getopts;
+use testing;
+
+use html::escape::Escape;
+use html::markdown::{Markdown, find_testable_code, reset_headers};
+use test::Collector;
+
+fn load_string(input: &Path) -> io::IoResult<Option<~str>> {
+    let mut f = try!(io::File::open(input));
+    let d = try!(f.read_to_end());
+    Ok(str::from_utf8_owned(d))
+}
+macro_rules! load_or_return {
+    ($input: expr, $cant_read: expr, $not_utf8: expr) => {
+        {
+            let input = Path::new($input);
+            match load_string(&input) {
+                Err(e) => {
+                    let _ = writeln!(&mut io::stderr(),
+                                     "error reading `{}`: {}", input.display(), e);
+                    return $cant_read;
+                }
+                Ok(None) => {
+                    let _ = writeln!(&mut io::stderr(),
+                                     "error reading `{}`: not UTF-8", input.display());
+                    return $not_utf8;
+                }
+                Ok(Some(s)) => s
+            }
+        }
+    }
+}
+
+/// Separate any lines at the start of the file that begin with `%`.
+fn extract_leading_metadata<'a>(s: &'a str) -> (Vec<&'a str>, &'a str) {
+    let mut metadata = Vec::new();
+    for line in s.lines() {
+        if line.starts_with("%") {
+            // remove %<whitespace>
+            metadata.push(line.slice_from(1).trim_left())
+        } else {
+            let line_start_byte = s.subslice_offset(line);
+            return (metadata, s.slice_from(line_start_byte));
+        }
+    }
+    // if we're here, then all lines were metadata % lines.
+    (metadata, "")
+}
+
+fn load_external_files(names: &[~str]) -> Option<~str> {
+    let mut out = ~"";
+    for name in names.iter() {
+        out.push_str(load_or_return!(name.as_slice(), None, None));
+        out.push_char('\n');
+    }
+    Some(out)
+}
+
+/// Render `input` (e.g. "foo.md") into an HTML file in `output`
+/// (e.g. output = "bar" => "bar/foo.html").
+pub fn render(input: &str, mut output: Path, matches: &getopts::Matches) -> int {
+    let input_p = Path::new(input);
+    output.push(input_p.filestem().unwrap());
+    output.set_extension("html");
+
+    let mut css = ~"";
+    for name in matches.opt_strs("markdown-css").iter() {
+        let s = format!("<link rel=\"stylesheet\" type=\"text/css\" href=\"{}\">\n", name);
+        css.push_str(s)
+    }
+
+    let input_str = load_or_return!(input, 1, 2);
+
+    let (in_header, before_content, after_content) =
+        match (load_external_files(matches.opt_strs("markdown-in-header")),
+               load_external_files(matches.opt_strs("markdown-before-content")),
+               load_external_files(matches.opt_strs("markdown-after-content"))) {
+        (Some(a), Some(b), Some(c)) => (a,b,c),
+        _ => return 3
+    };
+
+    let mut out = match io::File::create(&output) {
+        Err(e) => {
+            let _ = writeln!(&mut io::stderr(),
+                             "error opening `{}` for writing: {}",
+                             output.display(), e);
+            return 4;
+        }
+        Ok(f) => f
+    };
+
+    let (metadata, text) = extract_leading_metadata(input_str);
+    if metadata.len() == 0 {
+        let _ = writeln!(&mut io::stderr(),
+                         "invalid markdown file: expecting initial line with `% ...TITLE...`");
+        return 5;
+    }
+    let title = metadata.get(0).as_slice();
+
+    reset_headers();
+
+    let err = write!(
+        &mut out,
+        r#"<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="generator" content="rustdoc">
+    <title>{title}</title>
+
+    {css}
+    {in_header}
+</head>
+<body>
+    <!--[if lte IE 8]>
+    <div class="warning">
+        This old browser is unsupported and will most likely display funky
+        things.
+    </div>
+    <![endif]-->
+
+    {before_content}
+    <h1 class="title">{title}</h1>
+    {text}
+    {after_content}
+</body>
+</html>"#,
+        title = Escape(title),
+        css = css,
+        in_header = in_header,
+        before_content = before_content,
+        text = Markdown(text),
+        after_content = after_content);
+
+    match err {
+        Err(e) => {
+            let _ = writeln!(&mut io::stderr(),
+                             "error writing to `{}`: {}",
+                             output.display(), e);
+            6
+        }
+        Ok(_) => 0
+    }
+}
+
+/// Run any tests/code examples in the markdown file `input`.
+pub fn test(input: &str, libs: @RefCell<HashSet<Path>>, mut test_args: ~[~str]) -> int {
+    let input_str = load_or_return!(input, 1, 2);
+
+    let mut collector = Collector::new(input.to_owned(), libs, true);
+    find_testable_code(input_str, &mut collector);
+    test_args.unshift(~"rustdoctest");
+    testing::test_main(test_args, collector.tests);
+    0
+}
--- a/src/librustdoc/test.rs
+++ b/src/librustdoc/test.rs
@@ -9,6 +9,7 @@
 // except according to those terms.

 use std::cell::RefCell;
+use std::char;
 use std::io;
 use std::io::Process;
 use std::local_data;
@@ -22,7 +23,6 @@
 use rustc::driver::driver;
 use rustc::driver::session;
 use rustc::metadata::creader::Loader;
-use getopts;
 use syntax::diagnostic;
 use syntax::parse;
 use syntax::codemap::CodeMap;
@@ -35,11 +35,9 @@
 use passes;
 use visit_ast::RustdocVisitor;

-pub fn run(input: &str, matches: &getopts::Matches) -> int {
+pub fn run(input: &str, libs: @RefCell<HashSet<Path>>, mut test_args: ~[~str]) -> int {
    let input_path = Path::new(input);
    let input = driver::FileInput(input_path.clone());
-    let libs = matches.opt_strs("L").map(|s| Path::new(s.as_slice()));
-    let libs = @RefCell::new(libs.move_iter().collect());

    let sessopts = @session::Options {
        maybe_sysroot: Some(@os::self_exe_path().unwrap().dir_path()),
@@ -79,21 +77,12 @@ pub fn run(input: &str, matches: &getopts::Matches) -> int {
    let (krate, _) = passes::unindent_comments(krate);
    let (krate, _) = passes::collapse_docs(krate);

-    let mut collector = Collector {
-        tests: ~[],
-        names: ~[],
-        cnt: 0,
-        libs: libs,
-        cratename: krate.name.to_owned(),
-    };
+    let mut collector = Collector::new(krate.name.to_owned(), libs, false);
    collector.fold_crate(krate);

-    let args = matches.opt_strs("test-args");
-    let mut args = args.iter().flat_map(|s| s.words()).map(|s| s.to_owned());
-    let mut args = args.to_owned_vec();
-    args.unshift(~"rustdoctest");
+    test_args.unshift(~"rustdoctest");

-    testing::test_main(args, collector.tests);
+    testing::test_main(test_args, collector.tests);

    0
 }
@@ -198,17 +187,35 @@ fn maketest(s: &str, cratename: &str) -> ~str {
 }

 pub struct Collector {
-    priv tests: ~[testing::TestDescAndFn],
+    tests: ~[testing::TestDescAndFn],
    priv names: ~[~str],
    priv libs: @RefCell<HashSet<Path>>,
    priv cnt: uint,
+    priv use_headers: bool,
+    priv current_header: Option<~str>,
    priv cratename: ~str,
 }

 impl Collector {
-    pub fn add_test(&mut self, test: &str, should_fail: bool, no_run: bool) {
-        let test = test.to_owned();
-        let name = format!("{}_{}", self.names.connect("::"), self.cnt);
+    pub fn new(cratename: ~str, libs: @RefCell<HashSet<Path>>, use_headers: bool) -> Collector {
+        Collector {
+            tests: ~[],
+            names: ~[],
+            libs: libs,
+            cnt: 0,
+            use_headers: use_headers,
+            current_header: None,
+            cratename: cratename
+        }
+    }
+
+    pub fn add_test(&mut self, test: ~str, should_fail: bool, no_run: bool) {
+        let name = if self.use_headers {
+            let s = self.current_header.as_ref().map(|s| s.as_slice()).unwrap_or("");
+            format!("{}_{}", s, self.cnt)
+        } else {
+            format!("{}_{}", self.names.connect("::"), self.cnt)
+        };
        self.cnt += 1;
        let libs = self.libs.borrow();
        let libs = (*libs.get()).clone();
@@ -225,6 +232,25 @@ pub fn add_test(&mut self, test: &str, should_fail: bool, no_run: bool) {
            }),
        });
    }
+
+    pub fn register_header(&mut self, name: &str, level: u32) {
+        if self.use_headers && level == 1 {
+            // we use these headings as test names, so it's good if
+            // they're valid identifiers.
+            let name = name.chars().enumerate().map(|(i, c)| {
+                    if (i == 0 && char::is_XID_start(c)) ||
+                        (i != 0 && char::is_XID_continue(c)) {
+                        c
+                    } else {
+                        '_'
+                    }
+                }).collect::<~str>();
+
+            // new header => reset count.
+            self.cnt = 0;
+            self.current_header = Some(name);
+        }
+    }
 }

 impl DocFolder for Collector {
@@ -237,7 +263,7 @@ fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
        match item.doc_value() {
            Some(doc) => {
                self.cnt = 0;
-                markdown::find_testable_code(doc, self);
+                markdown::find_testable_code(doc, &mut *self);
            }
            None => {}
        }