Auto merge of #46450 - Gilnaa:libtest_json_output, r=nrc

Libtest json output

A revisit to my [last PR](https://github.com/rust-lang/rust/pull/45923).

Events are now more atomic, printed in a flat hierarchy.

For the normal test output:
```
running 1 test
test f ... FAILED

failures:

---- f stdout ----
	thread 'f' panicked at 'assertion failed: `(left == right)`
  left: `3`,
 right: `4`', f.rs:3:1
note: Run with `RUST_BACKTRACE=1` for a backtrace.

failures:
    f

test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out
```

The JSON equivalent is:
```
{ "type": "suite", "event": "started", "test_count": "1" }
{ "type": "test", "event": "started", "name": "f" }
{ "type": "test", "event": "failed", "name": "f" }
{ "type": "suite", "event": "failed", "passed": 0, "failed": 1, "allowed_fail": 0, "ignored": 0,  "measured": 0, "filtered_out": "0" }
{ "type": "test_output", "name": "f", "output": "thread 'f' panicked at 'assertion failed: `(left == right)`
  left: `3`,
 right: `4`', f.rs:3:1
note: Run with `RUST_BACKTRACE=1` for a backtrace.
" }
```

src/libtest/formatters/json.rs

+// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::*;
+
+pub(crate) struct JsonFormatter<T> {
+    out: OutputLocation<T>,
+}
+
+impl<T: Write> JsonFormatter<T> {
+    pub fn new(out: OutputLocation<T>) -> Self {
+        Self { out }
+    }
+
+    fn write_message(&mut self, s: &str) -> io::Result<()> {
+        assert!(!s.contains('\n'));
+
+        self.out.write_all(s.as_ref())?;
+        self.out.write_all(b"\n")
+    }
+
+    fn write_event(
+        &mut self,
+        ty: &str,
+        name: &str,
+        evt: &str,
+        extra: Option<String>,
+    ) -> io::Result<()> {
+        if let Some(extras) = extra {
+            self.write_message(&*format!(
+                r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#,
+                ty,
+                name,
+                evt,
+                extras
+            ))
+        } else {
+            self.write_message(&*format!(
+                r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#,
+                ty,
+                name,
+                evt
+            ))
+        }
+    }
+}
+
+impl<T: Write> OutputFormatter for JsonFormatter<T> {
+    fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
+        self.write_message(&*format!(
+            r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#,
+            test_count
+        ))
+    }
+
+    fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
+        self.write_message(&*format!(
+            r#"{{ "type": "test", "event": "started", "name": "{}" }}"#,
+            desc.name
+        ))
+    }
+
+    fn write_result(
+        &mut self,
+        desc: &TestDesc,
+        result: &TestResult,
+        stdout: &[u8],
+    ) -> io::Result<()> {
+        match *result {
+            TrOk => self.write_event("test", desc.name.as_slice(), "ok", None),
+
+            TrFailed => {
+                let extra_data = if stdout.len() > 0 {
+                    Some(format!(
+                        r#""stdout": "{}""#,
+                        EscapedString(String::from_utf8_lossy(stdout))
+                    ))
+                } else {
+                    None
+                };
+
+                self.write_event("test", desc.name.as_slice(), "failed", extra_data)
+            }
+
+            TrFailedMsg(ref m) => {
+                self.write_event(
+                    "test",
+                    desc.name.as_slice(),
+                    "failed",
+                    Some(format!(r#""message": "{}""#, EscapedString(m))),
+                )
+            }
+
+            TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None),
+
+            TrAllowedFail => {
+                self.write_event("test", desc.name.as_slice(), "allowed_failure", None)
+            }
+
+            TrBench(ref bs) => {
+                let median = bs.ns_iter_summ.median as usize;
+                let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
+
+                let mbps = if bs.mb_s == 0 {
+                    "".into()
+                } else {
+                    format!(r#", "mib_per_second": {}"#, bs.mb_s)
+                };
+
+                let line = format!(
+                    "{{ \"type\": \"bench\", \
+                                \"name\": \"{}\", \
+                                \"median\": {}, \
+                                \"deviation\": {}{} }}",
+                    desc.name,
+                    median,
+                    deviation,
+                    mbps
+                );
+
+                self.write_message(&*line)
+            }
+        }
+    }
+
+    fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
+        self.write_message(&*format!(
+            r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#,
+            desc.name
+        ))
+    }
+
+    fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
+
+        self.write_message(&*format!(
+            "{{ \"type\": \"suite\", \
+            \"event\": \"{}\", \
+            \"passed\": {}, \
+            \"failed\": {}, \
+            \"allowed_fail\": {}, \
+            \"ignored\": {}, \
+            \"measured\": {}, \
+            \"filtered_out\": \"{}\" }}",
+            if state.failed == 0 { "ok" } else { "failed" },
+            state.passed,
+            state.failed + state.allowed_fail,
+            state.allowed_fail,
+            state.ignored,
+            state.measured,
+            state.filtered_out
+        ))?;
+
+        Ok(state.failed == 0)
+    }
+}
+
+/// A formatting utility used to print strings with characters in need of escaping.
+/// Base code taken form `libserialize::json::escape_str`
+struct EscapedString<S: AsRef<str>>(S);
+
+impl<S: AsRef<str>> ::std::fmt::Display for EscapedString<S> {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        let mut start = 0;
+
+        for (i, byte) in self.0.as_ref().bytes().enumerate() {
+            let escaped = match byte {
+                b'"' => "\\\"",
+                b'\\' => "\\\\",
+                b'\x00' => "\\u0000",
+                b'\x01' => "\\u0001",
+                b'\x02' => "\\u0002",
+                b'\x03' => "\\u0003",
+                b'\x04' => "\\u0004",
+                b'\x05' => "\\u0005",
+                b'\x06' => "\\u0006",
+                b'\x07' => "\\u0007",
+                b'\x08' => "\\b",
+                b'\t' => "\\t",
+                b'\n' => "\\n",
+                b'\x0b' => "\\u000b",
+                b'\x0c' => "\\f",
+                b'\r' => "\\r",
+                b'\x0e' => "\\u000e",
+                b'\x0f' => "\\u000f",
+                b'\x10' => "\\u0010",
+                b'\x11' => "\\u0011",
+                b'\x12' => "\\u0012",
+                b'\x13' => "\\u0013",
+                b'\x14' => "\\u0014",
+                b'\x15' => "\\u0015",
+                b'\x16' => "\\u0016",
+                b'\x17' => "\\u0017",
+                b'\x18' => "\\u0018",
+                b'\x19' => "\\u0019",
+                b'\x1a' => "\\u001a",
+                b'\x1b' => "\\u001b",
+                b'\x1c' => "\\u001c",
+                b'\x1d' => "\\u001d",
+                b'\x1e' => "\\u001e",
+                b'\x1f' => "\\u001f",
+                b'\x7f' => "\\u007f",
+                _ => {
+                    continue;
+                }
+            };
+
+            if start < i {
+                f.write_str(&self.0.as_ref()[start..i])?;
+            }
+
+            f.write_str(escaped)?;
+
+            start = i + 1;
+        }
+
+        if start != self.0.as_ref().len() {
+            f.write_str(&self.0.as_ref()[start..])?;
+        }
+
+        Ok(())
+    }
+}

src/libtest/formatters/mod.rs

+// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::*;
+
+mod pretty;
+mod json;
+mod terse;
+
+pub(crate) use self::pretty::PrettyFormatter;
+pub(crate) use self::json::JsonFormatter;
+pub(crate) use self::terse::TerseFormatter;
+
+pub(crate) trait OutputFormatter {
+    fn write_run_start(&mut self, test_count: usize) -> io::Result<()>;
+    fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>;
+    fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>;
+    fn write_result(
+        &mut self,
+        desc: &TestDesc,
+        result: &TestResult,
+        stdout: &[u8],
+    ) -> io::Result<()>;
+    fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool>;
+}

src/libtest/formatters/pretty.rs

+// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::*;
+
+pub(crate) struct PrettyFormatter<T> {
+    out: OutputLocation<T>,
+    use_color: bool,
+
+    /// Number of columns to fill when aligning names
+    max_name_len: usize,
+
+    is_multithreaded: bool,
+}
+
+impl<T: Write> PrettyFormatter<T> {
+    pub fn new(
+        out: OutputLocation<T>,
+        use_color: bool,
+        max_name_len: usize,
+        is_multithreaded: bool,
+    ) -> Self {
+        PrettyFormatter {
+            out,
+            use_color,
+            max_name_len,
+            is_multithreaded,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn output_location(&self) -> &OutputLocation<T> {
+        &self.out
+    }
+
+    pub fn write_ok(&mut self) -> io::Result<()> {
+        self.write_short_result("ok", term::color::GREEN)
+    }
+
+    pub fn write_failed(&mut self) -> io::Result<()> {
+        self.write_short_result("FAILED", term::color::RED)
+    }
+
+    pub fn write_ignored(&mut self) -> io::Result<()> {
+        self.write_short_result("ignored", term::color::YELLOW)
+    }
+
+    pub fn write_allowed_fail(&mut self) -> io::Result<()> {
+        self.write_short_result("FAILED (allowed)", term::color::YELLOW)
+    }
+
+    pub fn write_bench(&mut self) -> io::Result<()> {
+        self.write_pretty("bench", term::color::CYAN)
+    }
+
+    pub fn write_short_result(
+        &mut self,
+        result: &str,
+        color: term::color::Color,
+    ) -> io::Result<()> {
+        self.write_pretty(result, color)?;
+        self.write_plain("\n")
+    }
+
+    pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
+        match self.out {
+            Pretty(ref mut term) => {
+                if self.use_color {
+                    term.fg(color)?;
+                }
+                term.write_all(word.as_bytes())?;
+                if self.use_color {
+                    term.reset()?;
+                }
+                term.flush()
+            }
+            Raw(ref mut stdout) => {
+                stdout.write_all(word.as_bytes())?;
+                stdout.flush()
+            }
+        }
+    }
+
+    pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> {
+        let s = s.as_ref();
+        self.out.write_all(s.as_bytes())?;
+        self.out.flush()
+    }
+
+    pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> {
+        self.write_plain("\nsuccesses:\n")?;
+        let mut successes = Vec::new();
+        let mut stdouts = String::new();
+        for &(ref f, ref stdout) in &state.not_failures {
+            successes.push(f.name.to_string());
+            if !stdout.is_empty() {
+                stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name));
+                let output = String::from_utf8_lossy(stdout);
+                stdouts.push_str(&output);
+                stdouts.push_str("\n");
+            }
+        }
+        if !stdouts.is_empty() {
+            self.write_plain("\n")?;
+            self.write_plain(&stdouts)?;
+        }
+
+        self.write_plain("\nsuccesses:\n")?;
+        successes.sort();
+        for name in &successes {
+            self.write_plain(&format!("    {}\n", name))?;
+        }
+        Ok(())
+    }
+
+    pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> {
+        self.write_plain("\nfailures:\n")?;
+        let mut failures = Vec::new();
+        let mut fail_out = String::new();
+        for &(ref f, ref stdout) in &state.failures {
+            failures.push(f.name.to_string());
+            if !stdout.is_empty() {
+                fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
+                let output = String::from_utf8_lossy(stdout);
+                fail_out.push_str(&output);
+                fail_out.push_str("\n");
+            }
+        }
+        if !fail_out.is_empty() {
+            self.write_plain("\n")?;
+            self.write_plain(&fail_out)?;
+        }
+
+        self.write_plain("\nfailures:\n")?;
+        failures.sort();
+        for name in &failures {
+            self.write_plain(&format!("    {}\n", name))?;
+        }
+        Ok(())
+    }
+
+    fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
+        let name = desc.padded_name(self.max_name_len, desc.name.padding());
+        self.write_plain(&format!("test {} ... ", name))?;
+
+        Ok(())
+    }
+}
+
+impl<T: Write> OutputFormatter for PrettyFormatter<T> {
+    fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
+        let noun = if test_count != 1 { "tests" } else { "test" };
+        self.write_plain(&format!("\nrunning {} {}\n", test_count, noun))
+    }
+
+    fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
+        // When running tests concurrently, we should not print
+        // the test's name as the result will be mis-aligned.
+        // When running the tests serially, we print the name here so
+        // that the user can see which test hangs.
+        if !self.is_multithreaded {
+            self.write_test_name(desc)?;
+        }
+
+        Ok(())
+    }
+
+    fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> {
+        if self.is_multithreaded {
+            self.write_test_name(desc)?;
+        }
+
+        match *result {
+            TrOk => self.write_ok(),
+            TrFailed | TrFailedMsg(_) => self.write_failed(),
+            TrIgnored => self.write_ignored(),
+            TrAllowedFail => self.write_allowed_fail(),
+            TrBench(ref bs) => {
+                self.write_bench()?;
+                self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
+            }
+        }
+    }
+
+    fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
+        if self.is_multithreaded {
+            self.write_test_name(desc)?;
+        }
+
+        self.write_plain(&format!(
+            "test {} has been running for over {} seconds\n",
+            desc.name,
+            TEST_WARN_TIMEOUT_S
+        ))
+    }
+
+    fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
+        if state.options.display_output {
+            self.write_successes(state)?;
+        }
+        let success = state.failed == 0;
+        if !success {
+            self.write_failures(state)?;
+        }
+
+        self.write_plain("\ntest result: ")?;
+
+        if success {
+            // There's no parallelism at this point so it's safe to use color
+            self.write_pretty("ok", term::color::GREEN)?;
+        } else {
+            self.write_pretty("FAILED", term::color::RED)?;
+        }
+
+        let s = if state.allowed_fail > 0 {
+            format!(
+                ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n",
+                state.passed,
+                state.failed + state.allowed_fail,
+                state.allowed_fail,
+                state.ignored,
+                state.measured,
+                state.filtered_out
+            )
+        } else {
+            format!(
+                ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
+                state.passed,
+                state.failed,
+                state.ignored,
+                state.measured,
+                state.filtered_out
+            )
+        };
+
+        self.write_plain(&s)?;
+
+        Ok(success)
+    }
+}

src/libtest/formatters/terse.rs

+// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::*;
+
+pub(crate) struct TerseFormatter<T> {
+    out: OutputLocation<T>,
+    use_color: bool,
+    is_multithreaded: bool,
+    /// Number of columns to fill when aligning names
+    max_name_len: usize,
+
+    test_count: usize,
+}
+
+impl<T: Write> TerseFormatter<T> {
+    pub fn new(
+        out: OutputLocation<T>,
+        use_color: bool,
+        max_name_len: usize,
+        is_multithreaded: bool,
+    ) -> Self {
+        TerseFormatter {
+            out,
+            use_color,
+            max_name_len,
+            is_multithreaded,
+            test_count: 0,
+        }
+    }
+
+    pub fn write_ok(&mut self) -> io::Result<()> {
+        self.write_short_result(".", term::color::GREEN)
+    }
+
+    pub fn write_failed(&mut self) -> io::Result<()> {
+        self.write_short_result("F", term::color::RED)
+    }
+
+    pub fn write_ignored(&mut self) -> io::Result<()> {
+        self.write_short_result("i", term::color::YELLOW)
+    }
+
+    pub fn write_allowed_fail(&mut self) -> io::Result<()> {
+        self.write_short_result("a", term::color::YELLOW)
+    }
+
+    pub fn write_bench(&mut self) -> io::Result<()> {
+        self.write_pretty("bench", term::color::CYAN)
+    }
+
+    pub fn write_short_result(
+        &mut self,
+        result: &str,
+        color: term::color::Color,
+    ) -> io::Result<()> {
+        self.write_pretty(result, color)?;
+        if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 {
+            // we insert a new line every 100 dots in order to flush the
+            // screen when dealing with line-buffered output (e.g. piping to
+            // `stamp` in the rust CI).
+            self.write_plain("\n")?;
+        }
+
+        self.test_count += 1;
+        Ok(())
+    }
+
+    pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
+        match self.out {
+            Pretty(ref mut term) => {
+                if self.use_color {
+                    term.fg(color)?;
+                }
+                term.write_all(word.as_bytes())?;
+                if self.use_color {
+                    term.reset()?;
+                }
+                term.flush()
+            }
+            Raw(ref mut stdout) => {
+                stdout.write_all(word.as_bytes())?;
+                stdout.flush()
+            }
+        }
+    }
+
+    pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> {
+        let s = s.as_ref();
+        self.out.write_all(s.as_bytes())?;
+        self.out.flush()
+    }
+
+    pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> {
+        self.write_plain("\nsuccesses:\n")?;
+        let mut successes = Vec::new();
+        let mut stdouts = String::new();
+        for &(ref f, ref stdout) in &state.not_failures {
+            successes.push(f.name.to_string());
+            if !stdout.is_empty() {
+                stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name));
+                let output = String::from_utf8_lossy(stdout);
+                stdouts.push_str(&output);
+                stdouts.push_str("\n");
+            }
+        }
+        if !stdouts.is_empty() {
+            self.write_plain("\n")?;
+            self.write_plain(&stdouts)?;
+        }
+
+        self.write_plain("\nsuccesses:\n")?;
+        successes.sort();
+        for name in &successes {
+            self.write_plain(&format!("    {}\n", name))?;
+        }
+        Ok(())
+    }
+
+    pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> {
+        self.write_plain("\nfailures:\n")?;
+        let mut failures = Vec::new();
+        let mut fail_out = String::new();
+        for &(ref f, ref stdout) in &state.failures {
+            failures.push(f.name.to_string());
+            if !stdout.is_empty() {
+                fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
+                let output = String::from_utf8_lossy(stdout);
+                fail_out.push_str(&output);
+                fail_out.push_str("\n");
+            }
+        }
+        if !fail_out.is_empty() {
+            self.write_plain("\n")?;
+            self.write_plain(&fail_out)?;
+        }
+
+        self.write_plain("\nfailures:\n")?;
+        failures.sort();
+        for name in &failures {
+            self.write_plain(&format!("    {}\n", name))?;
+        }
+        Ok(())
+    }
+
+    fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
+        let name = desc.padded_name(self.max_name_len, desc.name.padding());
+        self.write_plain(&format!("test {} ... ", name))?;
+
+        Ok(())
+    }
+}
+
+impl<T: Write> OutputFormatter for TerseFormatter<T> {
+    fn write_run_start(&mut self, test_count: usize) -> io::Result<()> {
+        let noun = if test_count != 1 { "tests" } else { "test" };
+        self.write_plain(&format!("\nrunning {} {}\n", test_count, noun))
+    }
+
+    fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> {
+        // Remnants from old libtest code that used the padding value
+        // in order to indicate benchmarks.
+        // When running benchmarks, terse-mode should still print their name as if
+        // it is the Pretty formatter.
+        if !self.is_multithreaded && desc.name.padding() == PadOnRight {
+            self.write_test_name(desc)?;
+        }
+
+        Ok(())
+    }
+
+    fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> {
+        match *result {
+            TrOk => self.write_ok(),
+            TrFailed | TrFailedMsg(_) => self.write_failed(),
+            TrIgnored => self.write_ignored(),
+            TrAllowedFail => self.write_allowed_fail(),
+            TrBench(ref bs) => {
+                if self.is_multithreaded {
+                    self.write_test_name(desc)?;
+                }
+                self.write_bench()?;
+                self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
+            }
+        }
+    }
+
+    fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> {
+        self.write_plain(&format!(
+            "test {} has been running for over {} seconds\n",
+            desc.name,
+            TEST_WARN_TIMEOUT_S
+        ))
+    }
+
+    fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
+        if state.options.display_output {
+            self.write_outputs(state)?;
+        }
+        let success = state.failed == 0;
+        if !success {
+            self.write_failures(state)?;
+        }
+
+        self.write_plain("\ntest result: ")?;
+
+        if success {
+            // There's no parallelism at this point so it's safe to use color
+            self.write_pretty("ok", term::color::GREEN)?;
+        } else {
+            self.write_pretty("FAILED", term::color::RED)?;
+        }
+
+        let s = if state.allowed_fail > 0 {
+            format!(
+                ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n",
+                state.passed,
+                state.failed + state.allowed_fail,
+                state.allowed_fail,
+                state.ignored,
+                state.measured,
+                state.filtered_out
+            )
+        } else {
+            format!(
+                ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
+                state.passed,
+                state.failed,
+                state.ignored,
+                state.measured,
+                state.filtered_out
+            )
+        };
+
+        self.write_plain(&s)?;
+
+        Ok(success)
+    }
+}

src/libtest/stats.rs

@@ -400,16 +400,18 @@ fn test_norm2() {
     }
     #[test]
     fn test_norm10narrow() {
-        let val = &[966.0000000000,
-                    985.0000000000,
-                    1110.0000000000,
-                    848.0000000000,
-                    821.0000000000,
-                    975.0000000000,
-                    962.0000000000,
-                    1157.0000000000,
-                    1217.0000000000,
-                    955.0000000000];
+        let val = &[
+            966.0000000000,
+            985.0000000000,
+            1110.0000000000,
+            848.0000000000,
+            821.0000000000,
+            975.0000000000,
+            962.0000000000,
+            1157.0000000000,
+            1217.0000000000,
+            955.0000000000,
+        ];
         let summ = &Summary {
             sum: 9996.0000000000,
             min: 821.0000000000,
@@ -428,16 +430,18 @@ fn test_norm10narrow() {
     }
     #[test]
     fn test_norm10medium() {
-        let val = &[954.0000000000,
-                    1064.0000000000,
-                    855.0000000000,
-                    1000.0000000000,
-                    743.0000000000,
-                    1084.0000000000,
-                    704.0000000000,
-                    1023.0000000000,
-                    357.0000000000,
-                    869.0000000000];
+        let val = &[
+            954.0000000000,
+            1064.0000000000,
+            855.0000000000,
+            1000.0000000000,
+            743.0000000000,
+            1084.0000000000,
+            704.0000000000,
+            1023.0000000000,
+            357.0000000000,
+            869.0000000000,
+        ];
         let summ = &Summary {
             sum: 8653.0000000000,
             min: 357.0000000000,
@@ -456,16 +460,18 @@ fn test_norm10medium() {
     }
     #[test]
     fn test_norm10wide() {
-        let val = &[505.0000000000,
-                    497.0000000000,
-                    1591.0000000000,
-                    887.0000000000,
-                    1026.0000000000,
-                    136.0000000000,
-                    1580.0000000000,
-                    940.0000000000,
-                    754.0000000000,
-                    1433.0000000000];
+        let val = &[
+            505.0000000000,
+            497.0000000000,
+            1591.0000000000,
+            887.0000000000,
+            1026.0000000000,
+            136.0000000000,
+            1580.0000000000,
+            940.0000000000,
+            754.0000000000,
+            1433.0000000000,
+        ];
         let summ = &Summary {
             sum: 9349.0000000000,
             min: 136.0000000000,
@@ -484,31 +490,33 @@ fn test_norm10wide() {
     }
     #[test]
     fn test_norm25verynarrow() {
-        let val = &[991.0000000000,
-                    1018.0000000000,
-                    998.0000000000,
-                    1013.0000000000,
-                    974.0000000000,
-                    1007.0000000000,
-                    1014.0000000000,
-                    999.0000000000,
-                    1011.0000000000,
-                    978.0000000000,
-                    985.0000000000,
-                    999.0000000000,
-                    983.0000000000,
-                    982.0000000000,
-                    1015.0000000000,
-                    1002.0000000000,
-                    977.0000000000,
-                    948.0000000000,
-                    1040.0000000000,
-                    974.0000000000,
-                    996.0000000000,
-                    989.0000000000,
-                    1015.0000000000,
-                    994.0000000000,
-                    1024.0000000000];
+        let val = &[
+            991.0000000000,
+            1018.0000000000,
+            998.0000000000,
+            1013.0000000000,
+            974.0000000000,
+            1007.0000000000,
+            1014.0000000000,
+            999.0000000000,
+            1011.0000000000,
+            978.0000000000,
+            985.0000000000,
+            999.0000000000,
+            983.0000000000,
+            982.0000000000,
+            1015.0000000000,
+            1002.0000000000,
+            977.0000000000,
+            948.0000000000,
+            1040.0000000000,
+            974.0000000000,
+            996.0000000000,
+            989.0000000000,
+            1015.0000000000,
+            994.0000000000,
+            1024.0000000000,
+        ];
         let summ = &Summary {
             sum: 24926.0000000000,
             min: 948.0000000000,
@@ -527,16 +535,18 @@ fn test_norm25verynarrow() {
     }
     #[test]
     fn test_exp10a() {
-        let val = &[23.0000000000,
-                    11.0000000000,
-                    2.0000000000,
-                    57.0000000000,
-                    4.0000000000,
-                    12.0000000000,
-                    5.0000000000,
-                    29.0000000000,
-                    3.0000000000,
-                    21.0000000000];
+        let val = &[
+            23.0000000000,
+            11.0000000000,
+            2.0000000000,
+            57.0000000000,
+            4.0000000000,
+            12.0000000000,
+            5.0000000000,
+            29.0000000000,
+            3.0000000000,
+            21.0000000000,
+        ];
         let summ = &Summary {
             sum: 167.0000000000,
             min: 2.0000000000,
@@ -555,16 +565,18 @@ fn test_exp10a() {
     }
     #[test]
     fn test_exp10b() {
-        let val = &[24.0000000000,
-                    17.0000000000,
-                    6.0000000000,
-                    38.0000000000,
-                    25.0000000000,
-                    7.0000000000,
-                    51.0000000000,
-                    2.0000000000,
-                    61.0000000000,
-                    32.0000000000];
+        let val = &[
+            24.0000000000,
+            17.0000000000,
+            6.0000000000,
+            38.0000000000,
+            25.0000000000,
+            7.0000000000,
+            51.0000000000,
+            2.0000000000,
+            61.0000000000,
+            32.0000000000,
+        ];
         let summ = &Summary {
             sum: 263.0000000000,
             min: 2.0000000000,
@@ -583,16 +595,18 @@ fn test_exp10b() {
     }
     #[test]
     fn test_exp10c() {
-        let val = &[71.0000000000,
-                    2.0000000000,
-                    32.0000000000,
-                    1.0000000000,
-                    6.0000000000,
-                    28.0000000000,
-                    13.0000000000,
-                    37.0000000000,
-                    16.0000000000,
-                    36.0000000000];
+        let val = &[
+            71.0000000000,
+            2.0000000000,
+            32.0000000000,
+            1.0000000000,
+            6.0000000000,
+            28.0000000000,
+            13.0000000000,
+            37.0000000000,
+            16.0000000000,
+            36.0000000000,
+        ];
         let summ = &Summary {
             sum: 242.0000000000,
             min: 1.0000000000,
@@ -611,31 +625,33 @@ fn test_exp10c() {
     }
     #[test]
     fn test_exp25() {
-        let val = &[3.0000000000,
-                    24.0000000000,
-                    1.0000000000,
-                    19.0000000000,
-                    7.0000000000,
-                    5.0000000000,
-                    30.0000000000,
-                    39.0000000000,
-                    31.0000000000,
-                    13.0000000000,
-                    25.0000000000,
-                    48.0000000000,
-                    1.0000000000,
-                    6.0000000000,
-                    42.0000000000,
-                    63.0000000000,
-                    2.0000000000,
-                    12.0000000000,
-                    108.0000000000,
-                    26.0000000000,
-                    1.0000000000,
-                    7.0000000000,
-                    44.0000000000,
-                    25.0000000000,
-                    11.0000000000];
+        let val = &[
+            3.0000000000,
+            24.0000000000,
+            1.0000000000,
+            19.0000000000,
+            7.0000000000,
+            5.0000000000,
+            30.0000000000,
+            39.0000000000,
+            31.0000000000,
+            13.0000000000,
+            25.0000000000,
+            48.0000000000,
+            1.0000000000,
+            6.0000000000,
+            42.0000000000,
+            63.0000000000,
+            2.0000000000,
+            12.0000000000,
+            108.0000000000,
+            26.0000000000,
+            1.0000000000,
+            7.0000000000,
+            44.0000000000,
+            25.0000000000,
+            11.0000000000,
+        ];
         let summ = &Summary {
             sum: 593.0000000000,
             min: 1.0000000000,
@@ -654,31 +670,33 @@ fn test_exp25() {
     }
     #[test]
     fn test_binom25() {
-        let val = &[18.0000000000,
-                    17.0000000000,
-                    27.0000000000,
-                    15.0000000000,
-                    21.0000000000,
-                    25.0000000000,
-                    17.0000000000,
-                    24.0000000000,
-                    25.0000000000,
-                    24.0000000000,
-                    26.0000000000,
-                    26.0000000000,
-                    23.0000000000,
-                    15.0000000000,
-                    23.0000000000,
-                    17.0000000000,
-                    18.0000000000,
-                    18.0000000000,
-                    21.0000000000,
-                    16.0000000000,
-                    15.0000000000,
-                    31.0000000000,
-                    20.0000000000,
-                    17.0000000000,
-                    15.0000000000];
+        let val = &[
+            18.0000000000,
+            17.0000000000,
+            27.0000000000,
+            15.0000000000,
+            21.0000000000,
+            25.0000000000,
+            17.0000000000,
+            24.0000000000,
+            25.0000000000,
+            24.0000000000,
+            26.0000000000,
+            26.0000000000,
+            23.0000000000,
+            15.0000000000,
+            23.0000000000,
+            17.0000000000,
+            18.0000000000,
+            18.0000000000,
+            21.0000000000,
+            16.0000000000,
+            15.0000000000,
+            31.0000000000,
+            20.0000000000,
+            17.0000000000,
+            15.0000000000,
+        ];
         let summ = &Summary {
             sum: 514.0000000000,
             min: 15.0000000000,
@@ -697,31 +715,33 @@ fn test_binom25() {
     }
     #[test]
     fn test_pois25lambda30() {
-        let val = &[27.0000000000,
-                    33.0000000000,
-                    34.0000000000,
-                    34.0000000000,
-                    24.0000000000,
-                    39.0000000000,
-                    28.0000000000,
-                    27.0000000000,
-                    31.0000000000,
-                    28.0000000000,
-                    38.0000000000,
-                    21.0000000000,
-                    33.0000000000,
-                    36.0000000000,
-                    29.0000000000,
-                    37.0000000000,
-                    32.0000000000,
-                    34.0000000000,
-                    31.0000000000,
-                    39.0000000000,
-                    25.0000000000,
-                    31.0000000000,
-                    32.0000000000,
-                    40.0000000000,
-                    24.0000000000];
+        let val = &[
+            27.0000000000,
+            33.0000000000,
+            34.0000000000,
+            34.0000000000,
+            24.0000000000,
+            39.0000000000,
+            28.0000000000,
+            27.0000000000,
+            31.0000000000,
+            28.0000000000,
+            38.0000000000,
+            21.0000000000,
+            33.0000000000,
+            36.0000000000,
+            29.0000000000,
+            37.0000000000,
+            32.0000000000,
+            34.0000000000,
+            31.0000000000,
+            39.0000000000,
+            25.0000000000,
+            31.0000000000,
+            32.0000000000,
+            40.0000000000,
+            24.0000000000,
+        ];
         let summ = &Summary {
             sum: 787.0000000000,
             min: 21.0000000000,
@@ -740,31 +760,33 @@ fn test_pois25lambda30() {
     }
     #[test]
     fn test_pois25lambda40() {
-        let val = &[42.0000000000,
-                    50.0000000000,
-                    42.0000000000,
-                    46.0000000000,
-                    34.0000000000,
-                    45.0000000000,
-                    34.0000000000,
-                    49.0000000000,
-                    39.0000000000,
-                    28.0000000000,
-                    40.0000000000,
-                    35.0000000000,
-                    37.0000000000,
-                    39.0000000000,
-                    46.0000000000,
-                    44.0000000000,
-                    32.0000000000,
-                    45.0000000000,
-                    42.0000000000,
-                    37.0000000000,
-                    48.0000000000,
-                    42.0000000000,
-                    33.0000000000,
-                    42.0000000000,
-                    48.0000000000];
+        let val = &[
+            42.0000000000,
+            50.0000000000,
+            42.0000000000,
+            46.0000000000,
+            34.0000000000,
+            45.0000000000,
+            34.0000000000,
+            49.0000000000,
+            39.0000000000,
+            28.0000000000,
+            40.0000000000,
+            35.0000000000,
+            37.0000000000,
+            39.0000000000,
+            46.0000000000,
+            44.0000000000,
+            32.0000000000,
+            45.0000000000,
+            42.0000000000,
+            37.0000000000,
+            48.0000000000,
+            42.0000000000,
+            33.0000000000,
+            42.0000000000,
+            48.0000000000,
+        ];
         let summ = &Summary {
             sum: 1019.0000000000,
             min: 28.0000000000,
@@ -783,31 +805,33 @@ fn test_pois25lambda40() {
     }
     #[test]
     fn test_pois25lambda50() {
-        let val = &[45.0000000000,
-                    43.0000000000,
-                    44.0000000000,
-                    61.0000000000,
-                    51.0000000000,
-                    53.0000000000,
-                    59.0000000000,
-                    52.0000000000,
-                    49.0000000000,
-                    51.0000000000,
-                    51.0000000000,
-                    50.0000000000,
-                    49.0000000000,
-                    56.0000000000,
-                    42.0000000000,
-                    52.0000000000,
-                    51.0000000000,
-                    43.0000000000,
-                    48.0000000000,
-                    48.0000000000,
-                    50.0000000000,
-                    42.0000000000,
-                    43.0000000000,
-                    42.0000000000,
-                    60.0000000000];
+        let val = &[
+            45.0000000000,
+            43.0000000000,
+            44.0000000000,
+            61.0000000000,
+            51.0000000000,
+            53.0000000000,
+            59.0000000000,
+            52.0000000000,
+            49.0000000000,
+            51.0000000000,
+            51.0000000000,
+            50.0000000000,
+            49.0000000000,
+            56.0000000000,
+            42.0000000000,
+            52.0000000000,
+            51.0000000000,
+            43.0000000000,
+            48.0000000000,
+            48.0000000000,
+            50.0000000000,
+            42.0000000000,
+            43.0000000000,
+            42.0000000000,
+            60.0000000000,
+        ];
         let summ = &Summary {
             sum: 1235.0000000000,
             min: 42.0000000000,
@@ -826,31 +850,33 @@ fn test_pois25lambda50() {
     }
     #[test]
     fn test_unif25() {
-        let val = &[99.0000000000,
-                    55.0000000000,
-                    92.0000000000,
-                    79.0000000000,
-                    14.0000000000,
-                    2.0000000000,
-                    33.0000000000,
-                    49.0000000000,
-                    3.0000000000,
-                    32.0000000000,
-                    84.0000000000,
-                    59.0000000000,
-                    22.0000000000,
-                    86.0000000000,
-                    76.0000000000,
-                    31.0000000000,
-                    29.0000000000,
-                    11.0000000000,
-                    41.0000000000,
-                    53.0000000000,
-                    45.0000000000,
-                    44.0000000000,
-                    98.0000000000,
-                    98.0000000000,
-                    7.0000000000];
+        let val = &[
+            99.0000000000,
+            55.0000000000,
+            92.0000000000,
+            79.0000000000,
+            14.0000000000,
+            2.0000000000,
+            33.0000000000,
+            49.0000000000,
+            3.0000000000,
+            32.0000000000,
+            84.0000000000,
+            59.0000000000,
+            22.0000000000,
+            86.0000000000,
+            76.0000000000,
+            31.0000000000,
+            29.0000000000,
+            11.0000000000,
+            41.0000000000,
+            53.0000000000,
+            45.0000000000,
+            44.0000000000,
+            98.0000000000,
+            98.0000000000,
+            7.0000000000,
+        ];
         let summ = &Summary {
             sum: 1242.0000000000,
             min: 2.0000000000,
@@ -885,18 +911,14 @@ mod bench {
 
     #[bench]
     pub fn sum_three_items(b: &mut Bencher) {
-        b.iter(|| {
-            [1e20f64, 1.5f64, -1e20f64].sum();
-        })
+        b.iter(|| { [1e20f64, 1.5f64, -1e20f64].sum(); })
     }
     #[bench]
     pub fn sum_many_f64(b: &mut Bencher) {
         let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60];
         let v = (0..500).map(|i| nums[i % 5]).collect::<Vec<_>>();
 
-        b.iter(|| {
-            v.sum();
-        })
+        b.iter(|| { v.sum(); })
     }
 
     #[bench]

src/test/run-make/libtest-json/Makefile

+-include ../tools.mk
+
+# Test expected libtest's JSON output
+
+OUTPUT_FILE := $(TMPDIR)/libtest-json-output.json
+
+all:
+	$(RUSTC) --test f.rs
+	$(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true
+
+	cat $(OUTPUT_FILE) | "$(PYTHON)" validate_json.py
+
+	# Compare to output file
+	diff output.json $(OUTPUT_FILE)

src/test/run-make/libtest-json/f.rs

+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[test]
+fn a() {
+    // Should pass
+}
+
+#[test]
+fn b() {
+    assert!(false)
+}
+
+#[test]
+#[should_panic]
+fn c() {
+    assert!(false);
+}
+
+#[test]
+#[ignore]
+fn d() {
+    assert!(false);
+}
+

src/test/run-make/libtest-json/output.json

+{ "type": "suite", "event": "started", "test_count": "4" }
+{ "type": "test", "event": "started", "name": "a" }
+{ "type": "test", "name": "a", "event": "ok" }
+{ "type": "test", "event": "started", "name": "b" }
+{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" }
+{ "type": "test", "event": "started", "name": "c" }
+{ "type": "test", "name": "c", "event": "ok" }
+{ "type": "test", "event": "started", "name": "d" }
+{ "type": "test", "name": "d", "event": "ignored" }
+{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" }

src/test/run-make/libtest-json/validate_json.py

+#!/usr/bin/env python
+
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import sys
+import json
+
+# Try to decode line in order to ensure it is a valid JSON document
+for line in sys.stdin:
+    json.loads(line)

cargo @ 1d6dfea4

-Subproject commit 91e36aa86c7037de50642f2fec1cf47c3d18af02
+Subproject commit 1d6dfea44f97199d5d5c177c7dadcde393eaff9a

src/tools/compiletest/src/main.rs

@@ -485,7 +485,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
         filter: config.filter.clone(),
         filter_exact: config.filter_exact,
         run_ignored: config.run_ignored,
-        quiet: config.quiet,
+        format: if config.quiet { test::OutputFormat::Terse } else { test::OutputFormat::Pretty },
         logfile: config.logfile.clone(),
         run_tests: true,
         bench_benchmarks: true,