diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs index 4454e7ed115c23d09c85422209730ee36f1d30bb..08d87b90978960d70617198dbc3fe0ffa246301c 100644 --- a/src/libtest/formatters.rs +++ b/src/libtest/formatters.rs @@ -12,12 +12,12 @@ pub(crate) trait OutputFormatter { fn write_run_start(&mut self, len: usize) -> io::Result<()>; - fn write_test_start(&mut self, - test: &TestDesc, - align: NamePadding, - max_name_len: usize) -> io::Result<()>; + fn write_test_start(&mut self, test: &TestDesc) -> io::Result<()>; fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; - fn write_result(&mut self, result: &TestResult) -> io::Result<()>; + fn write_result(&mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8]) -> io::Result<()>; fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; } @@ -26,15 +26,17 @@ pub(crate) struct HumanFormatter { terse: bool, use_color: bool, test_count: usize, + max_name_len: usize, // number of columns to fill when aligning names } impl HumanFormatter { - pub fn new(out: OutputLocation, use_color: bool, terse: bool) -> Self { + pub fn new(out: OutputLocation, use_color: bool, terse: bool, max_name_len: usize) -> Self { HumanFormatter { out, terse, use_color, test_count: 0, + max_name_len, } } @@ -73,7 +75,7 @@ pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::co // `stamp` in the rust CI). self.write_plain("\n")?; } - + self.test_count += 1; Ok(()) } else { @@ -170,20 +172,18 @@ fn write_run_start(&mut self, len: usize) -> io::Result<()> { self.write_plain(&format!("\nrunning {} {}\n", len, noun)) } - fn write_test_start(&mut self, - test: &TestDesc, - align: NamePadding, - max_name_len: usize) -> io::Result<()> { - if self.terse && align != PadOnRight { - Ok(()) - } - else { - let name = test.padded_name(max_name_len, align); - self.write_plain(&format!("test {} ... ", name)) - } + fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> { + // Do not print header, as priting it at this point will result in + // an unreadable output when running tests concurrently. + Ok(()) } - fn write_result(&mut self, result: &TestResult) -> io::Result<()> { + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + if !(self.terse && desc.name.padding() != PadOnRight) { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + } + match *result { TrOk => self.write_ok(), TrFailed | TrFailedMsg(_) => self.write_failed(), @@ -244,3 +244,203 @@ fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { Ok(success) } } + +pub(crate) struct JsonFormatter { + out: OutputLocation +} + +impl JsonFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { + out, } + } + + fn write_str>(&mut self, s: S) -> io::Result<()> { + self.out.write_all(s.as_ref().as_ref())?; + self.out.write_all("\n".as_ref()) + } + + fn write_event(&mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option) -> io::Result<()> { + if let Some(extras) = extra { + self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, + name, + evt, + extras)) + } + else { + self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, + name, + evt)) + } + } +} + +impl OutputFormatter for JsonFormatter { + fn write_run_start(&mut self, len: usize) -> io::Result<()> { + self.write_str( + &*format!(r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, len)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_str(&*format!(r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name)) + } + + fn write_result(&mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8]) -> io::Result<()> { + match *result { + TrOk => { + self.write_event("test", desc.name.as_slice(), "ok", None) + }, + + TrFailed => { + let extra_data = if stdout.len() > 0 { + Some(format!(r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)))) + } + else { + None + }; + + self.write_event("test", desc.name.as_slice(), "failed", extra_data) + }, + + TrFailedMsg(ref m) => { + self.write_event("test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m)))) + }, + + TrIgnored => { + self.write_event("test", desc.name.as_slice(), "ignored", None) + }, + + TrAllowedFail => { + self.write_event("test", desc.name.as_slice(), "allowed_failure", None) + }, + + TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + "".into() + } + else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!("{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + desc.name, + median, + deviation, + mbps); + + self.write_str(&*line) + }, + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_str(&*format!(r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name)) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + + self.write_str(&*format!("{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"allowed_fail\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": \"{}\" }}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out))?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString>(S); + +impl> ::std::fmt::Display for EscapedString { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { continue; } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 2fb62c832f8024618be6606b40a82faeb5fe0ebd..b11f783770f838219ea2cf77fa7695876e56923d 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -71,6 +71,7 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Instant, Duration}; +use std::borrow::Cow; const TEST_WARN_TIMEOUT_S: u64 = 60; const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode @@ -97,14 +98,33 @@ pub mod test { pub enum TestName { StaticTestName(&'static str), DynTestName(String), + AlignedTestName(Cow<'static, str>, NamePadding), } impl TestName { fn as_slice(&self) -> &str { match *self { StaticTestName(s) => s, DynTestName(ref s) => s, + AlignedTestName(ref s, _) => &*s, } } + + fn padding(&self) -> NamePadding { + match self { + &AlignedTestName(_, p) => p, + _ => PadNone, + } + } + + fn with_padding(&self, padding: NamePadding) -> TestName { + let name = match self { + &TestName::StaticTestName(name) => Cow::Borrowed(name), + &TestName::DynTestName(ref name) => Cow::Owned(name.clone()), + &TestName::AlignedTestName(ref name, _) => name.clone(), + }; + + TestName::AlignedTestName(name, padding) + } } impl fmt::Display for TestName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -112,7 +132,7 @@ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { } } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum NamePadding { PadNone, PadOnRight, @@ -306,6 +326,13 @@ pub enum ColorConfig { NeverColor, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum OutputFormat { + Pretty, + Terse, + Json +} + #[derive(Debug)] pub struct TestOpts { pub list: bool, @@ -317,7 +344,7 @@ pub struct TestOpts { pub logfile: Option, pub nocapture: bool, pub color: ColorConfig, - pub quiet: bool, + pub format: OutputFormat, pub test_threads: Option, pub skip: Vec, pub options: Options, @@ -336,7 +363,7 @@ fn new() -> TestOpts { logfile: None, nocapture: false, color: AutoColor, - quiet: false, + format: OutputFormat::Pretty, test_threads: None, skip: vec![], options: Options::new(), @@ -362,13 +389,17 @@ fn optgroups() -> getopts::Options { in parallel", "n_threads") .optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ be used multiple times)","FILTER") - .optflag("q", "quiet", "Display one character per test instead of one line.\ - Equivalent to --format=terse") + .optflag("q", "quiet", "Display one character per test instead of one line. \ + Alias to --format=terse") .optflag("", "exact", "Exactly match filters rather than by substring") .optopt("", "color", "Configure coloring of output: auto = colorize if stdout is a tty and tests are run on serially (default); always = always colorize output; - never = never colorize output;", "auto|always|never"); + never = never colorize output;", "auto|always|never") + .optopt("", "format", "Configure formatting of output: + pretty = Print verbose output; + terse = Display one character per test; + json = Output a json document", "pretty|terse|json"); return opts } @@ -469,6 +500,19 @@ pub fn parse_opts(args: &[String]) -> Option { } }; + let format = match matches.opt_str("format").as_ref().map(|s| &**s) { + None if quiet => OutputFormat::Terse, + Some("pretty") | None => OutputFormat::Pretty, + Some("terse") => OutputFormat::Terse, + Some("json") => OutputFormat::Json, + + Some(v) => { + return Some(Err(format!("argument for --format must be pretty, terse, or json (was \ + {})", + v))) + } + }; + let test_opts = TestOpts { list, filter, @@ -479,7 +523,7 @@ pub fn parse_opts(args: &[String]) -> Option { logfile, nocapture, color, - quiet, + format, test_threads, skip: matches.opt_strs("skip"), options: Options::new(), @@ -539,7 +583,6 @@ struct ConsoleTestState { metrics: MetricMap, failures: Vec<(TestDesc, Vec)>, not_failures: Vec<(TestDesc, Vec)>, - max_name_len: usize, // number of columns to fill when aligning names options: Options, } @@ -562,7 +605,6 @@ pub fn new(opts: &TestOpts) -> io::Result { metrics: MetricMap::new(), failures: Vec::new(), not_failures: Vec::new(), - max_name_len: 0, options: opts.options, }) } @@ -641,7 +683,9 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res None => Raw(io::stdout()), Some(t) => Pretty(t), }; - let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + + let quiet = opts.format == OutputFormat::Terse; + let mut out = HumanFormatter::new(output, use_color(opts), quiet, 0); let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; @@ -668,11 +712,11 @@ fn plural(count: u32, s: &str) -> String { } } - if !opts.quiet { + if !quiet { if ntest != 0 || nbench != 0 { - st.write_plain("\n")?; + out.write_plain("\n")?; } - st.write_plain(format!("{}, {}\n", + out.write_plain(format!("{}, {}\n", plural(ntest, "test"), plural(nbench, "benchmark")))?; } @@ -682,6 +726,14 @@ fn plural(count: u32, s: &str) -> String { // A simple console test runner pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { + let tests = { + let mut tests = tests; + for test in tests.iter_mut() { + test.desc.name = test.desc.name.with_padding(test.testfn.padding()); + } + + tests + }; fn callback(event: &TestEvent, st: &mut ConsoleTestState, @@ -693,11 +745,11 @@ fn callback(event: &TestEvent, out.write_run_start(filtered_tests.len()) }, TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), - TeWait(ref test, padding) => out.write_test_start(test, padding, st.max_name_len), + TeWait(ref test) => out.write_test_start(test), TeTimeout(ref test) => out.write_timeout(test), TeResult(test, result, stdout) => { st.write_log_result(&test, &result)?; - out.write_result(&result)?; + out.write_result(&test, &result, &*stdout)?; match result { TrOk => { st.passed += 1; @@ -734,8 +786,25 @@ fn callback(event: &TestEvent, Some(t) => Pretty(t), }; - let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + let max_name_len = if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { + let n = t.desc.name.as_slice(); + n.len() + } + else { + 0 + }; + let mut out: Box = match opts.format { + OutputFormat::Pretty => Box::new(HumanFormatter::new(output, + use_color(opts), + false, + max_name_len)), + OutputFormat::Terse => Box::new(HumanFormatter::new(output, + use_color(opts), + true, + max_name_len)), + OutputFormat::Json => Box::new(JsonFormatter::new(output)), + }; let mut st = ConsoleTestState::new(opts)?; fn len_if_padded(t: &TestDescAndFn) -> usize { match t.testfn.padding() { @@ -743,11 +812,8 @@ fn len_if_padded(t: &TestDescAndFn) -> usize { PadOnRight => t.desc.name.as_slice().len(), } } - if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { - let n = t.desc.name.as_slice(); - st.max_name_len = n.len(); - } - run_tests(opts, tests, |x| callback(&x, &mut st, &mut out))?; + + run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?; assert!(st.current_test_count() == st.total); @@ -770,7 +836,7 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut out = HumanFormatter::new(Raw(Vec::new()), false, false); + let mut out = HumanFormatter::new(Raw(Vec::new()), false, false, 10); let st = ConsoleTestState { log_out: None, @@ -781,7 +847,6 @@ fn should_sort_failures_before_printing_them() { allowed_fail: 0, filtered_out: 0, measured: 0, - max_name_len: 10, metrics: MetricMap::new(), failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], options: Options::new(), @@ -839,7 +904,7 @@ fn stdout_isatty() -> bool { #[derive(Clone)] pub enum TestEvent { TeFiltered(Vec), - TeWait(TestDesc, NamePadding), + TeWait(TestDesc), TeResult(TestDesc, TestResult, Vec), TeTimeout(TestDesc), TeFilteredOut(usize), @@ -915,7 +980,7 @@ fn calc_timeout(running_tests: &HashMap) -> Option if concurrency == 1 { while !remaining.is_empty() { let test = remaining.pop().unwrap(); - callback(TeWait(test.desc.clone(), test.testfn.padding()))?; + callback(TeWait(test.desc.clone()))?; run_test(opts, !opts.run_tests, test, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -926,6 +991,7 @@ fn calc_timeout(running_tests: &HashMap) -> Option let test = remaining.pop().unwrap(); let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); running_tests.insert(test.desc.clone(), timeout); + callback(TeWait(test.desc.clone()))?; //here no pad run_test(opts, !opts.run_tests, test, tx.clone()); pending += 1; } @@ -949,7 +1015,6 @@ fn calc_timeout(running_tests: &HashMap) -> Option let (desc, result, stdout) = res.unwrap(); running_tests.remove(&desc); - callback(TeWait(desc.clone(), PadNone))?; callback(TeResult(desc, result, stdout))?; pending -= 1; } @@ -958,7 +1023,7 @@ fn calc_timeout(running_tests: &HashMap) -> Option if opts.bench_benchmarks { // All benchmarks run at the end, in serial. for b in filtered_benchs { - callback(TeWait(b.desc.clone(), b.testfn.padding()))?; + callback(TeWait(b.desc.clone()))?; run_test(opts, false, b, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -1239,10 +1304,7 @@ fn flush(&mut self) -> io::Result<()> { !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32"); if supports_threads { - let cfg = thread::Builder::new().name(match name { - DynTestName(ref name) => name.clone(), - StaticTestName(name) => name.to_owned(), - }); + let cfg = thread::Builder::new().name(name.as_slice().to_owned()); cfg.spawn(runtest).unwrap(); } else { runtest(); diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 533aaf9cd27353bf9e0b78eab246f8df8728da72..1c52ebd7dc54e4fed51141cef0ae0539f62f1c42 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -485,7 +485,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts { filter: config.filter.clone(), filter_exact: config.filter_exact, run_ignored: config.run_ignored, - quiet: config.quiet, + format: if config.quiet { test::OutputFormat::Terse } else { test::OutputFormat::Pretty }, logfile: config.logfile.clone(), run_tests: true, bench_benchmarks: true,