| // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT |
| // file at the top-level directory of this distribution and at |
| // http://rust-lang.org/COPYRIGHT. |
| // |
| // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
| // option. This file may not be copied, modified, or distributed |
| // except according to those terms. |
| |
| //! Simplified stable-compatible benchmark runner. |
| //! |
| //! Almost all user code will only be interested in `Bencher` and the |
| //! macros that are used to describe benchmarker functions and |
| //! the benchmark runner. |
| //! |
| //! NOTE: There's no proper `black_box` yet in this stable port of the |
| //! benchmark runner, only a workaround implementation. It may not work |
| //! exactly like the upstream `test::black_box`. |
| //! |
| //! One way to use this crate is to use it as dev-dependency and setup |
| //! cargo to compile a file in `benches/` that runs without the testing harness. |
| //! |
| //! In Cargo.toml: |
| //! |
| //! ```ignore |
| //! [[bench]] |
| //! name = "example" |
| //! harness = false |
| //! ``` |
| //! |
| //! In benches/example.rs: |
| //! |
| //! ``` |
| //! #[macro_use] |
| //! extern crate bencher; |
| //! |
| //! use bencher::Bencher; |
| //! |
| //! fn a(bench: &mut Bencher) { |
| //! bench.iter(|| { |
| //! (0..1000).fold(0, |x, y| x + y) |
| //! }) |
| //! } |
| //! |
| //! fn b(bench: &mut Bencher) { |
| //! const N: usize = 1024; |
| //! bench.iter(|| { |
| //! vec![0u8; N] |
| //! }); |
| //! |
| //! bench.bytes = N as u64; |
| //! } |
| //! |
| //! benchmark_group!(benches, a, b); |
| //! benchmark_main!(benches); |
| //! |
| //! # #[cfg(never)] |
| //! # fn main() { } |
| //! ``` |
| //! |
| //! Use `cargo bench` as usual. A command line argument can be used to filter |
| //! which benchmarks to run. |
| |
| pub use self::TestFn::*; |
| use self::TestResult::*; |
| use self::TestEvent::*; |
| use self::NamePadding::*; |
| use self::OutputLocation::*; |
| |
| use std::borrow::Cow; |
| use std::cmp; |
| use std::fmt; |
| use std::fs::File; |
| use std::io::prelude::*; |
| use std::io; |
| use std::iter::repeat; |
| use std::mem::forget; |
| use std::path::PathBuf; |
| use std::ptr; |
| use std::time::{Instant, Duration}; |
| |
| pub mod stats; |
| mod macros; |
| |
| // The name of a test. By convention this follows the rules for rust |
| // paths; i.e. it should be a series of identifiers separated by double |
| // colons. This way if some test runner wants to arrange the tests |
| // hierarchically it may. |
| |
| pub type TestName = Cow<'static, str>; |
| |
| #[derive(Clone, Copy, PartialEq, Eq)] |
| enum NamePadding { |
| PadOnRight, |
| } |
| |
| impl TestDesc { |
| fn padded_name(&self, column_count: usize, align: NamePadding) -> String { |
| let mut name = self.name.to_string(); |
| let fill = column_count.saturating_sub(name.len()); |
| let pad = repeat(" ").take(fill).collect::<String>(); |
| match align { |
| PadOnRight => { |
| name.push_str(&pad); |
| name |
| } |
| } |
| } |
| } |
| |
| /// Represents a benchmark function. |
| pub trait TDynBenchFn: Send { |
| fn run(&self, harness: &mut Bencher); |
| } |
| |
| // A function that runs a test. If the function returns successfully, |
| // the test succeeds; if the function panics then the test fails. We |
| // may need to come up with a more clever definition of test in order |
| // to support isolation of tests into threads. |
| pub enum TestFn { |
| StaticBenchFn(fn(&mut Bencher)), |
| DynBenchFn(Box<TDynBenchFn + 'static>), |
| } |
| |
| impl TestFn { |
| fn padding(&self) -> NamePadding { |
| match *self { |
| StaticBenchFn(..) | |
| DynBenchFn(..) => PadOnRight, |
| } |
| } |
| } |
| |
| impl fmt::Debug for TestFn { |
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
| f.write_str(match *self { |
| StaticBenchFn(..) => "StaticBenchFn(..)", |
| DynBenchFn(..) => "DynBenchFn(..)", |
| }) |
| } |
| } |
| |
| /// Manager of the benchmarking runs. |
| /// |
| /// This is fed into functions marked with `#[bench]` to allow for |
| /// set-up & tear-down before running a piece of code repeatedly via a |
| /// call to `iter`. |
| #[derive(Copy, Clone)] |
| pub struct Bencher { |
| iterations: u64, |
| dur: Duration, |
| pub bytes: u64, |
| } |
| |
| // The definition of a single test. A test runner will run a list of |
| // these. |
| #[derive(Clone, Debug, PartialEq, Eq, Hash)] |
| pub struct TestDesc { |
| pub name: TestName, |
| pub ignore: bool, |
| } |
| |
| #[derive(Clone)] |
| pub struct TestPaths { |
| pub file: PathBuf, // e.g., compile-test/foo/bar/baz.rs |
| pub base: PathBuf, // e.g., compile-test, auxiliary |
| pub relative_dir: PathBuf, // e.g., foo/bar |
| } |
| |
| #[derive(Debug)] |
| pub struct TestDescAndFn { |
| pub desc: TestDesc, |
| pub testfn: TestFn, |
| } |
| |
| #[derive(Default)] |
| pub struct TestOpts { |
| pub filter: Option<String>, |
| pub run_ignored: bool, |
| pub logfile: Option<PathBuf>, |
| pub quiet: bool, |
| pub test_threads: Option<usize>, |
| } |
| |
| #[derive(Clone, PartialEq)] |
| pub struct BenchSamples { |
| ns_iter_summ: stats::Summary, |
| mb_s: usize, |
| } |
| |
| #[derive(Clone, PartialEq)] |
| enum TestResult { |
| TrIgnored, |
| TrBench(BenchSamples), |
| } |
| |
| unsafe impl Send for TestResult {} |
| |
| enum OutputLocation<T> { |
| Raw(T), |
| } |
| |
| struct ConsoleTestState<T> { |
| log_out: Option<File>, |
| out: OutputLocation<T>, |
| quiet: bool, |
| total: usize, |
| passed: usize, |
| failed: usize, |
| ignored: usize, |
| measured: usize, |
| failures: Vec<(TestDesc, Vec<u8>)>, |
| max_name_len: usize, // number of columns to fill when aligning names |
| } |
| |
| impl ConsoleTestState<()> { |
| pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState<io::Stdout>> { |
| let log_out = match opts.logfile { |
| Some(ref path) => Some(try!(File::create(path))), |
| None => None, |
| }; |
| let out = Raw(io::stdout()); |
| |
| Ok(ConsoleTestState { |
| out: out, |
| log_out: log_out, |
| quiet: opts.quiet, |
| total: 0, |
| passed: 0, |
| failed: 0, |
| ignored: 0, |
| measured: 0, |
| failures: Vec::new(), |
| max_name_len: 0, |
| }) |
| } |
| } |
| |
| impl<T: Write> ConsoleTestState<T> { |
| pub fn write_ignored(&mut self) -> io::Result<()> { |
| self.write_short_result("ignored", "i") |
| } |
| |
| pub fn write_bench(&mut self) -> io::Result<()> { |
| self.write_pretty("bench") |
| } |
| |
| pub fn write_short_result(&mut self, verbose: &str, quiet: &str) |
| -> io::Result<()> { |
| if self.quiet { |
| self.write_pretty(quiet) |
| } else { |
| try!(self.write_pretty(verbose)); |
| self.write_plain("\n") |
| } |
| } |
| |
| pub fn write_pretty(&mut self, word: &str) -> io::Result<()> { |
| match self.out { |
| Raw(ref mut stdout) => { |
| try!(stdout.write_all(word.as_bytes())); |
| stdout.flush() |
| } |
| } |
| } |
| |
| pub fn write_plain(&mut self, s: &str) -> io::Result<()> { |
| match self.out { |
| Raw(ref mut stdout) => { |
| try!(stdout.write_all(s.as_bytes())); |
| stdout.flush() |
| } |
| } |
| } |
| |
| pub fn write_run_start(&mut self, len: usize) -> io::Result<()> { |
| self.total = len; |
| let noun = if len != 1 { |
| "tests" |
| } else { |
| "test" |
| }; |
| self.write_plain(&format!("\nrunning {} {}\n", len, noun)) |
| } |
| |
| pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> { |
| if self.quiet && align != PadOnRight { |
| Ok(()) |
| } else { |
| let name = test.padded_name(self.max_name_len, align); |
| self.write_plain(&format!("test {} ... ", name)) |
| } |
| } |
| |
| pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> { |
| match *result { |
| TrIgnored => self.write_ignored(), |
| TrBench(ref bs) => { |
| try!(self.write_bench()); |
| self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) |
| } |
| } |
| } |
| |
| pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> { |
| match self.log_out { |
| None => Ok(()), |
| Some(ref mut o) => { |
| let s = format!("{} {}\n", |
| match *result { |
| TrIgnored => "ignored".to_owned(), |
| TrBench(ref bs) => fmt_bench_samples(bs), |
| }, |
| test.name); |
| o.write_all(s.as_bytes()) |
| } |
| } |
| } |
| |
| pub fn write_failures(&mut self) -> io::Result<()> { |
| try!(self.write_plain("\nfailures:\n")); |
| let mut failures = Vec::new(); |
| let mut fail_out = String::new(); |
| for &(ref f, ref stdout) in &self.failures { |
| failures.push(f.name.to_string()); |
| if !stdout.is_empty() { |
| fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); |
| let output = String::from_utf8_lossy(stdout); |
| fail_out.push_str(&output); |
| fail_out.push_str("\n"); |
| } |
| } |
| if !fail_out.is_empty() { |
| try!(self.write_plain("\n")); |
| try!(self.write_plain(&fail_out)); |
| } |
| |
| try!(self.write_plain("\nfailures:\n")); |
| failures.sort(); |
| for name in &failures { |
| try!(self.write_plain(&format!(" {}\n", name))); |
| } |
| Ok(()) |
| } |
| |
| pub fn write_run_finish(&mut self) -> io::Result<bool> { |
| assert_eq!(self.passed + self.failed + self.ignored + self.measured, self.total); |
| |
| let success = self.failed == 0; |
| if !success { |
| try!(self.write_failures()); |
| } |
| |
| try!(self.write_plain("\ntest result: ")); |
| if success { |
| // There's no parallelism at this point so it's safe to use color |
| try!(self.write_pretty("ok")); |
| } else { |
| try!(self.write_pretty("FAILED")); |
| } |
| let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n", |
| self.passed, |
| self.failed, |
| self.ignored, |
| self.measured); |
| try!(self.write_plain(&s)); |
| Ok(success) |
| } |
| } |
| |
| // Format a number with thousands separators |
| fn fmt_thousands_sep(mut n: usize, sep: char) -> String { |
| use std::fmt::Write; |
| let mut output = String::new(); |
| let mut trailing = false; |
| for &pow in &[9, 6, 3, 0] { |
| let base = 10_usize.pow(pow); |
| if pow == 0 || trailing || n / base != 0 { |
| if !trailing { |
| output.write_fmt(format_args!("{}", n / base)).unwrap(); |
| } else { |
| output.write_fmt(format_args!("{:03}", n / base)).unwrap(); |
| } |
| if pow != 0 { |
| output.push(sep); |
| } |
| trailing = true; |
| } |
| n %= base; |
| } |
| |
| output |
| } |
| |
| pub fn fmt_bench_samples(bs: &BenchSamples) -> String { |
| use std::fmt::Write; |
| let mut output = String::new(); |
| |
| let median = bs.ns_iter_summ.median as usize; |
| let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; |
| |
| output.write_fmt(format_args!("{:>11} ns/iter (+/- {})", |
| fmt_thousands_sep(median, ','), |
| fmt_thousands_sep(deviation, ','))) |
| .unwrap(); |
| if bs.mb_s != 0 { |
| output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap(); |
| } |
| output |
| } |
| |
| // A simple console test runner |
| pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> { |
| |
| fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> { |
| match (*event).clone() { |
| TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()), |
| TeWait(ref test, padding) => st.write_test_start(test, padding), |
| TeResult(test, result, _) => { |
| try!(st.write_log(&test, &result)); |
| try!(st.write_result(&result)); |
| match result { |
| TrIgnored => st.ignored += 1, |
| TrBench(_) => { |
| st.measured += 1 |
| } |
| } |
| Ok(()) |
| } |
| } |
| } |
| |
| let mut st = try!(ConsoleTestState::new(opts)); |
| fn len_if_padded(t: &TestDescAndFn) -> usize { |
| match t.testfn.padding() { |
| PadOnRight => t.desc.name.len(), |
| } |
| } |
| if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { |
| let n = &t.desc.name; |
| st.max_name_len = n.len(); |
| } |
| try!(run_tests(opts, tests, |x| callback(&x, &mut st))); |
| st.write_run_finish() |
| } |
| |
| #[test] |
| fn should_sort_failures_before_printing_them() { |
| let test_a = TestDesc { |
| name: Cow::from("a"), |
| ignore: false, |
| }; |
| |
| let test_b = TestDesc { |
| name: Cow::from("b"), |
| ignore: false, |
| }; |
| |
| let mut st = ConsoleTestState { |
| log_out: None, |
| out: Raw(Vec::new()), |
| quiet: false, |
| total: 0, |
| passed: 0, |
| failed: 0, |
| ignored: 0, |
| measured: 0, |
| max_name_len: 10, |
| failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], |
| }; |
| |
| st.write_failures().unwrap(); |
| let s = match st.out { |
| Raw(ref m) => String::from_utf8_lossy(&m[..]), |
| }; |
| |
| let apos = s.find("a").unwrap(); |
| let bpos = s.find("b").unwrap(); |
| assert!(apos < bpos); |
| } |
| |
| #[derive(Clone)] |
| enum TestEvent { |
| TeFiltered(Vec<TestDesc>), |
| TeWait(TestDesc, NamePadding), |
| TeResult(TestDesc, TestResult, Vec<u8>), |
| } |
| |
| type MonitorMsg = (TestDesc, TestResult, Vec<u8>); |
| |
| |
| fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()> |
| where F: FnMut(TestEvent) -> io::Result<()> |
| { |
| |
| let filtered_tests = filter_tests(opts, tests); |
| |
| let filtered_descs = filtered_tests.iter() |
| .map(|t| t.desc.clone()) |
| .collect(); |
| |
| try!(callback(TeFiltered(filtered_descs))); |
| |
| let filtered_benchs_and_metrics = filtered_tests; |
| |
| // All benchmarks run at the end, in serial. |
| // (this includes metric fns) |
| for b in filtered_benchs_and_metrics { |
| try!(callback(TeWait(b.desc.clone(), b.testfn.padding()))); |
| let (test, result, stdout) = run_test(opts, false, b); |
| try!(callback(TeResult(test, result, stdout))); |
| } |
| Ok(()) |
| } |
| |
| fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> { |
| let mut filtered = tests; |
| |
| // Remove tests that don't match the test filter |
| filtered = match opts.filter { |
| None => filtered, |
| Some(ref filter) => { |
| filtered.into_iter() |
| .filter(|test| test.desc.name.contains(&filter[..])) |
| .collect() |
| } |
| }; |
| |
| // Maybe pull out the ignored test and unignore them |
| filtered = if !opts.run_ignored { |
| filtered |
| } else { |
| fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> { |
| if test.desc.ignore { |
| let TestDescAndFn {desc, testfn} = test; |
| Some(TestDescAndFn { |
| desc: TestDesc { ignore: false, ..desc }, |
| testfn: testfn, |
| }) |
| } else { |
| None |
| } |
| } |
| filtered.into_iter().filter_map(filter).collect() |
| }; |
| |
| // Sort the tests alphabetically |
| filtered.sort_by(|t1, t2| t1.desc.name.cmp(&t2.desc.name)); |
| |
| filtered |
| } |
| |
| fn run_test(_opts: &TestOpts, |
| force_ignore: bool, |
| test: TestDescAndFn) -> MonitorMsg |
| { |
| |
| let TestDescAndFn {desc, testfn} = test; |
| |
| if force_ignore || desc.ignore { |
| return (desc, TrIgnored, Vec::new()); |
| } |
| |
| match testfn { |
| DynBenchFn(bencher) => { |
| let bs = ::bench::benchmark(|harness| bencher.run(harness)); |
| (desc, TrBench(bs), Vec::new()) |
| } |
| StaticBenchFn(benchfn) => { |
| let bs = ::bench::benchmark(|harness| benchfn(harness)); |
| (desc, TrBench(bs), Vec::new()) |
| } |
| } |
| } |
| |
| |
| // Benchmarking |
| |
| // FIXME: We don't have black_box in stable rust |
| |
| /// NOTE: We don't have a proper black box in stable Rust. This is |
| /// a workaround implementation, that may have a too big performance overhead, |
| /// depending on operation, or it may fail to properly avoid having code |
| /// optimized out. It is good enough that it is used by default. |
| /// |
| /// A function that is opaque to the optimizer, to allow benchmarks to |
| /// pretend to use outputs to assist in avoiding dead-code |
| /// elimination. |
| pub fn black_box<T>(dummy: T) -> T { |
| unsafe { |
| let ret = ptr::read_volatile(&dummy); |
| forget(dummy); |
| ret |
| } |
| } |
| |
| |
| impl Bencher { |
| /// Callback for benchmark functions to run in their body. |
| pub fn iter<T, F>(&mut self, mut inner: F) |
| where F: FnMut() -> T |
| { |
| let start = Instant::now(); |
| let k = self.iterations; |
| for _ in 0..k { |
| black_box(inner()); |
| } |
| self.dur = start.elapsed(); |
| } |
| |
| pub fn ns_elapsed(&mut self) -> u64 { |
| self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64) |
| } |
| |
| pub fn ns_per_iter(&mut self) -> u64 { |
| if self.iterations == 0 { |
| 0 |
| } else { |
| self.ns_elapsed() / cmp::max(self.iterations, 1) |
| } |
| } |
| |
| pub fn bench_n<F>(&mut self, n: u64, f: F) |
| where F: FnOnce(&mut Bencher) |
| { |
| self.iterations = n; |
| f(self); |
| } |
| |
| // This is a more statistics-driven benchmark algorithm |
| pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary |
| where F: FnMut(&mut Bencher) |
| { |
| // Initial bench run to get ballpark figure. |
| let mut n = 1; |
| self.bench_n(n, |x| f(x)); |
| |
| // Try to estimate iter count for 1ms falling back to 1m |
| // iterations if first run took < 1ns. |
| if self.ns_per_iter() == 0 { |
| n = 1_000_000; |
| } else { |
| n = 1_000_000 / cmp::max(self.ns_per_iter(), 1); |
| } |
| // if the first run took more than 1ms we don't want to just |
| // be left doing 0 iterations on every loop. The unfortunate |
| // side effect of not being able to do as many runs is |
| // automatically handled by the statistical analysis below |
| // (i.e. larger error bars). |
| if n == 0 { |
| n = 1; |
| } |
| |
| let mut total_run = Duration::new(0, 0); |
| let samples: &mut [f64] = &mut [0.0_f64; 50]; |
| loop { |
| let loop_start = Instant::now(); |
| |
| for p in &mut *samples { |
| self.bench_n(n, |x| f(x)); |
| *p = self.ns_per_iter() as f64; |
| } |
| |
| stats::winsorize(samples, 5.0); |
| let summ = stats::Summary::new(samples); |
| |
| for p in &mut *samples { |
| self.bench_n(5 * n, |x| f(x)); |
| *p = self.ns_per_iter() as f64; |
| } |
| |
| stats::winsorize(samples, 5.0); |
| let summ5 = stats::Summary::new(samples); |
| let loop_run = loop_start.elapsed(); |
| |
| // If we've run for 100ms and seem to have converged to a |
| // stable median. |
| if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 && |
| summ.median - summ5.median < summ5.median_abs_dev { |
| return summ5; |
| } |
| |
| total_run += loop_run; |
| // Longest we ever run for is 3s. |
| if total_run > Duration::from_secs(3) { |
| return summ5; |
| } |
| |
| // If we overflow here just return the results so far. We check a |
| // multiplier of 10 because we're about to multiply by 2 and the |
| // next iteration of the loop will also multiply by 5 (to calculate |
| // the summ5 result) |
| n = match n.checked_mul(10) { |
| Some(_) => n * 2, |
| None => return summ5, |
| }; |
| } |
| } |
| } |
| |
| pub mod bench { |
| use std::cmp; |
| use std::time::Duration; |
| use super::{Bencher, BenchSamples}; |
| |
| pub fn benchmark<F>(f: F) -> BenchSamples |
| where F: FnMut(&mut Bencher) |
| { |
| let mut bs = Bencher { |
| iterations: 0, |
| dur: Duration::new(0, 0), |
| bytes: 0, |
| }; |
| |
| let ns_iter_summ = bs.auto_bench(f); |
| |
| let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); |
| let mb_s = bs.bytes * 1000 / ns_iter; |
| |
| BenchSamples { |
| ns_iter_summ: ns_iter_summ, |
| mb_s: mb_s as usize, |
| } |
| } |
| |
| pub fn run_once<F>(f: F) |
| where F: FnOnce(&mut Bencher) |
| { |
| let mut bs = Bencher { |
| iterations: 0, |
| dur: Duration::new(0, 0), |
| bytes: 0, |
| }; |
| bs.bench_n(1, f); |
| } |
| } |
| |