Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 1 | #include <cstdio> |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 2 | #include <cstring> |
Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 3 | #include <fstream> |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 4 | #include <iostream> |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 5 | #include <map> |
| 6 | #include <memory> |
Eric Fiselier | 114125f | 2018-12-14 03:37:13 +0000 | [diff] [blame] | 7 | #include <random> |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 8 | #include <sstream> |
Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 9 | #include <streambuf> |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 10 | |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 11 | #include "../src/benchmark_api_internal.h" |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 12 | #include "../src/check.h" // NOTE: check.h is for internal use only! |
| 13 | #include "../src/re.h" // NOTE: re.h is for internal use only |
| 14 | #include "output_test.h" |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 15 | |
| 16 | // ========================================================================= // |
| 17 | // ------------------------------ Internals -------------------------------- // |
| 18 | // ========================================================================= // |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 19 | namespace internal { |
| 20 | namespace { |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 21 | |
| 22 | using TestCaseList = std::vector<TestCase>; |
| 23 | |
| 24 | // Use a vector because the order elements are added matters during iteration. |
| 25 | // std::map/unordered_map don't guarantee that. |
| 26 | // For example: |
| 27 | // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); |
| 28 | // Substitute("%HelloWorld") // Always expands to Hello. |
| 29 | using SubMap = std::vector<std::pair<std::string, std::string>>; |
| 30 | |
| 31 | TestCaseList& GetTestCaseList(TestCaseID ID) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 32 | // Uses function-local statics to ensure initialization occurs |
| 33 | // before first use. |
| 34 | static TestCaseList lists[TC_NumID]; |
| 35 | return lists[ID]; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 36 | } |
| 37 | |
| 38 | SubMap& GetSubstitutions() { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 39 | // Don't use 'dec_re' from header because it may not yet be initialized. |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 40 | // clang-format off |
Eric Fiselier | 688edc7 | 2017-04-18 07:17:20 +0000 | [diff] [blame] | 41 | static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; |
Eric Fiselier | ea9230c | 2018-12-14 03:48:09 +0000 | [diff] [blame^] | 42 | static std::string time_re = "([0-9]+[.])?[0-9]+"; |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 43 | static SubMap map = { |
| 44 | {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 45 | // human-readable float |
| 46 | {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 47 | {"%int", "[ ]*[0-9]+"}, |
| 48 | {" %s ", "[ ]+"}, |
Eric Fiselier | ea9230c | 2018-12-14 03:48:09 +0000 | [diff] [blame^] | 49 | {"%time", "[ ]*" + time_re + "[ ]+ns"}, |
| 50 | {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, |
| 51 | {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, |
| 52 | {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, |
| 53 | {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 54 | {"%csv_header", |
| 55 | "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," |
| 56 | "items_per_second,label,error_occurred,error_message"}, |
Eric Fiselier | 688edc7 | 2017-04-18 07:17:20 +0000 | [diff] [blame] | 57 | {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, |
| 58 | {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 59 | {"%csv_bytes_report", |
Eric Fiselier | 688edc7 | 2017-04-18 07:17:20 +0000 | [diff] [blame] | 60 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 61 | {"%csv_items_report", |
Eric Fiselier | 688edc7 | 2017-04-18 07:17:20 +0000 | [diff] [blame] | 62 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 63 | {"%csv_bytes_items_report", |
| 64 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + |
| 65 | "," + safe_dec_re + ",,,"}, |
Eric Fiselier | 688edc7 | 2017-04-18 07:17:20 +0000 | [diff] [blame] | 66 | {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 67 | {"%csv_label_report_end", ",,"}}; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 68 | // clang-format on |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 69 | return map; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 70 | } |
| 71 | |
| 72 | std::string PerformSubstitutions(std::string source) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 73 | SubMap const& subs = GetSubstitutions(); |
| 74 | using SizeT = std::string::size_type; |
| 75 | for (auto const& KV : subs) { |
| 76 | SizeT pos; |
| 77 | SizeT next_start = 0; |
| 78 | while ((pos = source.find(KV.first, next_start)) != std::string::npos) { |
| 79 | next_start = pos + KV.second.size(); |
| 80 | source.replace(pos, KV.first.size(), KV.second); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 81 | } |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 82 | } |
| 83 | return source; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | void CheckCase(std::stringstream& remaining_output, TestCase const& TC, |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 87 | TestCaseList const& not_checks) { |
| 88 | std::string first_line; |
| 89 | bool on_first = true; |
| 90 | std::string line; |
| 91 | while (remaining_output.eof() == false) { |
| 92 | CHECK(remaining_output.good()); |
| 93 | std::getline(remaining_output, line); |
| 94 | if (on_first) { |
| 95 | first_line = line; |
| 96 | on_first = false; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 97 | } |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 98 | for (const auto& NC : not_checks) { |
| 99 | CHECK(!NC.regex->Match(line)) |
| 100 | << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" |
| 101 | << NC.regex_str << "\"" |
| 102 | << "\n actual regex string \"" << TC.substituted_regex << "\"" |
| 103 | << "\n started matching near: " << first_line; |
| 104 | } |
| 105 | if (TC.regex->Match(line)) return; |
| 106 | CHECK(TC.match_rule != MR_Next) |
| 107 | << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str |
| 108 | << "\"" |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 109 | << "\n actual regex string \"" << TC.substituted_regex << "\"" |
| 110 | << "\n started matching near: " << first_line; |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 111 | } |
| 112 | CHECK(remaining_output.eof() == false) |
| 113 | << "End of output reached before match for regex \"" << TC.regex_str |
| 114 | << "\" was found" |
| 115 | << "\n actual regex string \"" << TC.substituted_regex << "\"" |
| 116 | << "\n started matching near: " << first_line; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 117 | } |
| 118 | |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 119 | void CheckCases(TestCaseList const& checks, std::stringstream& output) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 120 | std::vector<TestCase> not_checks; |
| 121 | for (size_t i = 0; i < checks.size(); ++i) { |
| 122 | const auto& TC = checks[i]; |
| 123 | if (TC.match_rule == MR_Not) { |
| 124 | not_checks.push_back(TC); |
| 125 | continue; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 126 | } |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 127 | CheckCase(output, TC, not_checks); |
| 128 | not_checks.clear(); |
| 129 | } |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 130 | } |
| 131 | |
| 132 | class TestReporter : public benchmark::BenchmarkReporter { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 133 | public: |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 134 | TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 135 | : reporters_(reps) {} |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 136 | |
| 137 | virtual bool ReportContext(const Context& context) { |
| 138 | bool last_ret = false; |
| 139 | bool first = true; |
| 140 | for (auto rep : reporters_) { |
| 141 | bool new_ret = rep->ReportContext(context); |
| 142 | CHECK(first || new_ret == last_ret) |
| 143 | << "Reports return different values for ReportContext"; |
| 144 | first = false; |
| 145 | last_ret = new_ret; |
| 146 | } |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 147 | (void)first; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 148 | return last_ret; |
| 149 | } |
| 150 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 151 | void ReportRuns(const std::vector<Run>& report) { |
| 152 | for (auto rep : reporters_) rep->ReportRuns(report); |
| 153 | } |
| 154 | void Finalize() { |
| 155 | for (auto rep : reporters_) rep->Finalize(); |
| 156 | } |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 157 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 158 | private: |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 159 | std::vector<benchmark::BenchmarkReporter*> reporters_; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 160 | }; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 161 | } // namespace |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 162 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 163 | } // end namespace internal |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 164 | |
| 165 | // ========================================================================= // |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 166 | // -------------------------- Results checking ----------------------------- // |
| 167 | // ========================================================================= // |
| 168 | |
| 169 | namespace internal { |
| 170 | |
| 171 | // Utility class to manage subscribers for checking benchmark results. |
| 172 | // It works by parsing the CSV output to read the results. |
| 173 | class ResultsChecker { |
| 174 | public: |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 175 | struct PatternAndFn : public TestCase { // reusing TestCase for its regexes |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 176 | PatternAndFn(const std::string& rx, ResultsCheckFn fn_) |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 177 | : TestCase(rx), fn(fn_) {} |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 178 | ResultsCheckFn fn; |
| 179 | }; |
| 180 | |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 181 | std::vector<PatternAndFn> check_patterns; |
| 182 | std::vector<Results> results; |
| 183 | std::vector<std::string> field_names; |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 184 | |
| 185 | void Add(const std::string& entry_pattern, ResultsCheckFn fn); |
| 186 | |
| 187 | void CheckResults(std::stringstream& output); |
| 188 | |
| 189 | private: |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 190 | void SetHeader_(const std::string& csv_header); |
| 191 | void SetValues_(const std::string& entry_csv_line); |
| 192 | |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 193 | std::vector<std::string> SplitCsv_(const std::string& line); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 194 | }; |
| 195 | |
| 196 | // store the static ResultsChecker in a function to prevent initialization |
| 197 | // order problems |
| 198 | ResultsChecker& GetResultsChecker() { |
| 199 | static ResultsChecker rc; |
| 200 | return rc; |
| 201 | } |
| 202 | |
| 203 | // add a results checker for a benchmark |
| 204 | void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { |
| 205 | check_patterns.emplace_back(entry_pattern, fn); |
| 206 | } |
| 207 | |
| 208 | // check the results of all subscribed benchmarks |
| 209 | void ResultsChecker::CheckResults(std::stringstream& output) { |
| 210 | // first reset the stream to the start |
| 211 | { |
Eric Fiselier | 114125f | 2018-12-14 03:37:13 +0000 | [diff] [blame] | 212 | auto start = std::stringstream::pos_type(0); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 213 | // clear before calling tellg() |
| 214 | output.clear(); |
| 215 | // seek to zero only when needed |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 216 | if (output.tellg() > start) output.seekg(start); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 217 | // and just in case |
| 218 | output.clear(); |
| 219 | } |
| 220 | // now go over every line and publish it to the ResultsChecker |
| 221 | std::string line; |
| 222 | bool on_first = true; |
| 223 | while (output.eof() == false) { |
| 224 | CHECK(output.good()); |
| 225 | std::getline(output, line); |
| 226 | if (on_first) { |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 227 | SetHeader_(line); // this is important |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 228 | on_first = false; |
| 229 | continue; |
| 230 | } |
| 231 | SetValues_(line); |
| 232 | } |
| 233 | // finally we can call the subscribed check functions |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 234 | for (const auto& p : check_patterns) { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 235 | VLOG(2) << "--------------------------------\n"; |
| 236 | VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 237 | for (const auto& r : results) { |
| 238 | if (!p.regex->Match(r.name)) { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 239 | VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; |
| 240 | continue; |
| 241 | } else { |
| 242 | VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; |
| 243 | } |
| 244 | VLOG(1) << "Checking results of " << r.name << ": ... \n"; |
| 245 | p.fn(r); |
| 246 | VLOG(1) << "Checking results of " << r.name << ": OK.\n"; |
| 247 | } |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | // prepare for the names in this header |
| 252 | void ResultsChecker::SetHeader_(const std::string& csv_header) { |
| 253 | field_names = SplitCsv_(csv_header); |
| 254 | } |
| 255 | |
| 256 | // set the values for a benchmark |
| 257 | void ResultsChecker::SetValues_(const std::string& entry_csv_line) { |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 258 | if (entry_csv_line.empty()) return; // some lines are empty |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 259 | CHECK(!field_names.empty()); |
| 260 | auto vals = SplitCsv_(entry_csv_line); |
| 261 | CHECK_EQ(vals.size(), field_names.size()); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 262 | results.emplace_back(vals[0]); // vals[0] is the benchmark name |
| 263 | auto& entry = results.back(); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 264 | for (size_t i = 1, e = vals.size(); i < e; ++i) { |
| 265 | entry.values[field_names[i]] = vals[i]; |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | // a quick'n'dirty csv splitter (eliminating quotes) |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 270 | std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) { |
| 271 | std::vector<std::string> out; |
| 272 | if (line.empty()) return out; |
| 273 | if (!field_names.empty()) out.reserve(field_names.size()); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 274 | size_t prev = 0, pos = line.find_first_of(','), curr = pos; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 275 | while (pos != line.npos) { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 276 | CHECK(curr > 0); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 277 | if (line[prev] == '"') ++prev; |
| 278 | if (line[curr - 1] == '"') --curr; |
| 279 | out.push_back(line.substr(prev, curr - prev)); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 280 | prev = pos + 1; |
| 281 | pos = line.find_first_of(',', pos + 1); |
| 282 | curr = pos; |
| 283 | } |
| 284 | curr = line.size(); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 285 | if (line[prev] == '"') ++prev; |
| 286 | if (line[curr - 1] == '"') --curr; |
| 287 | out.push_back(line.substr(prev, curr - prev)); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 288 | return out; |
| 289 | } |
| 290 | |
| 291 | } // end namespace internal |
| 292 | |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 293 | size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { |
| 294 | auto& rc = internal::GetResultsChecker(); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 295 | rc.Add(bm_name, fn); |
| 296 | return rc.results.size(); |
| 297 | } |
| 298 | |
| 299 | int Results::NumThreads() const { |
| 300 | auto pos = name.find("/threads:"); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 301 | if (pos == name.npos) return 1; |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 302 | auto end = name.find('/', pos + 9); |
| 303 | std::stringstream ss; |
| 304 | ss << name.substr(pos + 9, end); |
| 305 | int num = 1; |
| 306 | ss >> num; |
| 307 | CHECK(!ss.fail()); |
| 308 | return num; |
| 309 | } |
| 310 | |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 311 | double Results::NumIterations() const { |
| 312 | return GetAs<double>("iterations"); |
| 313 | } |
| 314 | |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 315 | double Results::GetTime(BenchmarkTime which) const { |
| 316 | CHECK(which == kCpuTime || which == kRealTime); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 317 | const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; |
| 318 | double val = GetAs<double>(which_str); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 319 | auto unit = Get("time_unit"); |
| 320 | CHECK(unit); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 321 | if (*unit == "ns") { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 322 | return val * 1.e-9; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 323 | } else if (*unit == "us") { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 324 | return val * 1.e-6; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 325 | } else if (*unit == "ms") { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 326 | return val * 1.e-3; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 327 | } else if (*unit == "s") { |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 328 | return val; |
| 329 | } else { |
| 330 | CHECK(1 == 0) << "unknown time unit: " << *unit; |
| 331 | return 0; |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | // ========================================================================= // |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 336 | // -------------------------- Public API Definitions------------------------ // |
| 337 | // ========================================================================= // |
| 338 | |
| 339 | TestCase::TestCase(std::string re, int rule) |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 340 | : regex_str(std::move(re)), |
| 341 | match_rule(rule), |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 342 | substituted_regex(internal::PerformSubstitutions(regex_str)), |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 343 | regex(std::make_shared<benchmark::Regex>()) { |
| 344 | std::string err_str; |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 345 | regex->Init(substituted_regex, &err_str); |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 346 | CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex |
| 347 | << "\"" |
| 348 | << "\n originally \"" << regex_str << "\"" |
| 349 | << "\n got error: " << err_str; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 350 | } |
| 351 | |
| 352 | int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 353 | auto& L = internal::GetTestCaseList(ID); |
| 354 | L.insert(L.end(), il); |
| 355 | return 0; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 356 | } |
| 357 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 358 | int SetSubstitutions( |
| 359 | std::initializer_list<std::pair<std::string, std::string>> il) { |
| 360 | auto& subs = internal::GetSubstitutions(); |
| 361 | for (auto KV : il) { |
| 362 | bool exists = false; |
| 363 | KV.second = internal::PerformSubstitutions(KV.second); |
| 364 | for (auto& EKV : subs) { |
| 365 | if (EKV.first == KV.first) { |
| 366 | EKV.second = std::move(KV.second); |
| 367 | exists = true; |
| 368 | break; |
| 369 | } |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 370 | } |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 371 | if (!exists) subs.push_back(std::move(KV)); |
| 372 | } |
| 373 | return 0; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 374 | } |
| 375 | |
| 376 | void RunOutputTests(int argc, char* argv[]) { |
| 377 | using internal::GetTestCaseList; |
| 378 | benchmark::Initialize(&argc, argv); |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 379 | auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true); |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 380 | benchmark::ConsoleReporter CR(options); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 381 | benchmark::JSONReporter JR; |
| 382 | benchmark::CSVReporter CSVR; |
| 383 | struct ReporterTest { |
| 384 | const char* name; |
| 385 | std::vector<TestCase>& output_cases; |
| 386 | std::vector<TestCase>& error_cases; |
| 387 | benchmark::BenchmarkReporter& reporter; |
| 388 | std::stringstream out_stream; |
| 389 | std::stringstream err_stream; |
| 390 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 391 | ReporterTest(const char* n, std::vector<TestCase>& out_tc, |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 392 | std::vector<TestCase>& err_tc, |
| 393 | benchmark::BenchmarkReporter& br) |
| 394 | : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 395 | reporter.SetOutputStream(&out_stream); |
| 396 | reporter.SetErrorStream(&err_stream); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 397 | } |
| 398 | } TestCases[] = { |
| 399 | {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 400 | GetTestCaseList(TC_ConsoleErr), CR}, |
| 401 | {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), |
| 402 | JR}, |
| 403 | {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), |
| 404 | CSVR}, |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 405 | }; |
| 406 | |
| 407 | // Create the test reporter and run the benchmarks. |
| 408 | std::cout << "Running benchmarks...\n"; |
| 409 | internal::TestReporter test_rep({&CR, &JR, &CSVR}); |
| 410 | benchmark::RunSpecifiedBenchmarks(&test_rep); |
| 411 | |
| 412 | for (auto& rep_test : TestCases) { |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 413 | std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; |
| 414 | std::string banner(msg.size() - 1, '-'); |
| 415 | std::cout << banner << msg << banner << "\n"; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 416 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 417 | std::cerr << rep_test.err_stream.str(); |
| 418 | std::cout << rep_test.out_stream.str(); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 419 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 420 | internal::CheckCases(rep_test.error_cases, rep_test.err_stream); |
| 421 | internal::CheckCases(rep_test.output_cases, rep_test.out_stream); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 422 | |
Eric Fiselier | d87eb99 | 2016-11-05 00:30:27 +0000 | [diff] [blame] | 423 | std::cout << "\n"; |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 424 | } |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 425 | |
| 426 | // now that we know the output is as expected, we can dispatch |
| 427 | // the checks to subscribees. |
Eric Fiselier | ffc31db | 2018-07-10 04:02:00 +0000 | [diff] [blame] | 428 | auto& csv = TestCases[2]; |
Eric Fiselier | fd2e3e9 | 2018-01-18 04:23:01 +0000 | [diff] [blame] | 429 | // would use == but gcc spits a warning |
| 430 | CHECK(std::strcmp(csv.name, "CSVReporter") == 0); |
| 431 | internal::GetResultsChecker().CheckResults(csv.out_stream); |
Eric Fiselier | f76a087 | 2016-08-29 19:12:01 +0000 | [diff] [blame] | 432 | } |
Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 433 | |
| 434 | int SubstrCnt(const std::string& haystack, const std::string& pat) { |
| 435 | if (pat.length() == 0) return 0; |
| 436 | int count = 0; |
| 437 | for (size_t offset = haystack.find(pat); offset != std::string::npos; |
| 438 | offset = haystack.find(pat, offset + pat.length())) |
| 439 | ++count; |
| 440 | return count; |
| 441 | } |
| 442 | |
Eric Fiselier | 114125f | 2018-12-14 03:37:13 +0000 | [diff] [blame] | 443 | static char ToHex(int ch) { |
| 444 | return ch < 10 ? static_cast<char>('0' + ch) |
| 445 | : static_cast<char>('a' + (ch - 10)); |
| 446 | } |
| 447 | |
| 448 | static char RandomHexChar() { |
| 449 | static std::mt19937 rd{std::random_device{}()}; |
| 450 | static std::uniform_int_distribution<int> mrand{0, 15}; |
| 451 | return ToHex(mrand(rd)); |
| 452 | } |
| 453 | |
| 454 | static std::string GetRandomFileName() { |
| 455 | std::string model = "test.%%%%%%"; |
| 456 | for (auto & ch : model) { |
| 457 | if (ch == '%') |
| 458 | ch = RandomHexChar(); |
| 459 | } |
| 460 | return model; |
| 461 | } |
| 462 | |
| 463 | static bool FileExists(std::string const& name) { |
| 464 | std::ifstream in(name.c_str()); |
| 465 | return in.good(); |
| 466 | } |
| 467 | |
| 468 | static std::string GetTempFileName() { |
| 469 | // This function attempts to avoid race conditions where two tests |
| 470 | // create the same file at the same time. However, it still introduces races |
| 471 | // similar to tmpnam. |
| 472 | int retries = 3; |
| 473 | while (--retries) { |
| 474 | std::string name = GetRandomFileName(); |
| 475 | if (!FileExists(name)) |
| 476 | return name; |
| 477 | } |
| 478 | std::cerr << "Failed to create unique temporary file name" << std::endl; |
| 479 | std::abort(); |
| 480 | } |
| 481 | |
Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 482 | std::string GetFileReporterOutput(int argc, char* argv[]) { |
| 483 | std::vector<char*> new_argv(argv, argv + argc); |
| 484 | assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size()); |
| 485 | |
Eric Fiselier | 114125f | 2018-12-14 03:37:13 +0000 | [diff] [blame] | 486 | std::string tmp_file_name = GetTempFileName(); |
Eric Fiselier | 622fc11 | 2018-11-15 19:22:53 +0000 | [diff] [blame] | 487 | std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; |
| 488 | |
| 489 | std::string tmp = "--benchmark_out="; |
| 490 | tmp += tmp_file_name; |
| 491 | new_argv.emplace_back(const_cast<char*>(tmp.c_str())); |
| 492 | |
| 493 | argc = int(new_argv.size()); |
| 494 | |
| 495 | benchmark::Initialize(&argc, new_argv.data()); |
| 496 | benchmark::RunSpecifiedBenchmarks(); |
| 497 | |
| 498 | // Read the output back from the file, and delete the file. |
| 499 | std::ifstream tmp_stream(tmp_file_name); |
| 500 | std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)), |
| 501 | std::istreambuf_iterator<char>()); |
| 502 | std::remove(tmp_file_name.c_str()); |
| 503 | |
| 504 | return output; |
| 505 | } |