Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 1 | //=-- InstrProfReader.cpp - Instrumented profiling reader -------------------=// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file contains support for reading profiling data for clang's |
| 11 | // instrumentation based PGO and coverage. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "llvm/ProfileData/InstrProfReader.h" |
Benjamin Kramer | 0a446fd | 2015-03-01 21:28:53 +0000 | [diff] [blame] | 16 | #include "llvm/ADT/STLExtras.h" |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 17 | #include <cassert> |
| 18 | |
| 19 | using namespace llvm; |
| 20 | |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 21 | static ErrorOr<std::unique_ptr<MemoryBuffer>> |
| 22 | setupMemoryBuffer(std::string Path) { |
Rafael Espindola | adf21f2 | 2014-07-06 17:43:13 +0000 | [diff] [blame] | 23 | ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = |
| 24 | MemoryBuffer::getFileOrSTDIN(Path); |
| 25 | if (std::error_code EC = BufferOrErr.getError()) |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 26 | return EC; |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 27 | return std::move(BufferOrErr.get()); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 28 | } |
| 29 | |
Rafael Espindola | db4ed0b | 2014-06-13 02:24:39 +0000 | [diff] [blame] | 30 | static std::error_code initializeReader(InstrProfReader &Reader) { |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 31 | return Reader.readHeader(); |
| 32 | } |
| 33 | |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 34 | ErrorOr<std::unique_ptr<InstrProfReader>> |
| 35 | InstrProfReader::create(std::string Path) { |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 36 | // Set up the buffer to read. |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 37 | auto BufferOrError = setupMemoryBuffer(Path); |
| 38 | if (std::error_code EC = BufferOrError.getError()) |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 39 | return EC; |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 40 | return InstrProfReader::create(std::move(BufferOrError.get())); |
| 41 | } |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 42 | |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 43 | ErrorOr<std::unique_ptr<InstrProfReader>> |
| 44 | InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { |
| 45 | // Sanity check the buffer. |
| 46 | if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) |
| 47 | return instrprof_error::too_large; |
| 48 | |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 49 | std::unique_ptr<InstrProfReader> Result; |
Duncan P. N. Exon Smith | 09a67f4 | 2014-03-21 20:42:31 +0000 | [diff] [blame] | 50 | // Create the reader. |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 51 | if (IndexedInstrProfReader::hasFormat(*Buffer)) |
| 52 | Result.reset(new IndexedInstrProfReader(std::move(Buffer))); |
| 53 | else if (RawInstrProfReader64::hasFormat(*Buffer)) |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 54 | Result.reset(new RawInstrProfReader64(std::move(Buffer))); |
| 55 | else if (RawInstrProfReader32::hasFormat(*Buffer)) |
| 56 | Result.reset(new RawInstrProfReader32(std::move(Buffer))); |
Nathan Slingerland | 4f82366 | 2015-11-13 03:47:58 +0000 | [diff] [blame] | 57 | else if (TextInstrProfReader::hasFormat(*Buffer)) |
Nathan Slingerland | 911ced6 | 2015-11-12 18:39:26 +0000 | [diff] [blame] | 58 | Result.reset(new TextInstrProfReader(std::move(Buffer))); |
Nathan Slingerland | 4f82366 | 2015-11-13 03:47:58 +0000 | [diff] [blame] | 59 | else |
| 60 | return instrprof_error::unrecognized_format; |
Duncan P. N. Exon Smith | 09a67f4 | 2014-03-21 20:42:31 +0000 | [diff] [blame] | 61 | |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 62 | // Initialize the reader and return the result. |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 63 | if (std::error_code EC = initializeReader(*Result)) |
| 64 | return EC; |
| 65 | |
| 66 | return std::move(Result); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 67 | } |
| 68 | |
Justin Bogner | ab89ed7 | 2015-02-16 21:28:58 +0000 | [diff] [blame] | 69 | ErrorOr<std::unique_ptr<IndexedInstrProfReader>> |
| 70 | IndexedInstrProfReader::create(std::string Path) { |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 71 | // Set up the buffer to read. |
Diego Novillo | fcd5560 | 2014-11-03 00:51:45 +0000 | [diff] [blame] | 72 | auto BufferOrError = setupMemoryBuffer(Path); |
| 73 | if (std::error_code EC = BufferOrError.getError()) |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 74 | return EC; |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 75 | return IndexedInstrProfReader::create(std::move(BufferOrError.get())); |
| 76 | } |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 77 | |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 78 | |
| 79 | ErrorOr<std::unique_ptr<IndexedInstrProfReader>> |
| 80 | IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) { |
| 81 | // Sanity check the buffer. |
| 82 | if (Buffer->getBufferSize() > std::numeric_limits<unsigned>::max()) |
| 83 | return instrprof_error::too_large; |
Justin Bogner | ab89ed7 | 2015-02-16 21:28:58 +0000 | [diff] [blame] | 84 | |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 85 | // Create the reader. |
| 86 | if (!IndexedInstrProfReader::hasFormat(*Buffer)) |
| 87 | return instrprof_error::bad_magic; |
Justin Bogner | 2b6c537 | 2015-02-18 01:58:17 +0000 | [diff] [blame] | 88 | auto Result = llvm::make_unique<IndexedInstrProfReader>(std::move(Buffer)); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 89 | |
| 90 | // Initialize the reader and return the result. |
Justin Bogner | ab89ed7 | 2015-02-16 21:28:58 +0000 | [diff] [blame] | 91 | if (std::error_code EC = initializeReader(*Result)) |
| 92 | return EC; |
| 93 | |
| 94 | return std::move(Result); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 95 | } |
| 96 | |
| 97 | void InstrProfIterator::Increment() { |
| 98 | if (Reader->readNextRecord(Record)) |
| 99 | *this = InstrProfIterator(); |
| 100 | } |
| 101 | |
Nathan Slingerland | 4f82366 | 2015-11-13 03:47:58 +0000 | [diff] [blame] | 102 | bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { |
| 103 | // Verify that this really looks like plain ASCII text by checking a |
| 104 | // 'reasonable' number of characters (up to profile magic size). |
| 105 | size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t)); |
| 106 | StringRef buffer = Buffer.getBufferStart(); |
Vedant Kumar | 2491dd1 | 2015-12-11 00:40:05 +0000 | [diff] [blame] | 107 | return count == 0 || |
| 108 | std::all_of(buffer.begin(), buffer.begin() + count, |
| 109 | [](char c) { return ::isprint(c) || ::isspace(c); }); |
Nathan Slingerland | 4f82366 | 2015-11-13 03:47:58 +0000 | [diff] [blame] | 110 | } |
| 111 | |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 112 | // Read the profile variant flag from the header: ":FE" means this is a FE |
| 113 | // generated profile. ":IR" means this is an IR level profile. Other strings |
| 114 | // with a leading ':' will be reported an error format. |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 115 | std::error_code TextInstrProfReader::readHeader() { |
| 116 | Symtab.reset(new InstrProfSymtab()); |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 117 | bool IsIRInstr = false; |
| 118 | if (!Line->startswith(":")) { |
| 119 | IsIRLevelProfile = false; |
| 120 | return success(); |
| 121 | } |
| 122 | StringRef Str = (Line)->substr(1); |
| 123 | if (Str.equals_lower("ir")) |
| 124 | IsIRInstr = true; |
| 125 | else if (Str.equals_lower("fe")) |
| 126 | IsIRInstr = false; |
| 127 | else |
| 128 | return instrprof_error::bad_header; |
| 129 | |
| 130 | ++Line; |
| 131 | IsIRLevelProfile = IsIRInstr; |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 132 | return success(); |
| 133 | } |
| 134 | |
Xinliang David Li | e3bf4fd3 | 2015-12-14 18:44:01 +0000 | [diff] [blame] | 135 | std::error_code |
| 136 | TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { |
| 137 | |
| 138 | #define CHECK_LINE_END(Line) \ |
| 139 | if (Line.is_at_end()) \ |
| 140 | return error(instrprof_error::truncated); |
| 141 | #define READ_NUM(Str, Dst) \ |
| 142 | if ((Str).getAsInteger(10, (Dst))) \ |
| 143 | return error(instrprof_error::malformed); |
| 144 | #define VP_READ_ADVANCE(Val) \ |
| 145 | CHECK_LINE_END(Line); \ |
| 146 | uint32_t Val; \ |
| 147 | READ_NUM((*Line), (Val)); \ |
| 148 | Line++; |
| 149 | |
| 150 | if (Line.is_at_end()) |
| 151 | return success(); |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 152 | |
Xinliang David Li | e3bf4fd3 | 2015-12-14 18:44:01 +0000 | [diff] [blame] | 153 | uint32_t NumValueKinds; |
| 154 | if (Line->getAsInteger(10, NumValueKinds)) { |
| 155 | // No value profile data |
| 156 | return success(); |
| 157 | } |
| 158 | if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1) |
| 159 | return error(instrprof_error::malformed); |
| 160 | Line++; |
| 161 | |
| 162 | for (uint32_t VK = 0; VK < NumValueKinds; VK++) { |
| 163 | VP_READ_ADVANCE(ValueKind); |
| 164 | if (ValueKind > IPVK_Last) |
| 165 | return error(instrprof_error::malformed); |
| 166 | VP_READ_ADVANCE(NumValueSites); |
| 167 | if (!NumValueSites) |
| 168 | continue; |
| 169 | |
| 170 | Record.reserveSites(VK, NumValueSites); |
| 171 | for (uint32_t S = 0; S < NumValueSites; S++) { |
| 172 | VP_READ_ADVANCE(NumValueData); |
| 173 | |
| 174 | std::vector<InstrProfValueData> CurrentValues; |
| 175 | for (uint32_t V = 0; V < NumValueData; V++) { |
| 176 | CHECK_LINE_END(Line); |
| 177 | std::pair<StringRef, StringRef> VD = Line->split(':'); |
| 178 | uint64_t TakenCount, Value; |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 179 | if (VK == IPVK_IndirectCallTarget) { |
| 180 | Symtab->addFuncName(VD.first); |
| 181 | Value = IndexedInstrProf::ComputeHash(VD.first); |
| 182 | } else { |
Xinliang David Li | e3bf4fd3 | 2015-12-14 18:44:01 +0000 | [diff] [blame] | 183 | READ_NUM(VD.first, Value); |
| 184 | } |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 185 | READ_NUM(VD.second, TakenCount); |
Xinliang David Li | e3bf4fd3 | 2015-12-14 18:44:01 +0000 | [diff] [blame] | 186 | CurrentValues.push_back({Value, TakenCount}); |
| 187 | Line++; |
| 188 | } |
| 189 | Record.addValueData(VK, S, CurrentValues.data(), NumValueData, nullptr); |
| 190 | } |
| 191 | } |
| 192 | return success(); |
| 193 | |
| 194 | #undef CHECK_LINE_END |
| 195 | #undef READ_NUM |
| 196 | #undef VP_READ_ADVANCE |
| 197 | } |
| 198 | |
Rafael Espindola | db4ed0b | 2014-06-13 02:24:39 +0000 | [diff] [blame] | 199 | std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { |
Justin Bogner | cf36a36 | 2014-07-29 22:29:23 +0000 | [diff] [blame] | 200 | // Skip empty lines and comments. |
| 201 | while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 202 | ++Line; |
| 203 | // If we hit EOF while looking for a name, we're done. |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 204 | if (Line.is_at_end()) { |
| 205 | Symtab->finalizeSymtab(); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 206 | return error(instrprof_error::eof); |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 207 | } |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 208 | |
| 209 | // Read the function name. |
| 210 | Record.Name = *Line++; |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 211 | Symtab->addFuncName(Record.Name); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 212 | |
| 213 | // Read the function hash. |
| 214 | if (Line.is_at_end()) |
| 215 | return error(instrprof_error::truncated); |
Justin Bogner | f95ca07 | 2015-03-09 18:54:49 +0000 | [diff] [blame] | 216 | if ((Line++)->getAsInteger(0, Record.Hash)) |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 217 | return error(instrprof_error::malformed); |
| 218 | |
| 219 | // Read the number of counters. |
| 220 | uint64_t NumCounters; |
| 221 | if (Line.is_at_end()) |
| 222 | return error(instrprof_error::truncated); |
| 223 | if ((Line++)->getAsInteger(10, NumCounters)) |
| 224 | return error(instrprof_error::malformed); |
Justin Bogner | b59d7c7 | 2014-04-25 02:45:33 +0000 | [diff] [blame] | 225 | if (NumCounters == 0) |
| 226 | return error(instrprof_error::malformed); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 227 | |
| 228 | // Read each counter and fill our internal storage with the values. |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 229 | Record.Counts.clear(); |
| 230 | Record.Counts.reserve(NumCounters); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 231 | for (uint64_t I = 0; I < NumCounters; ++I) { |
| 232 | if (Line.is_at_end()) |
| 233 | return error(instrprof_error::truncated); |
| 234 | uint64_t Count; |
| 235 | if ((Line++)->getAsInteger(10, Count)) |
| 236 | return error(instrprof_error::malformed); |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 237 | Record.Counts.push_back(Count); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 238 | } |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 239 | |
Xinliang David Li | e3bf4fd3 | 2015-12-14 18:44:01 +0000 | [diff] [blame] | 240 | // Check if value profile data exists and read it if so. |
| 241 | if (std::error_code EC = readValueProfileData(Record)) |
| 242 | return EC; |
| 243 | |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 244 | // This is needed to avoid two pass parsing because llvm-profdata |
| 245 | // does dumping while reading. |
| 246 | Symtab->finalizeSymtab(); |
Justin Bogner | f8d7919 | 2014-03-21 17:24:48 +0000 | [diff] [blame] | 247 | return success(); |
| 248 | } |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 249 | |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 250 | template <class IntPtrT> |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 251 | bool RawInstrProfReader<IntPtrT>::hasFormat(const MemoryBuffer &DataBuffer) { |
Duncan P. N. Exon Smith | d7d8347 | 2014-03-24 00:47:18 +0000 | [diff] [blame] | 252 | if (DataBuffer.getBufferSize() < sizeof(uint64_t)) |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 253 | return false; |
Duncan P. N. Exon Smith | d7d8347 | 2014-03-24 00:47:18 +0000 | [diff] [blame] | 254 | uint64_t Magic = |
| 255 | *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart()); |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 256 | return RawInstrProf::getMagic<IntPtrT>() == Magic || |
| 257 | sys::getSwappedBytes(RawInstrProf::getMagic<IntPtrT>()) == Magic; |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 258 | } |
| 259 | |
| 260 | template <class IntPtrT> |
Rafael Espindola | db4ed0b | 2014-06-13 02:24:39 +0000 | [diff] [blame] | 261 | std::error_code RawInstrProfReader<IntPtrT>::readHeader() { |
Duncan P. N. Exon Smith | 09a67f4 | 2014-03-21 20:42:31 +0000 | [diff] [blame] | 262 | if (!hasFormat(*DataBuffer)) |
| 263 | return error(instrprof_error::bad_magic); |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 264 | if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header)) |
Duncan P. N. Exon Smith | 531bb48 | 2014-03-21 20:42:28 +0000 | [diff] [blame] | 265 | return error(instrprof_error::bad_header); |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 266 | auto *Header = reinterpret_cast<const RawInstrProf::Header *>( |
| 267 | DataBuffer->getBufferStart()); |
| 268 | ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>(); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 269 | return readHeader(*Header); |
| 270 | } |
| 271 | |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 272 | template <class IntPtrT> |
Rafael Espindola | db4ed0b | 2014-06-13 02:24:39 +0000 | [diff] [blame] | 273 | std::error_code |
| 274 | RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) { |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 275 | const char *End = DataBuffer->getBufferEnd(); |
| 276 | // Skip zero padding between profiles. |
| 277 | while (CurrentPos != End && *CurrentPos == 0) |
| 278 | ++CurrentPos; |
| 279 | // If there's nothing left, we're done. |
| 280 | if (CurrentPos == End) |
| 281 | return instrprof_error::eof; |
| 282 | // If there isn't enough space for another header, this is probably just |
| 283 | // garbage at the end of the file. |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 284 | if (CurrentPos + sizeof(RawInstrProf::Header) > End) |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 285 | return instrprof_error::malformed; |
Justin Bogner | 54b1128 | 2014-09-12 21:22:55 +0000 | [diff] [blame] | 286 | // The writer ensures each profile is padded to start at an aligned address. |
| 287 | if (reinterpret_cast<size_t>(CurrentPos) % alignOf<uint64_t>()) |
| 288 | return instrprof_error::malformed; |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 289 | // The magic should have the same byte order as in the previous header. |
| 290 | uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos); |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 291 | if (Magic != swap(RawInstrProf::getMagic<IntPtrT>())) |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 292 | return instrprof_error::bad_magic; |
| 293 | |
| 294 | // There's another profile to read, so we need to process the header. |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 295 | auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos); |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 296 | return readHeader(*Header); |
| 297 | } |
| 298 | |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 299 | template <class IntPtrT> |
Vedant Kumar | e44482f | 2016-04-21 21:07:25 +0000 | [diff] [blame^] | 300 | std::error_code |
| 301 | RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) { |
| 302 | std::error_code EC = Symtab.create(StringRef(NamesStart, NamesSize)); |
| 303 | if (EC) |
| 304 | return EC; |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 305 | for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) { |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 306 | const IntPtrT FPtr = swap(I->FunctionPointer); |
| 307 | if (!FPtr) |
| 308 | continue; |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 309 | Symtab.mapAddress(FPtr, I->NameRef); |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 310 | } |
| 311 | Symtab.finalizeSymtab(); |
Vedant Kumar | e44482f | 2016-04-21 21:07:25 +0000 | [diff] [blame^] | 312 | return success(); |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 313 | } |
| 314 | |
| 315 | template <class IntPtrT> |
| 316 | std::error_code |
| 317 | RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) { |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 318 | Version = swap(Header.Version); |
| 319 | if (GET_VERSION(Version) != RawInstrProf::Version) |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 320 | return error(instrprof_error::unsupported_version); |
| 321 | |
| 322 | CountersDelta = swap(Header.CountersDelta); |
| 323 | NamesDelta = swap(Header.NamesDelta); |
| 324 | auto DataSize = swap(Header.DataSize); |
| 325 | auto CountersSize = swap(Header.CountersSize); |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 326 | NamesSize = swap(Header.NamesSize); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 327 | auto ValueDataSize = swap(Header.ValueDataSize); |
| 328 | ValueKindLast = swap(Header.ValueKindLast); |
| 329 | |
| 330 | auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>); |
| 331 | auto PaddingSize = getNumPaddingBytes(NamesSize); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 332 | |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 333 | ptrdiff_t DataOffset = sizeof(RawInstrProf::Header); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 334 | ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes; |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 335 | ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 336 | ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; |
| 337 | size_t ProfileSize = ValueDataOffset + ValueDataSize; |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 338 | |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 339 | auto *Start = reinterpret_cast<const char *>(&Header); |
| 340 | if (Start + ProfileSize > DataBuffer->getBufferEnd()) |
Duncan P. N. Exon Smith | 531bb48 | 2014-03-21 20:42:28 +0000 | [diff] [blame] | 341 | return error(instrprof_error::bad_header); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 342 | |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 343 | Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>( |
| 344 | Start + DataOffset); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 345 | DataEnd = Data + DataSize; |
Duncan P. N. Exon Smith | d7d8347 | 2014-03-24 00:47:18 +0000 | [diff] [blame] | 346 | CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset); |
| 347 | NamesStart = Start + NamesOffset; |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 348 | ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset); |
Justin Bogner | a119f32 | 2014-05-16 00:38:00 +0000 | [diff] [blame] | 349 | ProfileEnd = Start + ProfileSize; |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 350 | |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 351 | std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>(); |
Vedant Kumar | e44482f | 2016-04-21 21:07:25 +0000 | [diff] [blame^] | 352 | if (auto EC = createSymtab(*NewSymtab.get())) |
| 353 | return EC; |
| 354 | |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 355 | Symtab = std::move(NewSymtab); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 356 | return success(); |
| 357 | } |
| 358 | |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 359 | template <class IntPtrT> |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 360 | std::error_code RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) { |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 361 | Record.Name = getName(Data->NameRef); |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 362 | return success(); |
| 363 | } |
| 364 | |
| 365 | template <class IntPtrT> |
| 366 | std::error_code RawInstrProfReader<IntPtrT>::readFuncHash( |
| 367 | InstrProfRecord &Record) { |
| 368 | Record.Hash = swap(Data->FuncHash); |
| 369 | return success(); |
| 370 | } |
| 371 | |
| 372 | template <class IntPtrT> |
| 373 | std::error_code RawInstrProfReader<IntPtrT>::readRawCounts( |
| 374 | InstrProfRecord &Record) { |
Justin Bogner | b59d7c7 | 2014-04-25 02:45:33 +0000 | [diff] [blame] | 375 | uint32_t NumCounters = swap(Data->NumCounters); |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 376 | IntPtrT CounterPtr = Data->CounterPtr; |
Justin Bogner | b59d7c7 | 2014-04-25 02:45:33 +0000 | [diff] [blame] | 377 | if (NumCounters == 0) |
| 378 | return error(instrprof_error::malformed); |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 379 | |
| 380 | auto RawCounts = makeArrayRef(getCounter(CounterPtr), NumCounters); |
| 381 | auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 382 | |
| 383 | // Check bounds. |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 384 | if (RawCounts.data() < CountersStart || |
Duncan P. N. Exon Smith | d7d8347 | 2014-03-24 00:47:18 +0000 | [diff] [blame] | 385 | RawCounts.data() + RawCounts.size() > NamesStartAsCounter) |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 386 | return error(instrprof_error::malformed); |
| 387 | |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 388 | if (ShouldSwapBytes) { |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 389 | Record.Counts.clear(); |
| 390 | Record.Counts.reserve(RawCounts.size()); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 391 | for (uint64_t Count : RawCounts) |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 392 | Record.Counts.push_back(swap(Count)); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 393 | } else |
| 394 | Record.Counts = RawCounts; |
| 395 | |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 396 | return success(); |
| 397 | } |
| 398 | |
| 399 | template <class IntPtrT> |
Xinliang David Li | 01cb9bd | 2015-12-04 01:02:10 +0000 | [diff] [blame] | 400 | std::error_code |
| 401 | RawInstrProfReader<IntPtrT>::readValueProfilingData(InstrProfRecord &Record) { |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 402 | |
| 403 | Record.clearValueData(); |
Xinliang David Li | d922c26 | 2015-12-11 06:53:53 +0000 | [diff] [blame] | 404 | CurValueDataSize = 0; |
| 405 | // Need to match the logic in value profile dumper code in compiler-rt: |
| 406 | uint32_t NumValueKinds = 0; |
| 407 | for (uint32_t I = 0; I < IPVK_Last + 1; I++) |
| 408 | NumValueKinds += (Data->NumValueSites[I] != 0); |
| 409 | |
| 410 | if (!NumValueKinds) |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 411 | return success(); |
| 412 | |
Xinliang David Li | 01cb9bd | 2015-12-04 01:02:10 +0000 | [diff] [blame] | 413 | ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr = |
Xinliang David Li | d922c26 | 2015-12-11 06:53:53 +0000 | [diff] [blame] | 414 | ValueProfData::getValueProfData(ValueDataStart, |
Xinliang David Li | 01cb9bd | 2015-12-04 01:02:10 +0000 | [diff] [blame] | 415 | (const unsigned char *)ProfileEnd, |
| 416 | getDataEndianness()); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 417 | |
Xinliang David Li | 01cb9bd | 2015-12-04 01:02:10 +0000 | [diff] [blame] | 418 | if (VDataPtrOrErr.getError()) |
| 419 | return VDataPtrOrErr.getError(); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 420 | |
Adam Nemet | 2f36f05 | 2016-03-28 18:27:44 +0000 | [diff] [blame] | 421 | // Note that besides deserialization, this also performs the conversion for |
| 422 | // indirect call targets. The function pointers from the raw profile are |
| 423 | // remapped into function name hashes. |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 424 | VDataPtrOrErr.get()->deserializeTo(Record, &Symtab->getAddrHashMap()); |
Xinliang David Li | d922c26 | 2015-12-11 06:53:53 +0000 | [diff] [blame] | 425 | CurValueDataSize = VDataPtrOrErr.get()->getSize(); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 426 | return success(); |
| 427 | } |
| 428 | |
| 429 | template <class IntPtrT> |
Xinliang David Li | 01cb9bd | 2015-12-04 01:02:10 +0000 | [diff] [blame] | 430 | std::error_code |
| 431 | RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) { |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 432 | if (atEnd()) |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 433 | if (std::error_code EC = readNextHeader(ProfileEnd)) |
| 434 | return EC; |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 435 | |
| 436 | // Read name ad set it in Record. |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 437 | if (std::error_code EC = readName(Record)) |
| 438 | return EC; |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 439 | |
| 440 | // Read FuncHash and set it in Record. |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 441 | if (std::error_code EC = readFuncHash(Record)) |
| 442 | return EC; |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 443 | |
| 444 | // Read raw counts and set Record. |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 445 | if (std::error_code EC = readRawCounts(Record)) |
| 446 | return EC; |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 447 | |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 448 | // Read value data and set Record. |
Xinliang David Li | 2d4803e | 2015-12-10 23:48:05 +0000 | [diff] [blame] | 449 | if (std::error_code EC = readValueProfilingData(Record)) |
| 450 | return EC; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 451 | |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 452 | // Iterate. |
Xinliang David Li | cf4a128 | 2015-10-28 19:34:04 +0000 | [diff] [blame] | 453 | advanceData(); |
Duncan P. N. Exon Smith | 24b4b65 | 2014-03-21 18:26:05 +0000 | [diff] [blame] | 454 | return success(); |
| 455 | } |
Duncan P. N. Exon Smith | 4680361 | 2014-03-23 03:38:12 +0000 | [diff] [blame] | 456 | |
| 457 | namespace llvm { |
| 458 | template class RawInstrProfReader<uint32_t>; |
| 459 | template class RawInstrProfReader<uint64_t>; |
| 460 | } |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 461 | |
Justin Bogner | b5d368e | 2014-04-18 22:00:22 +0000 | [diff] [blame] | 462 | InstrProfLookupTrait::hash_value_type |
| 463 | InstrProfLookupTrait::ComputeHash(StringRef K) { |
| 464 | return IndexedInstrProf::ComputeHash(HashType, K); |
| 465 | } |
| 466 | |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 467 | typedef InstrProfLookupTrait::data_type data_type; |
| 468 | typedef InstrProfLookupTrait::offset_type offset_type; |
| 469 | |
Xinliang David Li | be969c2 | 2015-11-28 05:06:00 +0000 | [diff] [blame] | 470 | bool InstrProfLookupTrait::readValueProfilingData( |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 471 | const unsigned char *&D, const unsigned char *const End) { |
Xinliang David Li | 9955687 | 2015-11-17 23:00:40 +0000 | [diff] [blame] | 472 | ErrorOr<std::unique_ptr<ValueProfData>> VDataPtrOrErr = |
| 473 | ValueProfData::getValueProfData(D, End, ValueProfDataEndianness); |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 474 | |
Xinliang David Li | ee41589 | 2015-11-10 00:24:45 +0000 | [diff] [blame] | 475 | if (VDataPtrOrErr.getError()) |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 476 | return false; |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 477 | |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 478 | VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr); |
Xinliang David Li | ee41589 | 2015-11-10 00:24:45 +0000 | [diff] [blame] | 479 | D += VDataPtrOrErr.get()->TotalSize; |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 480 | |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 481 | return true; |
| 482 | } |
| 483 | |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 484 | data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, |
| 485 | offset_type N) { |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 486 | // Check if the data is corrupt. If so, don't try to read it. |
| 487 | if (N % sizeof(uint64_t)) |
| 488 | return data_type(); |
| 489 | |
| 490 | DataBuffer.clear(); |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 491 | std::vector<uint64_t> CounterBuffer; |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 492 | |
| 493 | using namespace support; |
| 494 | const unsigned char *End = D + N; |
| 495 | while (D < End) { |
Xinliang David Li | c758387 | 2015-10-13 16:35:59 +0000 | [diff] [blame] | 496 | // Read hash. |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 497 | if (D + sizeof(uint64_t) >= End) |
| 498 | return data_type(); |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 499 | uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D); |
| 500 | |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 501 | // Initialize number of counters for GET_VERSION(FormatVersion) == 1. |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 502 | uint64_t CountsSize = N / sizeof(uint64_t) - 1; |
Xinliang David Li | c758387 | 2015-10-13 16:35:59 +0000 | [diff] [blame] | 503 | // If format version is different then read the number of counters. |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 504 | if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) { |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 505 | if (D + sizeof(uint64_t) > End) |
| 506 | return data_type(); |
| 507 | CountsSize = endian::readNext<uint64_t, little, unaligned>(D); |
| 508 | } |
Xinliang David Li | c758387 | 2015-10-13 16:35:59 +0000 | [diff] [blame] | 509 | // Read counter values. |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 510 | if (D + CountsSize * sizeof(uint64_t) > End) |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 511 | return data_type(); |
| 512 | |
| 513 | CounterBuffer.clear(); |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 514 | CounterBuffer.reserve(CountsSize); |
| 515 | for (uint64_t J = 0; J < CountsSize; ++J) |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 516 | CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D)); |
| 517 | |
Xinliang David Li | 2004f00 | 2015-11-02 05:08:23 +0000 | [diff] [blame] | 518 | DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 519 | |
Xinliang David Li | c758387 | 2015-10-13 16:35:59 +0000 | [diff] [blame] | 520 | // Read value profiling data. |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 521 | if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && |
Xinliang David Li | a6b2c4f | 2016-01-14 02:47:01 +0000 | [diff] [blame] | 522 | !readValueProfilingData(D, End)) { |
Justin Bogner | 9e9a057 | 2015-09-29 22:13:58 +0000 | [diff] [blame] | 523 | DataBuffer.clear(); |
| 524 | return data_type(); |
| 525 | } |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 526 | } |
| 527 | return DataBuffer; |
| 528 | } |
| 529 | |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 530 | template <typename HashTableImpl> |
| 531 | std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords( |
| 532 | StringRef FuncName, ArrayRef<InstrProfRecord> &Data) { |
| 533 | auto Iter = HashTable->find(FuncName); |
| 534 | if (Iter == HashTable->end()) |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 535 | return instrprof_error::unknown_function; |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 536 | |
| 537 | Data = (*Iter); |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 538 | if (Data.empty()) |
| 539 | return instrprof_error::malformed; |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 540 | |
| 541 | return instrprof_error::success; |
| 542 | } |
| 543 | |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 544 | template <typename HashTableImpl> |
| 545 | std::error_code InstrProfReaderIndex<HashTableImpl>::getRecords( |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 546 | ArrayRef<InstrProfRecord> &Data) { |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 547 | if (atEnd()) |
| 548 | return instrprof_error::eof; |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 549 | |
| 550 | Data = *RecordIterator; |
| 551 | |
Xinliang David Li | 2d4803e | 2015-12-10 23:48:05 +0000 | [diff] [blame] | 552 | if (Data.empty()) |
| 553 | return instrprof_error::malformed; |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 554 | |
| 555 | return instrprof_error::success; |
| 556 | } |
| 557 | |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 558 | template <typename HashTableImpl> |
| 559 | InstrProfReaderIndex<HashTableImpl>::InstrProfReaderIndex( |
| 560 | const unsigned char *Buckets, const unsigned char *const Payload, |
| 561 | const unsigned char *const Base, IndexedInstrProf::HashT HashType, |
| 562 | uint64_t Version) { |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 563 | FormatVersion = Version; |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 564 | HashTable.reset(HashTableImpl::Create( |
| 565 | Buckets, Payload, Base, |
| 566 | typename HashTableImpl::InfoType(HashType, Version))); |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 567 | RecordIterator = HashTable->data_begin(); |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 568 | } |
| 569 | |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 570 | bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 571 | if (DataBuffer.getBufferSize() < 8) |
| 572 | return false; |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 573 | using namespace support; |
| 574 | uint64_t Magic = |
| 575 | endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart()); |
Xinliang David Li | c758387 | 2015-10-13 16:35:59 +0000 | [diff] [blame] | 576 | // Verify that it's magical. |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 577 | return Magic == IndexedInstrProf::Magic; |
| 578 | } |
| 579 | |
Xinliang David Li | 6c93ee8 | 2016-02-03 04:08:18 +0000 | [diff] [blame] | 580 | const unsigned char * |
| 581 | IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, |
| 582 | const unsigned char *Cur) { |
| 583 | using namespace support; |
| 584 | if (Version >= IndexedInstrProf::Version4) { |
| 585 | const IndexedInstrProf::Summary *SummaryInLE = |
| 586 | reinterpret_cast<const IndexedInstrProf::Summary *>(Cur); |
| 587 | uint64_t NFields = |
| 588 | endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields); |
| 589 | uint64_t NEntries = |
| 590 | endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries); |
| 591 | uint32_t SummarySize = |
| 592 | IndexedInstrProf::Summary::getSize(NFields, NEntries); |
| 593 | std::unique_ptr<IndexedInstrProf::Summary> SummaryData = |
| 594 | IndexedInstrProf::allocSummary(SummarySize); |
| 595 | |
| 596 | const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE); |
| 597 | uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get()); |
| 598 | for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) |
| 599 | Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]); |
| 600 | |
Easwaran Raman | 4309570 | 2016-02-17 18:18:47 +0000 | [diff] [blame] | 601 | // initialize InstrProfSummary using the SummaryData from disk. |
| 602 | this->Summary = llvm::make_unique<InstrProfSummary>(*(SummaryData.get())); |
Xinliang David Li | 6c93ee8 | 2016-02-03 04:08:18 +0000 | [diff] [blame] | 603 | return Cur + SummarySize; |
| 604 | } else { |
| 605 | // For older version of profile data, we need to compute on the fly: |
| 606 | using namespace IndexedInstrProf; |
Easwaran Raman | 40ee23d | 2016-02-19 03:15:33 +0000 | [diff] [blame] | 607 | this->Summary = |
| 608 | llvm::make_unique<InstrProfSummary>(ProfileSummary::DefaultCutoffs); |
Xinliang David Li | 6c93ee8 | 2016-02-03 04:08:18 +0000 | [diff] [blame] | 609 | this->Summary->computeDetailedSummary(); |
| 610 | return Cur; |
| 611 | } |
| 612 | } |
| 613 | |
Rafael Espindola | db4ed0b | 2014-06-13 02:24:39 +0000 | [diff] [blame] | 614 | std::error_code IndexedInstrProfReader::readHeader() { |
Aaron Ballman | a7c9ed5 | 2014-05-01 17:16:24 +0000 | [diff] [blame] | 615 | const unsigned char *Start = |
| 616 | (const unsigned char *)DataBuffer->getBufferStart(); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 617 | const unsigned char *Cur = Start; |
Aaron Ballman | a7c9ed5 | 2014-05-01 17:16:24 +0000 | [diff] [blame] | 618 | if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24) |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 619 | return error(instrprof_error::truncated); |
| 620 | |
| 621 | using namespace support; |
| 622 | |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 623 | auto *Header = reinterpret_cast<const IndexedInstrProf::Header *>(Cur); |
| 624 | Cur += sizeof(IndexedInstrProf::Header); |
| 625 | |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 626 | // Check the magic number. |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 627 | uint64_t Magic = endian::byte_swap<uint64_t, little>(Header->Magic); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 628 | if (Magic != IndexedInstrProf::Magic) |
| 629 | return error(instrprof_error::bad_magic); |
| 630 | |
| 631 | // Read the version. |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 632 | uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version); |
Rong Xu | 33c76c0 | 2016-02-10 17:18:30 +0000 | [diff] [blame] | 633 | if (GET_VERSION(FormatVersion) > |
| 634 | IndexedInstrProf::ProfVersion::CurrentVersion) |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 635 | return error(instrprof_error::unsupported_version); |
| 636 | |
Xinliang David Li | 6c93ee8 | 2016-02-03 04:08:18 +0000 | [diff] [blame] | 637 | Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 638 | |
| 639 | // Read the hash type and start offset. |
| 640 | IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>( |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 641 | endian::byte_swap<uint64_t, little>(Header->HashType)); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 642 | if (HashType > IndexedInstrProf::HashT::Last) |
| 643 | return error(instrprof_error::unsupported_hash_type); |
Xinliang David Li | dab183ed | 2015-10-18 01:02:29 +0000 | [diff] [blame] | 644 | |
| 645 | uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 646 | |
| 647 | // The rest of the file is an on disk hash table. |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 648 | InstrProfReaderIndexBase *IndexPtr = nullptr; |
| 649 | IndexPtr = new InstrProfReaderIndex<OnDiskHashTableImplV3>( |
| 650 | Start + HashOffset, Cur, Start, HashType, FormatVersion); |
| 651 | Index.reset(IndexPtr); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 652 | return success(); |
| 653 | } |
| 654 | |
Xinliang David Li | a716cc5 | 2015-12-20 06:22:13 +0000 | [diff] [blame] | 655 | InstrProfSymtab &IndexedInstrProfReader::getSymtab() { |
| 656 | if (Symtab.get()) |
| 657 | return *Symtab.get(); |
| 658 | |
| 659 | std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>(); |
| 660 | Index->populateSymtab(*NewSymtab.get()); |
| 661 | |
| 662 | Symtab = std::move(NewSymtab); |
| 663 | return *Symtab.get(); |
| 664 | } |
| 665 | |
Xinliang David Li | 6aa216c | 2015-11-06 07:54:21 +0000 | [diff] [blame] | 666 | ErrorOr<InstrProfRecord> |
| 667 | IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, |
| 668 | uint64_t FuncHash) { |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 669 | ArrayRef<InstrProfRecord> Data; |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 670 | std::error_code EC = Index->getRecords(FuncName, Data); |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 671 | if (EC != instrprof_error::success) |
| 672 | return EC; |
Justin Bogner | 821d747 | 2014-08-01 22:50:07 +0000 | [diff] [blame] | 673 | // Found it. Look for counters with the right hash. |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 674 | for (unsigned I = 0, E = Data.size(); I < E; ++I) { |
Justin Bogner | 821d747 | 2014-08-01 22:50:07 +0000 | [diff] [blame] | 675 | // Check for a match and fill the vector if there is one. |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 676 | if (Data[I].Hash == FuncHash) { |
Xinliang David Li | 2004f00 | 2015-11-02 05:08:23 +0000 | [diff] [blame] | 677 | return std::move(Data[I]); |
Justin Bogner | 821d747 | 2014-08-01 22:50:07 +0000 | [diff] [blame] | 678 | } |
| 679 | } |
| 680 | return error(instrprof_error::hash_mismatch); |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 681 | } |
| 682 | |
Xinliang David Li | 6aa216c | 2015-11-06 07:54:21 +0000 | [diff] [blame] | 683 | std::error_code |
| 684 | IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, uint64_t FuncHash, |
| 685 | std::vector<uint64_t> &Counts) { |
Xinliang David Li | 2004f00 | 2015-11-02 05:08:23 +0000 | [diff] [blame] | 686 | ErrorOr<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash); |
Xinliang David Li | 6aa216c | 2015-11-06 07:54:21 +0000 | [diff] [blame] | 687 | if (std::error_code EC = Record.getError()) |
| 688 | return EC; |
Xinliang David Li | 2004f00 | 2015-11-02 05:08:23 +0000 | [diff] [blame] | 689 | |
| 690 | Counts = Record.get().Counts; |
| 691 | return success(); |
| 692 | } |
| 693 | |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 694 | std::error_code IndexedInstrProfReader::readNextRecord( |
| 695 | InstrProfRecord &Record) { |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 696 | static unsigned RecordIndex = 0; |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 697 | |
| 698 | ArrayRef<InstrProfRecord> Data; |
| 699 | |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 700 | std::error_code EC = Index->getRecords(Data); |
Xinliang David Li | 4c3ab81 | 2015-11-12 00:32:17 +0000 | [diff] [blame] | 701 | if (EC != instrprof_error::success) |
| 702 | return error(EC); |
Xinliang David Li | 140f4c4 | 2015-10-28 04:20:31 +0000 | [diff] [blame] | 703 | |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 704 | Record = Data[RecordIndex++]; |
| 705 | if (RecordIndex >= Data.size()) { |
Xinliang David Li | a28306d | 2015-12-01 20:26:26 +0000 | [diff] [blame] | 706 | Index->advanceToNextKey(); |
Justin Bogner | 3a7d44c | 2015-06-22 23:58:05 +0000 | [diff] [blame] | 707 | RecordIndex = 0; |
Justin Bogner | 821d747 | 2014-08-01 22:50:07 +0000 | [diff] [blame] | 708 | } |
Justin Bogner | b7aa263 | 2014-04-18 21:48:40 +0000 | [diff] [blame] | 709 | return success(); |
| 710 | } |