Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 1 | /*===-- InstrProfData.inc - instr profiling runtime structures -----------=== *\ |
| 2 | |* |
| 3 | |* The LLVM Compiler Infrastructure |
| 4 | |* |
| 5 | |* This file is distributed under the University of Illinois Open Source |
| 6 | |* License. See LICENSE.TXT for details. |
| 7 | |* |
| 8 | \*===----------------------------------------------------------------------===*/ |
| 9 | /* |
| 10 | * This is the master file that defines all the data structure, signature, |
| 11 | * constant literals that are shared across profiling runtime library, |
| 12 | * compiler (instrumentation), and host tools (reader/writer). The entities |
| 13 | * defined in this file affect the profile runtime ABI, the raw profile format, |
| 14 | * or both. |
| 15 | * |
| 16 | * The file has two identical copies. The master copy lives in LLVM and |
| 17 | * the other one sits in compiler-rt/lib/profile directory. To make changes |
| 18 | * in this file, first modify the master copy and copy it over to compiler-rt. |
| 19 | * Testing of any change in this file can start only after the two copies are |
| 20 | * synced up. |
| 21 | * |
| 22 | * The first part of the file includes macros that defines types, names, and |
| 23 | * initializers for the member fields of the core data structures. The field |
| 24 | * declarations for one structure is enabled by defining the field activation |
| 25 | * macro associated with that structure. Only one field activation record |
| 26 | * can be defined at one time and the rest definitions will be filtered out by |
| 27 | * the preprocessor. |
| 28 | * |
| 29 | * Examples of how the template is used to instantiate structure definition: |
| 30 | * 1. To declare a structure: |
| 31 | * |
| 32 | * struct ProfData { |
| 33 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
| 34 | * Type Name; |
| 35 | * #include "llvm/ProfileData/InstrProfData.inc" |
| 36 | * }; |
| 37 | * |
| 38 | * 2. To construct LLVM type arrays for the struct type: |
| 39 | * |
| 40 | * Type *DataTypes[] = { |
| 41 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
| 42 | * LLVMType, |
| 43 | * #include "llvm/ProfileData/InstrProfData.inc" |
| 44 | * }; |
| 45 | * |
| 46 | * 4. To construct constant array for the initializers: |
| 47 | * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ |
| 48 | * Initializer, |
| 49 | * Constant *ConstantVals[] = { |
| 50 | * #include "llvm/ProfileData/InstrProfData.inc" |
| 51 | * }; |
| 52 | * |
| 53 | * |
| 54 | * The second part of the file includes definitions all other entities that |
| 55 | * are related to runtime ABI and format. When no field activation macro is |
| 56 | * defined, this file can be included to introduce the definitions. |
| 57 | * |
| 58 | \*===----------------------------------------------------------------------===*/ |
| 59 | |
| 60 | /* INSTR_PROF_DATA start. */ |
| 61 | /* Definition of member fields of the per-function control structure. */ |
| 62 | #ifndef INSTR_PROF_DATA |
| 63 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) |
| 64 | #else |
| 65 | #define INSTR_PROF_DATA_DEFINED |
| 66 | #endif |
| 67 | |
| 68 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ |
| 69 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ |
| 70 | NamePtr->getType()->getPointerElementType()->getArrayNumElements())) |
| 71 | INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ |
| 72 | ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) |
| 73 | INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
| 74 | ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ |
| 75 | Inc->getHash()->getZExtValue())) |
| 76 | INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \ |
| 77 | ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx))) |
| 78 | INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ |
| 79 | ConstantExpr::getBitCast(CounterPtr, \ |
| 80 | llvm::Type::getInt64PtrTy(Ctx))) |
| 81 | INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ |
| 82 | FunctionAddr) |
| 83 | INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ |
| 84 | ConstantPointerNull::get(Int8PtrTy)) |
| 85 | INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ |
| 86 | ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) |
| 87 | #undef INSTR_PROF_DATA |
| 88 | /* INSTR_PROF_DATA end. */ |
| 89 | |
| 90 | /* INSTR_PROF_RAW_HEADER start */ |
| 91 | /* Definition of member fields of the raw profile header data structure. */ |
| 92 | #ifndef INSTR_PROF_RAW_HEADER |
| 93 | #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) |
| 94 | #else |
| 95 | #define INSTR_PROF_DATA_DEFINED |
| 96 | #endif |
| 97 | INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) |
| 98 | INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) |
| 99 | INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) |
| 100 | INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) |
| 101 | INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) |
| 102 | INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) |
| 103 | INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) |
| 104 | INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) |
| 105 | INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize) |
| 106 | INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin) |
| 107 | #undef INSTR_PROF_RAW_HEADER |
| 108 | /* INSTR_PROF_RAW_HEADER end */ |
| 109 | |
| 110 | /* VALUE_PROF_FUNC_PARAM start */ |
| 111 | /* Definition of parameter types of the runtime API used to do value profiling |
| 112 | * for a given value site. |
| 113 | */ |
| 114 | #ifndef VALUE_PROF_FUNC_PARAM |
| 115 | #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) |
| 116 | #define INSTR_PROF_COMMA |
| 117 | #else |
| 118 | #define INSTR_PROF_DATA_DEFINED |
| 119 | #define INSTR_PROF_COMMA , |
| 120 | #endif |
| 121 | VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ |
| 122 | INSTR_PROF_COMMA |
| 123 | VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA |
| 124 | VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) |
| 125 | #undef VALUE_PROF_FUNC_PARAM |
| 126 | #undef INSTR_PROF_COMMA |
| 127 | /* VALUE_PROF_FUNC_PARAM end */ |
| 128 | |
| 129 | /* VALUE_PROF_KIND start */ |
| 130 | #ifndef VALUE_PROF_KIND |
| 131 | #define VALUE_PROF_KIND(Enumerator, Value) |
| 132 | #else |
| 133 | #define INSTR_PROF_DATA_DEFINED |
| 134 | #endif |
| 135 | VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) |
| 136 | /* These two kinds must be the last to be |
| 137 | * declared. This is to make sure the string |
| 138 | * array created with the template can be |
| 139 | * indexed with the kind value. |
| 140 | */ |
| 141 | VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) |
| 142 | VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) |
| 143 | |
| 144 | #undef VALUE_PROF_KIND |
| 145 | /* VALUE_PROF_KIND end */ |
| 146 | |
| 147 | /* COVMAP_FUNC_RECORD start */ |
| 148 | /* Definition of member fields of the function record structure in coverage |
| 149 | * map. |
| 150 | */ |
| 151 | #ifndef COVMAP_FUNC_RECORD |
| 152 | #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) |
| 153 | #else |
| 154 | #define INSTR_PROF_DATA_DEFINED |
| 155 | #endif |
| 156 | COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ |
| 157 | NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ |
| 158 | llvm::Type::getInt8PtrTy(Ctx))) |
| 159 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ |
| 160 | llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ |
| 161 | NameValue.size())) |
| 162 | COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ |
| 163 | llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ |
| 164 | CoverageMapping.size())) |
| 165 | COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ |
| 166 | llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) |
| 167 | #undef COVMAP_FUNC_RECORD |
| 168 | /* COVMAP_FUNC_RECORD end. */ |
| 169 | |
| 170 | |
| 171 | #ifdef INSTR_PROF_VALUE_PROF_DATA |
| 172 | #define INSTR_PROF_DATA_DEFINED |
| 173 | |
| 174 | /*! |
| 175 | * This is the header of the data structure that defines the on-disk |
| 176 | * layout of the value profile data of a particular kind for one function. |
| 177 | */ |
| 178 | typedef struct ValueProfRecord { |
| 179 | /* The kind of the value profile record. */ |
| 180 | uint32_t Kind; |
| 181 | /* |
| 182 | * The number of value profile sites. It is guaranteed to be non-zero; |
| 183 | * otherwise the record for this kind won't be emitted. |
| 184 | */ |
| 185 | uint32_t NumValueSites; |
| 186 | /* |
| 187 | * The first element of the array that stores the number of profiled |
| 188 | * values for each value site. The size of the array is NumValueSites. |
| 189 | * Since NumValueSites is greater than zero, there is at least one |
| 190 | * element in the array. |
| 191 | */ |
| 192 | uint8_t SiteCountArray[1]; |
| 193 | |
| 194 | /* |
| 195 | * The fake declaration is for documentation purpose only. |
| 196 | * Align the start of next field to be on 8 byte boundaries. |
| 197 | uint8_t Padding[X]; |
| 198 | */ |
| 199 | |
| 200 | /* The array of value profile data. The size of the array is the sum |
| 201 | * of all elements in SiteCountArray[]. |
| 202 | InstrProfValueData ValueData[]; |
| 203 | */ |
| 204 | |
| 205 | #ifdef __cplusplus |
| 206 | /*! |
| 207 | * \brief Return the number of value sites. |
| 208 | */ |
| 209 | uint32_t getNumValueSites() const { return NumValueSites; } |
| 210 | /*! |
| 211 | * \brief Read data from this record and save it to Record. |
| 212 | */ |
| 213 | void deserializeTo(InstrProfRecord &Record, |
| 214 | InstrProfRecord::ValueMapType *VMap); |
| 215 | /* |
| 216 | * In-place byte swap: |
| 217 | * Do byte swap for this instance. \c Old is the original order before |
| 218 | * the swap, and \c New is the New byte order. |
| 219 | */ |
| 220 | void swapBytes(support::endianness Old, support::endianness New); |
| 221 | #endif |
| 222 | } ValueProfRecord; |
| 223 | |
| 224 | /*! |
| 225 | * Per-function header/control data structure for value profiling |
| 226 | * data in indexed format. |
| 227 | */ |
| 228 | typedef struct ValueProfData { |
| 229 | /* |
| 230 | * Total size in bytes including this field. It must be a multiple |
| 231 | * of sizeof(uint64_t). |
| 232 | */ |
| 233 | uint32_t TotalSize; |
| 234 | /* |
| 235 | *The number of value profile kinds that has value profile data. |
| 236 | * In this implementation, a value profile kind is considered to |
| 237 | * have profile data if the number of value profile sites for the |
| 238 | * kind is not zero. More aggressively, the implementation can |
| 239 | * choose to check the actual data value: if none of the value sites |
| 240 | * has any profiled values, the kind can be skipped. |
| 241 | */ |
| 242 | uint32_t NumValueKinds; |
| 243 | |
| 244 | /* |
| 245 | * Following are a sequence of variable length records. The prefix/header |
| 246 | * of each record is defined by ValueProfRecord type. The number of |
| 247 | * records is NumValueKinds. |
| 248 | * ValueProfRecord Record_1; |
| 249 | * ValueProfRecord Record_N; |
| 250 | */ |
| 251 | |
| 252 | #if __cplusplus |
| 253 | /*! |
| 254 | * Return the total size in bytes of the on-disk value profile data |
| 255 | * given the data stored in Record. |
| 256 | */ |
| 257 | static uint32_t getSize(const InstrProfRecord &Record); |
| 258 | /*! |
| 259 | * Return a pointer to \c ValueProfData instance ready to be streamed. |
| 260 | */ |
| 261 | static std::unique_ptr<ValueProfData> |
| 262 | serializeFrom(const InstrProfRecord &Record); |
| 263 | /*! |
| 264 | * Check the integrity of the record. Return the error code when |
| 265 | * an error is detected, otherwise return instrprof_error::success. |
| 266 | */ |
| 267 | instrprof_error checkIntegrity(); |
| 268 | /*! |
| 269 | * Return a pointer to \c ValueProfileData instance ready to be read. |
| 270 | * All data in the instance are properly byte swapped. The input |
| 271 | * data is assumed to be in little endian order. |
| 272 | */ |
| 273 | static ErrorOr<std::unique_ptr<ValueProfData>> |
| 274 | getValueProfData(const unsigned char *SrcBuffer, |
| 275 | const unsigned char *const SrcBufferEnd, |
| 276 | support::endianness SrcDataEndianness); |
| 277 | /*! |
| 278 | * Swap byte order from \c Endianness order to host byte order. |
| 279 | */ |
| 280 | void swapBytesToHost(support::endianness Endianness); |
| 281 | /*! |
| 282 | * Swap byte order from host byte order to \c Endianness order. |
| 283 | */ |
| 284 | void swapBytesFromHost(support::endianness Endianness); |
| 285 | /*! |
| 286 | * Return the total size of \c ValueProfileData. |
| 287 | */ |
| 288 | uint32_t getSize() const { return TotalSize; } |
| 289 | /*! |
| 290 | * Read data from this data and save it to \c Record. |
| 291 | */ |
| 292 | void deserializeTo(InstrProfRecord &Record, |
| 293 | InstrProfRecord::ValueMapType *VMap); |
| 294 | void operator delete(void *ptr) { ::operator delete(ptr); } |
| 295 | #endif |
| 296 | } ValueProfData; |
| 297 | |
| 298 | /* |
| 299 | * The closure is designed to abstact away two types of value profile data: |
| 300 | * - InstrProfRecord which is the primary data structure used to |
| 301 | * represent profile data in host tools (reader, writer, and profile-use) |
| 302 | * - value profile runtime data structure suitable to be used by C |
| 303 | * runtime library. |
| 304 | * |
| 305 | * Both sources of data need to serialize to disk/memory-buffer in common |
| 306 | * format: ValueProfData. The abstraction allows compiler-rt's raw profiler |
| 307 | * writer to share the same format and code with indexed profile writer. |
| 308 | * |
| 309 | * For documentation of the member methods below, refer to corresponding methods |
| 310 | * in class InstrProfRecord. |
| 311 | */ |
| 312 | typedef struct ValueProfRecordClosure { |
| 313 | const void *Record; |
| 314 | uint32_t (*GetNumValueKinds)(const void *Record); |
| 315 | uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); |
| 316 | uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); |
| 317 | uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); |
| 318 | |
| 319 | /* |
| 320 | * After extracting the value profile data from the value profile record, |
| 321 | * this method is used to map the in-memory value to on-disk value. If |
| 322 | * the method is null, value will be written out untranslated. |
| 323 | */ |
| 324 | uint64_t (*RemapValueData)(uint32_t, uint64_t Value); |
| 325 | void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, |
| 326 | uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)); |
| 327 | ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); |
| 328 | } ValueProfRecordClosure; |
| 329 | |
| 330 | /* |
| 331 | * A wrapper struct that represents value profile runtime data. |
| 332 | * Like InstrProfRecord class which is used by profiling host tools, |
| 333 | * ValueProfRuntimeRecord also implements the abstract intefaces defined in |
| 334 | * ValueProfRecordClosure so that the runtime data can be serialized using |
| 335 | * shared C implementation. In this structure, NumValueSites and Nodes |
| 336 | * members are the primary fields while other fields hold the derived |
| 337 | * information for fast implementation of closure interfaces. |
| 338 | */ |
| 339 | typedef struct ValueProfRuntimeRecord { |
| 340 | /* Number of sites for each value profile kind. */ |
| 341 | const uint16_t *NumValueSites; |
| 342 | /* An array of linked-list headers. The size of of the array is the |
| 343 | * total number of value profile sites : sum(NumValueSites[*])). Each |
| 344 | * linked-list stores the values profiled for a value profile site. */ |
| 345 | ValueProfNode **Nodes; |
| 346 | |
| 347 | /* Total number of value profile kinds which have at least one |
| 348 | * value profile sites. */ |
| 349 | uint32_t NumValueKinds; |
| 350 | /* An array recording the number of values tracked at each site. |
| 351 | * The size of the array is TotalNumValueSites. */ |
| 352 | uint8_t *SiteCountArray[IPVK_Last + 1]; |
| 353 | ValueProfNode **NodesKind[IPVK_Last + 1]; |
| 354 | } ValueProfRuntimeRecord; |
| 355 | |
| 356 | /* Forward declarations of C interfaces. */ |
| 357 | int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, |
| 358 | const uint16_t *NumValueSites, |
| 359 | ValueProfNode **Nodes); |
| 360 | void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); |
| 361 | uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); |
| 362 | ValueProfData * |
| 363 | serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, |
| 364 | ValueProfData *Dst); |
| 365 | uint32_t getNumValueKindsRT(const void *R); |
| 366 | |
| 367 | #undef INSTR_PROF_VALUE_PROF_DATA |
| 368 | #endif /* INSTR_PROF_VALUE_PROF_DATA */ |
| 369 | |
| 370 | |
| 371 | #ifdef INSTR_PROF_COMMON_API_IMPL |
| 372 | #define INSTR_PROF_DATA_DEFINED |
| 373 | #ifdef __cplusplus |
| 374 | #define INSTR_PROF_INLINE inline |
| 375 | #else |
| 376 | #define INSTR_PROF_INLINE |
| 377 | #endif |
| 378 | |
| 379 | #ifndef offsetof |
| 380 | #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) |
| 381 | #endif |
| 382 | |
| 383 | /*! |
| 384 | * \brief Return the \c ValueProfRecord header size including the |
| 385 | * padding bytes. |
| 386 | */ |
| 387 | INSTR_PROF_INLINE |
| 388 | uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { |
| 389 | uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + |
| 390 | sizeof(uint8_t) * NumValueSites; |
| 391 | /* Round the size to multiple of 8 bytes. */ |
| 392 | Size = (Size + 7) & ~7; |
| 393 | return Size; |
| 394 | } |
| 395 | |
| 396 | /*! |
| 397 | * \brief Return the total size of the value profile record including the |
| 398 | * header and the value data. |
| 399 | */ |
| 400 | INSTR_PROF_INLINE |
| 401 | uint32_t getValueProfRecordSize(uint32_t NumValueSites, |
| 402 | uint32_t NumValueData) { |
| 403 | return getValueProfRecordHeaderSize(NumValueSites) + |
| 404 | sizeof(InstrProfValueData) * NumValueData; |
| 405 | } |
| 406 | |
| 407 | /*! |
| 408 | * \brief Return the pointer to the start of value data array. |
| 409 | */ |
| 410 | INSTR_PROF_INLINE |
| 411 | InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { |
| 412 | return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( |
| 413 | This->NumValueSites)); |
| 414 | } |
| 415 | |
| 416 | /*! |
| 417 | * \brief Return the total number of value data for \c This record. |
| 418 | */ |
| 419 | INSTR_PROF_INLINE |
| 420 | uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { |
| 421 | uint32_t NumValueData = 0; |
| 422 | uint32_t I; |
| 423 | for (I = 0; I < This->NumValueSites; I++) |
| 424 | NumValueData += This->SiteCountArray[I]; |
| 425 | return NumValueData; |
| 426 | } |
| 427 | |
| 428 | /*! |
| 429 | * \brief Use this method to advance to the next \c This \c ValueProfRecord. |
| 430 | */ |
| 431 | INSTR_PROF_INLINE |
| 432 | ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { |
| 433 | uint32_t NumValueData = getValueProfRecordNumValueData(This); |
| 434 | return (ValueProfRecord *)((char *)This + |
| 435 | getValueProfRecordSize(This->NumValueSites, |
| 436 | NumValueData)); |
| 437 | } |
| 438 | |
| 439 | /*! |
| 440 | * \brief Return the first \c ValueProfRecord instance. |
| 441 | */ |
| 442 | INSTR_PROF_INLINE |
| 443 | ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { |
| 444 | return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); |
| 445 | } |
| 446 | |
| 447 | /* Closure based interfaces. */ |
| 448 | |
| 449 | /*! |
| 450 | * Return the total size in bytes of the on-disk value profile data |
| 451 | * given the data stored in Record. |
| 452 | */ |
| 453 | uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { |
| 454 | uint32_t Kind; |
| 455 | uint32_t TotalSize = sizeof(ValueProfData); |
| 456 | const void *Record = Closure->Record; |
| 457 | uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); |
| 458 | if (NumValueKinds == 0) |
| 459 | return TotalSize; |
| 460 | |
| 461 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
| 462 | uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); |
| 463 | if (!NumValueSites) |
| 464 | continue; |
| 465 | TotalSize += getValueProfRecordSize(NumValueSites, |
| 466 | Closure->GetNumValueData(Record, Kind)); |
| 467 | } |
| 468 | return TotalSize; |
| 469 | } |
| 470 | |
| 471 | /*! |
| 472 | * Extract value profile data of a function for the profile kind \c ValueKind |
| 473 | * from the \c Closure and serialize the data into \c This record instance. |
| 474 | */ |
| 475 | void serializeValueProfRecordFrom(ValueProfRecord *This, |
| 476 | ValueProfRecordClosure *Closure, |
| 477 | uint32_t ValueKind, uint32_t NumValueSites) { |
| 478 | uint32_t S; |
| 479 | const void *Record = Closure->Record; |
| 480 | This->Kind = ValueKind; |
| 481 | This->NumValueSites = NumValueSites; |
| 482 | InstrProfValueData *DstVD = getValueProfRecordValueData(This); |
| 483 | |
| 484 | for (S = 0; S < NumValueSites; S++) { |
| 485 | uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); |
| 486 | This->SiteCountArray[S] = ND; |
| 487 | Closure->GetValueForSite(Record, DstVD, ValueKind, S, |
| 488 | Closure->RemapValueData); |
| 489 | DstVD += ND; |
| 490 | } |
| 491 | } |
| 492 | |
| 493 | /*! |
| 494 | * Extract value profile data of a function from the \c Closure |
| 495 | * and serialize the data into \c DstData if it is not NULL or heap |
| 496 | * memory allocated by the \c Closure's allocator method. |
| 497 | */ |
| 498 | ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, |
| 499 | ValueProfData *DstData) { |
| 500 | uint32_t Kind; |
| 501 | uint32_t TotalSize = getValueProfDataSize(Closure); |
| 502 | |
| 503 | ValueProfData *VPD = |
| 504 | DstData ? DstData : Closure->AllocValueProfData(TotalSize); |
| 505 | |
| 506 | VPD->TotalSize = TotalSize; |
| 507 | VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); |
| 508 | ValueProfRecord *VR = getFirstValueProfRecord(VPD); |
| 509 | for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { |
| 510 | uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); |
| 511 | if (!NumValueSites) |
| 512 | continue; |
| 513 | serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); |
| 514 | VR = getValueProfRecordNext(VR); |
| 515 | } |
| 516 | return VPD; |
| 517 | } |
| 518 | |
| 519 | /* |
| 520 | * The value profiler runtime library stores the value profile data |
| 521 | * for a given function in \c NumValueSites and \c Nodes structures. |
| 522 | * \c ValueProfRuntimeRecord class is used to encapsulate the runtime |
| 523 | * profile data and provides fast interfaces to retrieve the profile |
| 524 | * information. This interface is used to initialize the runtime record |
| 525 | * and pre-compute the information needed for efficient implementation |
| 526 | * of callbacks required by ValueProfRecordClosure class. |
| 527 | */ |
| 528 | int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, |
| 529 | const uint16_t *NumValueSites, |
| 530 | ValueProfNode **Nodes) { |
| 531 | unsigned I, J, S = 0, NumValueKinds = 0; |
| 532 | RuntimeRecord->NumValueSites = NumValueSites; |
| 533 | RuntimeRecord->Nodes = Nodes; |
| 534 | for (I = 0; I <= IPVK_Last; I++) { |
| 535 | uint16_t N = NumValueSites[I]; |
| 536 | if (!N) { |
| 537 | RuntimeRecord->SiteCountArray[I] = 0; |
| 538 | continue; |
| 539 | } |
| 540 | NumValueKinds++; |
| 541 | RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); |
| 542 | if (!RuntimeRecord->SiteCountArray[I]) |
| 543 | return 1; |
| 544 | RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL; |
| 545 | for (J = 0; J < N; J++) { |
| 546 | /* Compute value count for each site. */ |
| 547 | uint32_t C = 0; |
| 548 | ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL; |
| 549 | while (Site) { |
| 550 | C++; |
| 551 | Site = Site->Next; |
| 552 | } |
| 553 | if (C > UCHAR_MAX) |
| 554 | C = UCHAR_MAX; |
| 555 | RuntimeRecord->SiteCountArray[I][J] = C; |
| 556 | } |
| 557 | S += N; |
| 558 | } |
| 559 | RuntimeRecord->NumValueKinds = NumValueKinds; |
| 560 | return 0; |
| 561 | } |
| 562 | |
| 563 | void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { |
| 564 | unsigned I; |
| 565 | for (I = 0; I <= IPVK_Last; I++) { |
| 566 | if (RuntimeRecord->SiteCountArray[I]) |
| 567 | free(RuntimeRecord->SiteCountArray[I]); |
| 568 | } |
| 569 | } |
| 570 | |
| 571 | /* ValueProfRecordClosure Interface implementation for |
| 572 | * ValueProfDataRuntimeRecord. */ |
| 573 | uint32_t getNumValueKindsRT(const void *R) { |
| 574 | return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; |
| 575 | } |
| 576 | |
| 577 | uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { |
| 578 | return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; |
| 579 | } |
| 580 | |
| 581 | uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { |
| 582 | const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; |
| 583 | return Record->SiteCountArray[VK][S]; |
| 584 | } |
| 585 | |
| 586 | uint32_t getNumValueDataRT(const void *R, uint32_t VK) { |
| 587 | unsigned I, S = 0; |
| 588 | const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; |
| 589 | if (Record->SiteCountArray[VK] == 0) |
| 590 | return 0; |
| 591 | for (I = 0; I < Record->NumValueSites[VK]; I++) |
| 592 | S += Record->SiteCountArray[VK][I]; |
| 593 | return S; |
| 594 | } |
| 595 | |
| 596 | void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, |
| 597 | uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { |
| 598 | unsigned I, N = 0; |
| 599 | const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; |
| 600 | N = getNumValueDataForSiteRT(R, VK, S); |
| 601 | if (N == 0) |
| 602 | return; |
| 603 | ValueProfNode *VNode = Record->NodesKind[VK][S]; |
| 604 | for (I = 0; I < N; I++) { |
| 605 | Dst[I] = VNode->VData; |
| 606 | VNode = VNode->Next; |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { |
| 611 | return (ValueProfData *)calloc(TotalSizeInBytes, 1); |
| 612 | } |
| 613 | |
| 614 | static ValueProfRecordClosure RTRecordClosure = {0, |
| 615 | getNumValueKindsRT, |
| 616 | getNumValueSitesRT, |
| 617 | getNumValueDataRT, |
| 618 | getNumValueDataForSiteRT, |
| 619 | 0, |
| 620 | getValueForSiteRT, |
| 621 | allocValueProfDataRT}; |
| 622 | |
| 623 | /* |
| 624 | * Return the size of ValueProfData structure to store data |
| 625 | * recorded in the runtime record. |
| 626 | */ |
| 627 | uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { |
| 628 | RTRecordClosure.Record = Record; |
| 629 | return getValueProfDataSize(&RTRecordClosure); |
| 630 | } |
| 631 | |
| 632 | /* |
| 633 | * Return a ValueProfData instance that stores the data collected |
| 634 | * from runtime. If \c DstData is provided by the caller, the value |
| 635 | * profile data will be store in *DstData and DstData is returned, |
| 636 | * otherwise the method will allocate space for the value data and |
| 637 | * return pointer to the newly allocated space. |
| 638 | */ |
| 639 | ValueProfData * |
| 640 | serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, |
| 641 | ValueProfData *DstData) { |
| 642 | RTRecordClosure.Record = Record; |
| 643 | return serializeValueProfDataFrom(&RTRecordClosure, DstData); |
| 644 | } |
| 645 | |
| 646 | |
| 647 | #undef INSTR_PROF_COMMON_API_IMPL |
| 648 | #endif /* INSTR_PROF_COMMON_API_IMPL */ |
| 649 | |
| 650 | /*============================================================================*/ |
| 651 | |
| 652 | |
| 653 | #ifndef INSTR_PROF_DATA_DEFINED |
| 654 | |
| 655 | #ifndef INSTR_PROF_DATA_INC_ |
| 656 | #define INSTR_PROF_DATA_INC_ |
| 657 | |
| 658 | /* Helper macros. */ |
| 659 | #define INSTR_PROF_SIMPLE_QUOTE(x) #x |
| 660 | #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) |
| 661 | #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y |
| 662 | #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) |
| 663 | |
| 664 | /* Magic number to detect file format and endianness. |
| 665 | * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, |
| 666 | * so that utilities, like strings, don't grab it as a string. 129 is also |
| 667 | * invalid UTF-8, and high enough to be interesting. |
| 668 | * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" |
| 669 | * for 32-bit platforms. |
| 670 | */ |
| 671 | #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
| 672 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
| 673 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 |
| 674 | #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ |
| 675 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ |
| 676 | (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 |
| 677 | |
| 678 | /* Raw profile format version. */ |
| 679 | #define INSTR_PROF_RAW_VERSION 2 |
| 680 | |
| 681 | /* Runtime section names and name strings. */ |
| 682 | #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data |
| 683 | #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names |
| 684 | #define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts |
| 685 | |
| 686 | #define INSTR_PROF_DATA_SECT_NAME_STR \ |
| 687 | INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) |
| 688 | #define INSTR_PROF_NAME_SECT_NAME_STR \ |
| 689 | INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) |
| 690 | #define INSTR_PROF_CNTS_SECT_NAME_STR \ |
| 691 | INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) |
| 692 | |
| 693 | /* Macros to define start/stop section symbol for a given |
| 694 | * section on Linux. For instance |
| 695 | * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will |
| 696 | * expand to __start___llvm_prof_data |
| 697 | */ |
| 698 | #define INSTR_PROF_SECT_START(Sect) \ |
| 699 | INSTR_PROF_CONCAT(__start_,Sect) |
| 700 | #define INSTR_PROF_SECT_STOP(Sect) \ |
| 701 | INSTR_PROF_CONCAT(__stop_,Sect) |
| 702 | |
| 703 | /* Value Profiling API linkage name. */ |
| 704 | #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target |
| 705 | #define INSTR_PROF_VALUE_PROF_FUNC_STR \ |
| 706 | INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) |
| 707 | |
| 708 | /* InstrProfile per-function control data alignment. */ |
| 709 | #define INSTR_PROF_DATA_ALIGNMENT 8 |
| 710 | |
| 711 | /* The data structure that represents a tracked value by the |
| 712 | * value profiler. |
| 713 | */ |
| 714 | typedef struct InstrProfValueData { |
| 715 | /* Profiled value. */ |
| 716 | uint64_t Value; |
| 717 | /* Number of times the value appears in the training run. */ |
| 718 | uint64_t Count; |
| 719 | } InstrProfValueData; |
| 720 | |
| 721 | /* This is an internal data structure used by value profiler. It |
| 722 | * is defined here to allow serialization code sharing by LLVM |
| 723 | * to be used in unit test. |
| 724 | */ |
| 725 | typedef struct ValueProfNode { |
| 726 | InstrProfValueData VData; |
| 727 | struct ValueProfNode *Next; |
| 728 | } ValueProfNode; |
| 729 | |
| 730 | #endif /* INSTR_PROF_DATA_INC_ */ |
| 731 | |
| 732 | #else |
| 733 | #undef INSTR_PROF_DATA_DEFINED |
| 734 | #endif |
| 735 | |