blob: 9b7dd400fc92e540929453ab8cfc3527d644ecd1 [file] [log] [blame]
Andrew Trick153ebe62013-10-31 22:11:56 +00001//===---------------------------- StackMaps.cpp ---------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
Andrew Trick153ebe62013-10-31 22:11:56 +000010#include "llvm/CodeGen/StackMaps.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000011#include "llvm/CodeGen/AsmPrinter.h"
Juergen Ributzkafb4d6482014-01-30 18:58:27 +000012#include "llvm/CodeGen/MachineFrameInfo.h"
Chandler Carruth442f7842014-03-04 10:07:28 +000013#include "llvm/CodeGen/MachineFunction.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000014#include "llvm/CodeGen/MachineInstr.h"
Lang Hames39609992013-11-29 03:07:54 +000015#include "llvm/IR/DataLayout.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000016#include "llvm/MC/MCContext.h"
17#include "llvm/MC/MCExpr.h"
Lang Hames8a065702013-11-08 22:30:52 +000018#include "llvm/MC/MCObjectFileInfo.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000019#include "llvm/MC/MCSectionMachO.h"
20#include "llvm/MC/MCStreamer.h"
Juergen Ributzka673a7622014-05-01 22:21:30 +000021#include "llvm/Support/CommandLine.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000022#include "llvm/Target/TargetMachine.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000023#include "llvm/Target/TargetOpcodes.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000024#include "llvm/Target/TargetRegisterInfo.h"
Eric Christopherd9134482014-08-04 21:25:23 +000025#include "llvm/Target/TargetSubtargetInfo.h"
Andrew Trick153ebe62013-10-31 22:11:56 +000026#include <iterator>
27
28using namespace llvm;
29
Chandler Carruth1b9dde02014-04-22 02:02:50 +000030#define DEBUG_TYPE "stackmaps"
31
Juergen Ributzkad25407e2015-07-08 22:42:09 +000032static cl::opt<int> StackMapVersion(
Sanjoy Das23f06e52016-09-14 20:22:03 +000033 "stackmap-version", cl::init(2),
34 cl::desc("Specify the stackmap encoding version (default = 2)"));
Juergen Ributzka673a7622014-05-01 22:21:30 +000035
Juergen Ributzka37fc0a82014-05-01 22:39:26 +000036const char *StackMaps::WSMP = "Stack Maps: ";
37
Philip Reamese83c4b32016-08-23 23:33:29 +000038StackMapOpers::StackMapOpers(const MachineInstr *MI)
39 : MI(MI) {
Philip Reames570dd002016-08-23 21:21:43 +000040 assert(getVarIdx() <= MI->getNumOperands() &&
41 "invalid stackmap definition");
42}
43
Juergen Ributzkac26b68a2013-12-14 23:06:19 +000044PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
Juergen Ributzkad25407e2015-07-08 22:42:09 +000045 : MI(MI), HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
Philip Reamesd06a1b42016-08-23 23:58:08 +000046 !MI->getOperand(0).isImplicit()) {
Andrew Trickd4e3dc62013-11-19 03:29:56 +000047#ifndef NDEBUG
Andrew Trickd4e3dc62013-11-19 03:29:56 +000048 unsigned CheckStartIdx = 0, e = MI->getNumOperands();
49 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
50 MI->getOperand(CheckStartIdx).isDef() &&
51 !MI->getOperand(CheckStartIdx).isImplicit())
52 ++CheckStartIdx;
53
54 assert(getMetaIdx() == CheckStartIdx &&
Alp Tokercb402912014-01-24 17:20:08 +000055 "Unexpected additional definition in Patchpoint intrinsic.");
Andrew Trickd4e3dc62013-11-19 03:29:56 +000056#endif
57}
58
59unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
60 if (!StartIdx)
61 StartIdx = getVarIdx();
62
63 // Find the next scratch register (implicit def and early clobber)
64 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
65 while (ScratchIdx < e &&
66 !(MI->getOperand(ScratchIdx).isReg() &&
67 MI->getOperand(ScratchIdx).isDef() &&
68 MI->getOperand(ScratchIdx).isImplicit() &&
69 MI->getOperand(ScratchIdx).isEarlyClobber()))
70 ++ScratchIdx;
71
72 assert(ScratchIdx != e && "No scratch register available");
73 return ScratchIdx;
74}
75
Juergen Ributzka673a7622014-05-01 22:21:30 +000076StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
Sanjoy Das23f06e52016-09-14 20:22:03 +000077 if (StackMapVersion != 2)
Juergen Ributzka673a7622014-05-01 22:21:30 +000078 llvm_unreachable("Unsupported stackmap version!");
79}
80
Eric Christophercef8e712015-03-20 16:03:42 +000081/// Go up the super-register chain until we hit a valid dwarf register number.
82static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +000083 int RegNum = TRI->getDwarfRegNum(Reg, false);
84 for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNum < 0; ++SR)
85 RegNum = TRI->getDwarfRegNum(*SR, false);
Eric Christophercef8e712015-03-20 16:03:42 +000086
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +000087 assert(RegNum >= 0 && "Invalid Dwarf register number.");
88 return (unsigned)RegNum;
Eric Christophercef8e712015-03-20 16:03:42 +000089}
90
Juergen Ributzkac26b68a2013-12-14 23:06:19 +000091MachineInstr::const_mop_iterator
Lang Hames39609992013-11-29 03:07:54 +000092StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
Juergen Ributzkad25407e2015-07-08 22:42:09 +000093 MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
94 LiveOutVec &LiveOuts) const {
Eric Christophercef8e712015-03-20 16:03:42 +000095 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
Juergen Ributzkac26b68a2013-12-14 23:06:19 +000096 if (MOI->isImm()) {
97 switch (MOI->getImm()) {
Juergen Ributzkad25407e2015-07-08 22:42:09 +000098 default:
99 llvm_unreachable("Unrecognized operand type.");
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000100 case StackMaps::DirectMemRefOp: {
Mehdi Aminibd7287e2015-07-16 06:11:10 +0000101 auto &DL = AP.MF->getDataLayout();
102
103 unsigned Size = DL.getPointerSizeInBits();
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000104 assert((Size % 8) == 0 && "Need pointer size in bytes.");
105 Size /= 8;
106 unsigned Reg = (++MOI)->getReg();
107 int64_t Imm = (++MOI)->getImm();
Juergen Ributzkae4685a12015-07-09 17:11:08 +0000108 Locs.emplace_back(StackMaps::Location::Direct, Size,
109 getDwarfRegNum(Reg, TRI), Imm);
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000110 break;
Lang Hames39609992013-11-29 03:07:54 +0000111 }
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000112 case StackMaps::IndirectMemRefOp: {
113 int64_t Size = (++MOI)->getImm();
114 assert(Size > 0 && "Need a valid size for indirect memory locations.");
115 unsigned Reg = (++MOI)->getReg();
116 int64_t Imm = (++MOI)->getImm();
Juergen Ributzkae4685a12015-07-09 17:11:08 +0000117 Locs.emplace_back(StackMaps::Location::Indirect, Size,
118 getDwarfRegNum(Reg, TRI), Imm);
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000119 break;
120 }
121 case StackMaps::ConstantOp: {
122 ++MOI;
123 assert(MOI->isImm() && "Expected constant operand.");
124 int64_t Imm = MOI->getImm();
Juergen Ributzkae4685a12015-07-09 17:11:08 +0000125 Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, Imm);
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000126 break;
127 }
128 }
129 return ++MOI;
Lang Hames39609992013-11-29 03:07:54 +0000130 }
131
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000132 // The physical register number will ultimately be encoded as a DWARF regno.
133 // The stack map also records the size of a spill slot that can hold the
134 // register content. (The runtime can track the actual size of the data type
135 // if it needs to.)
136 if (MOI->isReg()) {
137 // Skip implicit registers (this includes our scratch registers)
138 if (MOI->isImplicit())
139 return ++MOI;
Juergen Ributzkae8294752013-12-14 06:53:06 +0000140
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000141 assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) &&
142 "Virtreg operands should have been rewritten before now.");
Eric Christophercef8e712015-03-20 16:03:42 +0000143 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(MOI->getReg());
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000144 assert(!MOI->getSubReg() && "Physical subreg still around.");
Eric Christophercef8e712015-03-20 16:03:42 +0000145
146 unsigned Offset = 0;
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000147 unsigned DwarfRegNum = getDwarfRegNum(MOI->getReg(), TRI);
148 unsigned LLVMRegNum = TRI->getLLVMRegNum(DwarfRegNum, false);
149 unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNum, MOI->getReg());
Eric Christophercef8e712015-03-20 16:03:42 +0000150 if (SubRegIdx)
151 Offset = TRI->getSubRegIdxOffset(SubRegIdx);
152
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000153 Locs.emplace_back(Location::Register, RC->getSize(), DwarfRegNum, Offset);
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000154 return ++MOI;
155 }
156
157 if (MOI->isRegLiveOut())
158 LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
159
160 return ++MOI;
Lang Hames39609992013-11-29 03:07:54 +0000161}
162
Eric Christophercef8e712015-03-20 16:03:42 +0000163void StackMaps::print(raw_ostream &OS) {
164 const TargetRegisterInfo *TRI =
165 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr;
166 OS << WSMP << "callsites:\n";
167 for (const auto &CSI : CSInfos) {
168 const LocationVec &CSLocs = CSI.Locations;
169 const LiveOutVec &LiveOuts = CSI.LiveOuts;
Juergen Ributzkae8294752013-12-14 06:53:06 +0000170
Eric Christophercef8e712015-03-20 16:03:42 +0000171 OS << WSMP << "callsite " << CSI.ID << "\n";
172 OS << WSMP << " has " << CSLocs.size() << " locations\n";
173
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000174 unsigned Idx = 0;
Eric Christophercef8e712015-03-20 16:03:42 +0000175 for (const auto &Loc : CSLocs) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000176 OS << WSMP << "\t\tLoc " << Idx << ": ";
177 switch (Loc.Type) {
Eric Christophercef8e712015-03-20 16:03:42 +0000178 case Location::Unprocessed:
179 OS << "<Unprocessed operand>";
180 break;
181 case Location::Register:
182 OS << "Register ";
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000183 if (TRI)
184 OS << TRI->getName(Loc.Reg);
185 else
186 OS << Loc.Reg;
Eric Christophercef8e712015-03-20 16:03:42 +0000187 break;
188 case Location::Direct:
189 OS << "Direct ";
190 if (TRI)
191 OS << TRI->getName(Loc.Reg);
192 else
193 OS << Loc.Reg;
194 if (Loc.Offset)
195 OS << " + " << Loc.Offset;
196 break;
197 case Location::Indirect:
198 OS << "Indirect ";
199 if (TRI)
200 OS << TRI->getName(Loc.Reg);
201 else
202 OS << Loc.Reg;
203 OS << "+" << Loc.Offset;
204 break;
205 case Location::Constant:
206 OS << "Constant " << Loc.Offset;
207 break;
208 case Location::ConstantIndex:
209 OS << "Constant Index " << Loc.Offset;
210 break;
211 }
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000212 OS << "\t[encoding: .byte " << Loc.Type << ", .byte " << Loc.Size
Eric Christophercef8e712015-03-20 16:03:42 +0000213 << ", .short " << Loc.Reg << ", .int " << Loc.Offset << "]\n";
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000214 Idx++;
Eric Christophercef8e712015-03-20 16:03:42 +0000215 }
216
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000217 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
Eric Christophercef8e712015-03-20 16:03:42 +0000218
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000219 Idx = 0;
Eric Christophercef8e712015-03-20 16:03:42 +0000220 for (const auto &LO : LiveOuts) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000221 OS << WSMP << "\t\tLO " << Idx << ": ";
Eric Christophercef8e712015-03-20 16:03:42 +0000222 if (TRI)
223 OS << TRI->getName(LO.Reg);
224 else
225 OS << LO.Reg;
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000226 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
Eric Christophercef8e712015-03-20 16:03:42 +0000227 << LO.Size << "]\n";
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000228 Idx++;
Eric Christophercef8e712015-03-20 16:03:42 +0000229 }
230 }
Juergen Ributzkae8294752013-12-14 06:53:06 +0000231}
232
233/// Create a live-out register record for the given register Reg.
234StackMaps::LiveOutReg
Juergen Ributzka73a7fcc2014-02-10 23:30:26 +0000235StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000236 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
Juergen Ributzkae8294752013-12-14 06:53:06 +0000237 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000238 return LiveOutReg(Reg, DwarfRegNum, Size);
Juergen Ributzkae8294752013-12-14 06:53:06 +0000239}
240
241/// Parse the register live-out mask and return a vector of live-out registers
242/// that need to be recorded in the stackmap.
243StackMaps::LiveOutVec
244StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
245 assert(Mask && "No register mask specified");
Eric Christopherd83003e2015-03-19 23:27:42 +0000246 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
Juergen Ributzkae8294752013-12-14 06:53:06 +0000247 LiveOutVec LiveOuts;
248
249 // Create a LiveOutReg for each bit that is set in the register mask.
250 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
251 if ((Mask[Reg / 32] >> Reg % 32) & 1)
Juergen Ributzka73a7fcc2014-02-10 23:30:26 +0000252 LiveOuts.push_back(createLiveOutReg(Reg, TRI));
Juergen Ributzkae8294752013-12-14 06:53:06 +0000253
254 // We don't need to keep track of a register if its super-register is already
255 // in the list. Merge entries that refer to the same dwarf register and use
256 // the maximum size that needs to be spilled.
Juergen Ributzka216ed032015-07-09 17:11:15 +0000257
258 std::sort(LiveOuts.begin(), LiveOuts.end(),
259 [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
260 // Only sort by the dwarf register number.
261 return LHS.DwarfRegNum < RHS.DwarfRegNum;
262 });
263
264 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
265 for (auto II = std::next(I); II != E; ++II) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000266 if (I->DwarfRegNum != II->DwarfRegNum) {
Juergen Ributzkae8294752013-12-14 06:53:06 +0000267 // Skip all the now invalid entries.
268 I = --II;
269 break;
270 }
271 I->Size = std::max(I->Size, II->Size);
272 if (TRI->isSuperRegister(I->Reg, II->Reg))
273 I->Reg = II->Reg;
Juergen Ributzka216ed032015-07-09 17:11:15 +0000274 II->Reg = 0; // mark for deletion.
Juergen Ributzkae8294752013-12-14 06:53:06 +0000275 }
276 }
Juergen Ributzka216ed032015-07-09 17:11:15 +0000277
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000278 LiveOuts.erase(
David Majnemerc7004902016-08-12 04:32:37 +0000279 remove_if(LiveOuts, [](const LiveOutReg &LO) { return LO.Reg == 0; }),
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000280 LiveOuts.end());
Juergen Ributzka216ed032015-07-09 17:11:15 +0000281
Juergen Ributzkae8294752013-12-14 06:53:06 +0000282 return LiveOuts;
283}
284
Andrew Tricke8cba372013-12-13 18:37:10 +0000285void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000286 MachineInstr::const_mop_iterator MOI,
287 MachineInstr::const_mop_iterator MOE,
288 bool recordResult) {
Andrew Trick153ebe62013-10-31 22:11:56 +0000289
Lang Hames9ff69c82015-04-24 19:11:51 +0000290 MCContext &OutContext = AP.OutStreamer->getContext();
Jim Grosbach6f482002015-05-18 18:43:14 +0000291 MCSymbol *MILabel = OutContext.createTempSymbol();
Lang Hames9ff69c82015-04-24 19:11:51 +0000292 AP.OutStreamer->EmitLabel(MILabel);
Andrew Trick153ebe62013-10-31 22:11:56 +0000293
Juergen Ributzkae8294752013-12-14 06:53:06 +0000294 LocationVec Locations;
295 LiveOutVec LiveOuts;
Andrew Trick153ebe62013-10-31 22:11:56 +0000296
Juergen Ributzka9969d3e2013-11-08 23:28:16 +0000297 if (recordResult) {
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000298 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000299 parseOperand(MI.operands_begin(), std::next(MI.operands_begin()), Locations,
300 LiveOuts);
Juergen Ributzka9969d3e2013-11-08 23:28:16 +0000301 }
302
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000303 // Parse operands.
Andrew Trick153ebe62013-10-31 22:11:56 +0000304 while (MOI != MOE) {
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000305 MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
306 }
Andrew Trick153ebe62013-10-31 22:11:56 +0000307
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000308 // Move large constants into the constant pool.
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000309 for (auto &Loc : Locations) {
Andrew Trick32e1be72014-01-09 00:22:31 +0000310 // Constants are encoded as sign-extended integers.
311 // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000312 if (Loc.Type == Location::Constant && !isInt<32>(Loc.Offset)) {
313 Loc.Type = Location::ConstantIndex;
Sanjoy Dase8399652014-11-04 00:59:21 +0000314 // ConstPool is intentionally a MapVector of 'uint64_t's (as
315 // opposed to 'int64_t's). We should never be in a situation
316 // where we have to insert either the tombstone or the empty
317 // keys into a map, and for a DenseMap<uint64_t, T> these are
318 // (uint64_t)0 and (uint64_t)-1. They can be and are
319 // represented using 32 bit integers.
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000320 assert((uint64_t)Loc.Offset != DenseMapInfo<uint64_t>::getEmptyKey() &&
321 (uint64_t)Loc.Offset !=
322 DenseMapInfo<uint64_t>::getTombstoneKey() &&
Sanjoy Dase8399652014-11-04 00:59:21 +0000323 "empty and tombstone keys should fit in 32 bits!");
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000324 auto Result = ConstPool.insert(std::make_pair(Loc.Offset, Loc.Offset));
325 Loc.Offset = Result.first - ConstPool.begin();
Andrew Trick153ebe62013-10-31 22:11:56 +0000326 }
Andrew Trick153ebe62013-10-31 22:11:56 +0000327 }
328
Juergen Ributzkafb4d6482014-01-30 18:58:27 +0000329 // Create an expression to calculate the offset of the callsite from function
330 // entry.
Jim Grosbach13760bd2015-05-30 01:25:56 +0000331 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000332 MCSymbolRefExpr::create(MILabel, OutContext),
333 MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext);
Andrew Trick153ebe62013-10-31 22:11:56 +0000334
Benjamin Kramerc6cc58e2014-10-04 16:55:56 +0000335 CSInfos.emplace_back(CSOffsetExpr, ID, std::move(Locations),
336 std::move(LiveOuts));
Juergen Ributzkafb4d6482014-01-30 18:58:27 +0000337
Sanjoy Das23f06e52016-09-14 20:22:03 +0000338 // Record the stack size of the current function and update callsite count.
Matthias Braun941a7052016-07-28 18:40:00 +0000339 const MachineFrameInfo &MFI = AP.MF->getFrameInfo();
Eric Christopherfc6de422014-08-05 02:39:49 +0000340 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000341 bool HasDynamicFrameSize =
Matthias Braun941a7052016-07-28 18:40:00 +0000342 MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(*(AP.MF));
Sanjoy Das23f06e52016-09-14 20:22:03 +0000343 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize();
344
345 auto CurrentIt = FnInfos.find(AP.CurrentFnSym);
346 if (CurrentIt != FnInfos.end())
347 CurrentIt->second.RecordCount++;
348 else
349 FnInfos.insert(std::make_pair(AP.CurrentFnSym, FunctionInfo(FrameSize)));
Andrew Trick153ebe62013-10-31 22:11:56 +0000350}
351
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000352void StackMaps::recordStackMap(const MachineInstr &MI) {
Juergen Ributzkae8294752013-12-14 06:53:06 +0000353 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000354
Philip Reames570dd002016-08-23 21:21:43 +0000355 StackMapOpers opers(&MI);
356 const int64_t ID = MI.getOperand(PatchPointOpers::IDPos).getImm();
357 recordStackMapOpers(MI, ID, std::next(MI.operands_begin(), opers.getVarIdx()),
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000358 MI.operands_end());
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000359}
360
361void StackMaps::recordPatchPoint(const MachineInstr &MI) {
Juergen Ributzkae8294752013-12-14 06:53:06 +0000362 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000363
364 PatchPointOpers opers(&MI);
Philip Reamese83c4b32016-08-23 23:33:29 +0000365 const int64_t ID = opers.getID();
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000366 auto MOI = std::next(MI.operands_begin(), opers.getStackMapStartIdx());
Juergen Ributzkac26b68a2013-12-14 23:06:19 +0000367 recordStackMapOpers(MI, ID, MOI, MI.operands_end(),
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000368 opers.isAnyReg() && opers.hasDef());
369
370#ifndef NDEBUG
371 // verify anyregcc
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000372 auto &Locations = CSInfos.back().Locations;
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000373 if (opers.isAnyReg()) {
Philip Reamese83c4b32016-08-23 23:33:29 +0000374 unsigned NArgs = opers.getNumCallArgs();
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000375 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000376 assert(Locations[i].Type == Location::Register &&
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000377 "anyreg arg must be in reg.");
378 }
379#endif
380}
Philip Reames0365f1a2014-12-01 22:52:56 +0000381void StackMaps::recordStatepoint(const MachineInstr &MI) {
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000382 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
Philip Reames0365f1a2014-12-01 22:52:56 +0000383
384 StatepointOpers opers(&MI);
385 // Record all the deopt and gc operands (they're contiguous and run from the
386 // initial index to the end of the operand list)
387 const unsigned StartIdx = opers.getVarIdx();
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000388 recordStackMapOpers(MI, opers.getID(), MI.operands_begin() + StartIdx,
389 MI.operands_end(), false);
Philip Reames0365f1a2014-12-01 22:52:56 +0000390}
Andrew Trickd4e3dc62013-11-19 03:29:56 +0000391
Juergen Ributzka63401952014-05-01 22:21:27 +0000392/// Emit the stackmap header.
Andrew Trick153ebe62013-10-31 22:11:56 +0000393///
Juergen Ributzkae1179922014-03-31 22:14:04 +0000394/// Header {
Sanjoy Das23f06e52016-09-14 20:22:03 +0000395/// uint8 : Stack Map Version (currently 2)
Juergen Ributzkae1179922014-03-31 22:14:04 +0000396/// uint8 : Reserved (expected to be 0)
397/// uint16 : Reserved (expected to be 0)
Juergen Ributzkafb4d6482014-01-30 18:58:27 +0000398/// }
Juergen Ributzkae1179922014-03-31 22:14:04 +0000399/// uint32 : NumFunctions
Andrew Trick153ebe62013-10-31 22:11:56 +0000400/// uint32 : NumConstants
Andrew Trick153ebe62013-10-31 22:11:56 +0000401/// uint32 : NumRecords
Juergen Ributzka63401952014-05-01 22:21:27 +0000402void StackMaps::emitStackmapHeader(MCStreamer &OS) {
403 // Header.
Juergen Ributzka673a7622014-05-01 22:21:30 +0000404 OS.EmitIntValue(StackMapVersion, 1); // Version.
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000405 OS.EmitIntValue(0, 1); // Reserved.
406 OS.EmitIntValue(0, 2); // Reserved.
Juergen Ributzka63401952014-05-01 22:21:27 +0000407
408 // Num functions.
Sanjoy Das23f06e52016-09-14 20:22:03 +0000409 DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n');
410 OS.EmitIntValue(FnInfos.size(), 4);
Juergen Ributzka63401952014-05-01 22:21:27 +0000411 // Num constants.
412 DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
413 OS.EmitIntValue(ConstPool.size(), 4);
414 // Num callsites.
415 DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
416 OS.EmitIntValue(CSInfos.size(), 4);
417}
418
419/// Emit the function frame record for each function.
420///
Juergen Ributzkae1179922014-03-31 22:14:04 +0000421/// StkSizeRecord[NumFunctions] {
422/// uint64 : Function Address
423/// uint64 : Stack Size
Sanjoy Das23f06e52016-09-14 20:22:03 +0000424/// uint64 : Record Count
Juergen Ributzkae1179922014-03-31 22:14:04 +0000425/// }
Juergen Ributzka63401952014-05-01 22:21:27 +0000426void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
427 // Function Frame records.
428 DEBUG(dbgs() << WSMP << "functions:\n");
Sanjoy Das23f06e52016-09-14 20:22:03 +0000429 for (auto const &FR : FnInfos) {
Juergen Ributzka63401952014-05-01 22:21:27 +0000430 DEBUG(dbgs() << WSMP << "function addr: " << FR.first
Sanjoy Das23f06e52016-09-14 20:22:03 +0000431 << " frame size: " << FR.second.StackSize
432 << " callsite count: " << FR.second.RecordCount << '\n');
Juergen Ributzka63401952014-05-01 22:21:27 +0000433 OS.EmitSymbolValue(FR.first, 8);
Sanjoy Das23f06e52016-09-14 20:22:03 +0000434 OS.EmitIntValue(FR.second.StackSize, 8);
435 OS.EmitIntValue(FR.second.RecordCount, 8);
Juergen Ributzka63401952014-05-01 22:21:27 +0000436 }
437}
438
439/// Emit the constant pool.
440///
Juergen Ributzkae1179922014-03-31 22:14:04 +0000441/// int64 : Constants[NumConstants]
Juergen Ributzka63401952014-05-01 22:21:27 +0000442void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
443 // Constant pool entries.
444 DEBUG(dbgs() << WSMP << "constants:\n");
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000445 for (const auto &ConstEntry : ConstPool) {
Juergen Ributzka63401952014-05-01 22:21:27 +0000446 DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
447 OS.EmitIntValue(ConstEntry.second, 8);
448 }
449}
450
451/// Emit the callsite info for each callsite.
452///
Andrew Trick153ebe62013-10-31 22:11:56 +0000453/// StkMapRecord[NumRecords] {
Andrew Tricke8cba372013-12-13 18:37:10 +0000454/// uint64 : PatchPoint ID
Andrew Trick153ebe62013-10-31 22:11:56 +0000455/// uint32 : Instruction Offset
456/// uint16 : Reserved (record flags)
457/// uint16 : NumLocations
458/// Location[NumLocations] {
459/// uint8 : Register | Direct | Indirect | Constant | ConstantIndex
Andrew Trick10d5be42013-11-17 01:36:23 +0000460/// uint8 : Size in Bytes
Andrew Trick153ebe62013-10-31 22:11:56 +0000461/// uint16 : Dwarf RegNum
462/// int32 : Offset
463/// }
Juergen Ributzkae1179922014-03-31 22:14:04 +0000464/// uint16 : Padding
Juergen Ributzkae8294752013-12-14 06:53:06 +0000465/// uint16 : NumLiveOuts
Juergen Ributzkae1179922014-03-31 22:14:04 +0000466/// LiveOuts[NumLiveOuts] {
Juergen Ributzkae8294752013-12-14 06:53:06 +0000467/// uint16 : Dwarf RegNum
468/// uint8 : Reserved
469/// uint8 : Size in Bytes
Juergen Ributzkae1179922014-03-31 22:14:04 +0000470/// }
471/// uint32 : Padding (only if required to align to 8 byte)
Andrew Trick153ebe62013-10-31 22:11:56 +0000472/// }
473///
474/// Location Encoding, Type, Value:
475/// 0x1, Register, Reg (value in register)
476/// 0x2, Direct, Reg + Offset (frame index)
477/// 0x3, Indirect, [Reg + Offset] (spilled value)
478/// 0x4, Constant, Offset (small constant)
479/// 0x5, ConstIndex, Constants[Offset] (large constant)
Eric Christopher594fa96a2015-03-20 21:05:18 +0000480void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
Eric Christophercef8e712015-03-20 16:03:42 +0000481 DEBUG(print(dbgs()));
Juergen Ributzkae1179922014-03-31 22:14:04 +0000482 // Callsite entries.
Juergen Ributzka63401952014-05-01 22:21:27 +0000483 for (const auto &CSI : CSInfos) {
484 const LocationVec &CSLocs = CSI.Locations;
485 const LiveOutVec &LiveOuts = CSI.LiveOuts;
Andrew Trick153ebe62013-10-31 22:11:56 +0000486
Andrew Trick153ebe62013-10-31 22:11:56 +0000487 // Verify stack map entry. It's better to communicate a problem to the
488 // runtime than crash in case of in-process compilation. Currently, we do
489 // simple overflow checks, but we may eventually communicate other
490 // compilation errors this way.
Juergen Ributzkae8294752013-12-14 06:53:06 +0000491 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
Juergen Ributzka63401952014-05-01 22:21:27 +0000492 OS.EmitIntValue(UINT64_MAX, 8); // Invalid ID.
493 OS.EmitValue(CSI.CSOffsetExpr, 4);
494 OS.EmitIntValue(0, 2); // Reserved.
495 OS.EmitIntValue(0, 2); // 0 locations.
496 OS.EmitIntValue(0, 2); // padding.
497 OS.EmitIntValue(0, 2); // 0 live-out registers.
498 OS.EmitIntValue(0, 4); // padding.
Andrew Trick153ebe62013-10-31 22:11:56 +0000499 continue;
500 }
501
Juergen Ributzka63401952014-05-01 22:21:27 +0000502 OS.EmitIntValue(CSI.ID, 8);
503 OS.EmitValue(CSI.CSOffsetExpr, 4);
Andrew Trick153ebe62013-10-31 22:11:56 +0000504
505 // Reserved for flags.
Juergen Ributzka63401952014-05-01 22:21:27 +0000506 OS.EmitIntValue(0, 2);
Juergen Ributzka63401952014-05-01 22:21:27 +0000507 OS.EmitIntValue(CSLocs.size(), 2);
Andrew Trick153ebe62013-10-31 22:11:56 +0000508
Juergen Ributzka63401952014-05-01 22:21:27 +0000509 for (const auto &Loc : CSLocs) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000510 OS.EmitIntValue(Loc.Type, 1);
Juergen Ributzka63401952014-05-01 22:21:27 +0000511 OS.EmitIntValue(Loc.Size, 1);
Eric Christophercef8e712015-03-20 16:03:42 +0000512 OS.EmitIntValue(Loc.Reg, 2);
513 OS.EmitIntValue(Loc.Offset, 4);
Andrew Trick153ebe62013-10-31 22:11:56 +0000514 }
Juergen Ributzkae8294752013-12-14 06:53:06 +0000515
Juergen Ributzkae1179922014-03-31 22:14:04 +0000516 // Num live-out registers and padding to align to 4 byte.
Juergen Ributzka63401952014-05-01 22:21:27 +0000517 OS.EmitIntValue(0, 2);
518 OS.EmitIntValue(LiveOuts.size(), 2);
Juergen Ributzkae8294752013-12-14 06:53:06 +0000519
Juergen Ributzka63401952014-05-01 22:21:27 +0000520 for (const auto &LO : LiveOuts) {
Juergen Ributzkaaef76ca2015-07-09 17:11:11 +0000521 OS.EmitIntValue(LO.DwarfRegNum, 2);
Juergen Ributzka63401952014-05-01 22:21:27 +0000522 OS.EmitIntValue(0, 1);
523 OS.EmitIntValue(LO.Size, 1);
Juergen Ributzkae8294752013-12-14 06:53:06 +0000524 }
Juergen Ributzkae1179922014-03-31 22:14:04 +0000525 // Emit alignment to 8 byte.
Juergen Ributzka63401952014-05-01 22:21:27 +0000526 OS.EmitValueToAlignment(8);
Andrew Trick153ebe62013-10-31 22:11:56 +0000527 }
Juergen Ributzka63401952014-05-01 22:21:27 +0000528}
Andrew Trick153ebe62013-10-31 22:11:56 +0000529
Juergen Ributzka63401952014-05-01 22:21:27 +0000530/// Serialize the stackmap data.
531void StackMaps::serializeToStackMapSection() {
Juergen Ributzkad25407e2015-07-08 22:42:09 +0000532 (void)WSMP;
Juergen Ributzka63401952014-05-01 22:21:27 +0000533 // Bail out if there's no stack map data.
Hans Wennborge9134892016-04-11 20:35:01 +0000534 assert((!CSInfos.empty() || ConstPool.empty()) &&
Juergen Ributzka63401952014-05-01 22:21:27 +0000535 "Expected empty constant pool too!");
Sanjoy Das23f06e52016-09-14 20:22:03 +0000536 assert((!CSInfos.empty() || FnInfos.empty()) &&
Juergen Ributzka63401952014-05-01 22:21:27 +0000537 "Expected empty function record too!");
538 if (CSInfos.empty())
539 return;
Andrew Trick153ebe62013-10-31 22:11:56 +0000540
Lang Hames9ff69c82015-04-24 19:11:51 +0000541 MCContext &OutContext = AP.OutStreamer->getContext();
542 MCStreamer &OS = *AP.OutStreamer;
Juergen Ributzka63401952014-05-01 22:21:27 +0000543
544 // Create the section.
Rafael Espindola0709a7b2015-05-21 19:20:38 +0000545 MCSection *StackMapSection =
546 OutContext.getObjectFileInfo()->getStackMapSection();
Juergen Ributzka63401952014-05-01 22:21:27 +0000547 OS.SwitchSection(StackMapSection);
548
549 // Emit a dummy symbol to force section inclusion.
Jim Grosbach6f482002015-05-18 18:43:14 +0000550 OS.EmitLabel(OutContext.getOrCreateSymbol(Twine("__LLVM_StackMaps")));
Juergen Ributzka63401952014-05-01 22:21:27 +0000551
552 // Serialize data.
553 DEBUG(dbgs() << "********** Stack Map Output **********\n");
554 emitStackmapHeader(OS);
555 emitFunctionFrameRecords(OS);
556 emitConstantPoolEntries(OS);
Eric Christopher594fa96a2015-03-20 21:05:18 +0000557 emitCallsiteEntries(OS);
Juergen Ributzka63401952014-05-01 22:21:27 +0000558 OS.AddBlankLine();
559
560 // Clean up.
Andrew Trick153ebe62013-10-31 22:11:56 +0000561 CSInfos.clear();
Juergen Ributzkaf01e8092014-05-01 22:21:24 +0000562 ConstPool.clear();
Andrew Trick153ebe62013-10-31 22:11:56 +0000563}