blob: 773e51c6e34ac9b0ef64dff144b7d6c0a156f753 [file] [log] [blame]
Tim Northover69fa84a2016-10-14 22:18:18 +00001//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
Tim Northover33b07d62016-07-22 20:03:43 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
Tim Northover69fa84a2016-10-14 22:18:18 +000010/// \file This file implements the LegalizerHelper class to legalize
Tim Northover33b07d62016-07-22 20:03:43 +000011/// individual instructions and the LegalizeMachineIR wrapper pass for the
12/// primary legalization.
13//
14//===----------------------------------------------------------------------===//
15
Tim Northover69fa84a2016-10-14 22:18:18 +000016#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
Tim Northoveredb3c8c2016-08-29 19:07:16 +000017#include "llvm/CodeGen/GlobalISel/CallLowering.h"
Tim Northover69fa84a2016-10-14 22:18:18 +000018#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000019#include "llvm/CodeGen/MachineRegisterInfo.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000020#include "llvm/CodeGen/TargetLowering.h"
21#include "llvm/CodeGen/TargetSubtargetInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000022#include "llvm/Support/Debug.h"
23#include "llvm/Support/raw_ostream.h"
Tim Northover33b07d62016-07-22 20:03:43 +000024
Tim Northover33b07d62016-07-22 20:03:43 +000025
Daniel Sanders5377fb32017-04-20 15:46:12 +000026#define DEBUG_TYPE "legalizer"
Tim Northover33b07d62016-07-22 20:03:43 +000027
28using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000029using namespace LegalizeActions;
Tim Northover33b07d62016-07-22 20:03:43 +000030
Tim Northover69fa84a2016-10-14 22:18:18 +000031LegalizerHelper::LegalizerHelper(MachineFunction &MF)
Volkan Keles685fbda2017-03-10 18:34:57 +000032 : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
Tim Northover33b07d62016-07-22 20:03:43 +000033 MIRBuilder.setMF(MF);
34}
35
Tim Northover69fa84a2016-10-14 22:18:18 +000036LegalizerHelper::LegalizeResult
Volkan Keles685fbda2017-03-10 18:34:57 +000037LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
Daniel Sanders5377fb32017-04-20 15:46:12 +000038 DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
39
Daniel Sanders262ed0e2018-01-24 17:17:46 +000040 auto Step = LI.getAction(MI, MRI);
41 switch (Step.Action) {
Daniel Sanders9ade5592018-01-29 17:37:29 +000042 case Legal:
Daniel Sanders5377fb32017-04-20 15:46:12 +000043 DEBUG(dbgs() << ".. Already legal\n");
Tim Northover33b07d62016-07-22 20:03:43 +000044 return AlreadyLegal;
Daniel Sanders9ade5592018-01-29 17:37:29 +000045 case Libcall:
Daniel Sanders5377fb32017-04-20 15:46:12 +000046 DEBUG(dbgs() << ".. Convert to libcall\n");
Tim Northoveredb3c8c2016-08-29 19:07:16 +000047 return libcall(MI);
Daniel Sanders9ade5592018-01-29 17:37:29 +000048 case NarrowScalar:
Daniel Sanders5377fb32017-04-20 15:46:12 +000049 DEBUG(dbgs() << ".. Narrow scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +000050 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +000051 case WidenScalar:
Daniel Sanders5377fb32017-04-20 15:46:12 +000052 DEBUG(dbgs() << ".. Widen scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +000053 return widenScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +000054 case Lower:
Daniel Sanders5377fb32017-04-20 15:46:12 +000055 DEBUG(dbgs() << ".. Lower\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +000056 return lower(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +000057 case FewerElements:
Daniel Sanders5377fb32017-04-20 15:46:12 +000058 DEBUG(dbgs() << ".. Reduce number of elements\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +000059 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +000060 case Custom:
Daniel Sanders5377fb32017-04-20 15:46:12 +000061 DEBUG(dbgs() << ".. Custom legalization\n");
Volkan Keles685fbda2017-03-10 18:34:57 +000062 return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
63 : UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +000064 default:
Daniel Sanders5377fb32017-04-20 15:46:12 +000065 DEBUG(dbgs() << ".. Unable to legalize\n");
Tim Northover33b07d62016-07-22 20:03:43 +000066 return UnableToLegalize;
67 }
68}
69
Tim Northover69fa84a2016-10-14 22:18:18 +000070void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
71 SmallVectorImpl<unsigned> &VRegs) {
Tim Northoverbf017292017-03-03 22:46:09 +000072 for (int i = 0; i < NumParts; ++i)
Tim Northover0f140c72016-09-09 11:46:34 +000073 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
Tim Northoverbf017292017-03-03 22:46:09 +000074 MIRBuilder.buildUnmerge(VRegs, Reg);
Tim Northover33b07d62016-07-22 20:03:43 +000075}
76
Tim Northovere0418412017-02-08 23:23:39 +000077static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
78 switch (Opcode) {
Diana Picuse97822e2017-04-24 07:22:31 +000079 case TargetOpcode::G_SDIV:
80 assert(Size == 32 && "Unsupported size");
81 return RTLIB::SDIV_I32;
82 case TargetOpcode::G_UDIV:
83 assert(Size == 32 && "Unsupported size");
84 return RTLIB::UDIV_I32;
Diana Picus02e11012017-06-15 10:53:31 +000085 case TargetOpcode::G_SREM:
86 assert(Size == 32 && "Unsupported size");
87 return RTLIB::SREM_I32;
88 case TargetOpcode::G_UREM:
89 assert(Size == 32 && "Unsupported size");
90 return RTLIB::UREM_I32;
Diana Picus1314a282017-04-11 10:52:34 +000091 case TargetOpcode::G_FADD:
92 assert((Size == 32 || Size == 64) && "Unsupported size");
93 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
Javed Absar5cde1cc2017-10-30 13:51:56 +000094 case TargetOpcode::G_FSUB:
95 assert((Size == 32 || Size == 64) && "Unsupported size");
96 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
Diana Picus9faa09b2017-11-23 12:44:20 +000097 case TargetOpcode::G_FMUL:
98 assert((Size == 32 || Size == 64) && "Unsupported size");
99 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
Diana Picusc01f7f12017-11-23 13:26:07 +0000100 case TargetOpcode::G_FDIV:
101 assert((Size == 32 || Size == 64) && "Unsupported size");
102 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000103 case TargetOpcode::G_FREM:
104 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
105 case TargetOpcode::G_FPOW:
106 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
Diana Picuse74243d2018-01-12 11:30:45 +0000107 case TargetOpcode::G_FMA:
108 assert((Size == 32 || Size == 64) && "Unsupported size");
109 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000110 }
111 llvm_unreachable("Unknown libcall function");
112}
113
Diana Picusfc1675e2017-07-05 12:57:24 +0000114LegalizerHelper::LegalizeResult
115llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
116 const CallLowering::ArgInfo &Result,
117 ArrayRef<CallLowering::ArgInfo> Args) {
Diana Picuse97822e2017-04-24 07:22:31 +0000118 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
119 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
Diana Picuse97822e2017-04-24 07:22:31 +0000120 const char *Name = TLI.getLibcallName(Libcall);
Diana Picusd0104ea2017-07-06 09:09:33 +0000121
Diana Picuse97822e2017-04-24 07:22:31 +0000122 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
Diana Picus02e11012017-06-15 10:53:31 +0000123 if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
124 MachineOperand::CreateES(Name), Result, Args))
125 return LegalizerHelper::UnableToLegalize;
Diana Picusd0104ea2017-07-06 09:09:33 +0000126
Diana Picuse97822e2017-04-24 07:22:31 +0000127 return LegalizerHelper::Legalized;
128}
129
Diana Picus65ed3642018-01-17 13:34:10 +0000130// Useful for libcalls where all operands have the same type.
Diana Picus02e11012017-06-15 10:53:31 +0000131static LegalizerHelper::LegalizeResult
132simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
133 Type *OpType) {
134 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
Diana Picuse74243d2018-01-12 11:30:45 +0000135
136 SmallVector<CallLowering::ArgInfo, 3> Args;
137 for (unsigned i = 1; i < MI.getNumOperands(); i++)
138 Args.push_back({MI.getOperand(i).getReg(), OpType});
Diana Picusfc1675e2017-07-05 12:57:24 +0000139 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
Diana Picuse74243d2018-01-12 11:30:45 +0000140 Args);
Diana Picus02e11012017-06-15 10:53:31 +0000141}
142
Diana Picus65ed3642018-01-17 13:34:10 +0000143static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
144 Type *FromType) {
145 auto ToMVT = MVT::getVT(ToType);
146 auto FromMVT = MVT::getVT(FromType);
147
148 switch (Opcode) {
149 case TargetOpcode::G_FPEXT:
150 return RTLIB::getFPEXT(FromMVT, ToMVT);
151 case TargetOpcode::G_FPTRUNC:
152 return RTLIB::getFPROUND(FromMVT, ToMVT);
Diana Picus4ed0ee72018-01-30 07:54:52 +0000153 case TargetOpcode::G_FPTOSI:
154 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
155 case TargetOpcode::G_FPTOUI:
156 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
Diana Picus517531e2018-01-30 09:15:17 +0000157 case TargetOpcode::G_SITOFP:
158 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
159 case TargetOpcode::G_UITOFP:
160 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
Diana Picus65ed3642018-01-17 13:34:10 +0000161 }
162 llvm_unreachable("Unsupported libcall function");
163}
164
165static LegalizerHelper::LegalizeResult
166conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
167 Type *FromType) {
168 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
169 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
170 {{MI.getOperand(1).getReg(), FromType}});
171}
172
Tim Northover69fa84a2016-10-14 22:18:18 +0000173LegalizerHelper::LegalizeResult
174LegalizerHelper::libcall(MachineInstr &MI) {
Diana Picus02e11012017-06-15 10:53:31 +0000175 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
176 unsigned Size = LLTy.getSizeInBits();
Matthias Braunf1caa282017-12-15 22:22:58 +0000177 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000178
Diana Picusfc1675e2017-07-05 12:57:24 +0000179 MIRBuilder.setInstr(MI);
180
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000181 switch (MI.getOpcode()) {
182 default:
183 return UnableToLegalize;
Diana Picuse97822e2017-04-24 07:22:31 +0000184 case TargetOpcode::G_SDIV:
Diana Picus02e11012017-06-15 10:53:31 +0000185 case TargetOpcode::G_UDIV:
186 case TargetOpcode::G_SREM:
187 case TargetOpcode::G_UREM: {
188 Type *HLTy = Type::getInt32Ty(Ctx);
Diana Picusfc1675e2017-07-05 12:57:24 +0000189 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
190 if (Status != Legalized)
191 return Status;
192 break;
Diana Picuse97822e2017-04-24 07:22:31 +0000193 }
Diana Picus1314a282017-04-11 10:52:34 +0000194 case TargetOpcode::G_FADD:
Javed Absar5cde1cc2017-10-30 13:51:56 +0000195 case TargetOpcode::G_FSUB:
Diana Picus9faa09b2017-11-23 12:44:20 +0000196 case TargetOpcode::G_FMUL:
Diana Picusc01f7f12017-11-23 13:26:07 +0000197 case TargetOpcode::G_FDIV:
Diana Picuse74243d2018-01-12 11:30:45 +0000198 case TargetOpcode::G_FMA:
Tim Northovere0418412017-02-08 23:23:39 +0000199 case TargetOpcode::G_FPOW:
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000200 case TargetOpcode::G_FREM: {
Diana Picus02e11012017-06-15 10:53:31 +0000201 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
Diana Picusfc1675e2017-07-05 12:57:24 +0000202 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
203 if (Status != Legalized)
204 return Status;
205 break;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000206 }
Diana Picus65ed3642018-01-17 13:34:10 +0000207 case TargetOpcode::G_FPEXT: {
208 // FIXME: Support other floating point types (half, fp128 etc)
209 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
210 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
211 if (ToSize != 64 || FromSize != 32)
212 return UnableToLegalize;
213 LegalizeResult Status = conversionLibcall(
214 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
215 if (Status != Legalized)
216 return Status;
217 break;
218 }
219 case TargetOpcode::G_FPTRUNC: {
220 // FIXME: Support other floating point types (half, fp128 etc)
221 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
222 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
223 if (ToSize != 32 || FromSize != 64)
224 return UnableToLegalize;
225 LegalizeResult Status = conversionLibcall(
226 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
227 if (Status != Legalized)
228 return Status;
229 break;
230 }
Diana Picus4ed0ee72018-01-30 07:54:52 +0000231 case TargetOpcode::G_FPTOSI:
232 case TargetOpcode::G_FPTOUI: {
233 // FIXME: Support other types
234 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
235 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
236 if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
237 return UnableToLegalize;
238 LegalizeResult Status = conversionLibcall(
239 MI, MIRBuilder, Type::getInt32Ty(Ctx),
240 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
241 if (Status != Legalized)
242 return Status;
243 break;
244 }
Diana Picus517531e2018-01-30 09:15:17 +0000245 case TargetOpcode::G_SITOFP:
246 case TargetOpcode::G_UITOFP: {
247 // FIXME: Support other types
248 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
249 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
250 if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
251 return UnableToLegalize;
252 LegalizeResult Status = conversionLibcall(
253 MI, MIRBuilder,
254 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
255 Type::getInt32Ty(Ctx));
256 if (Status != Legalized)
257 return Status;
258 break;
259 }
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000260 }
Diana Picusfc1675e2017-07-05 12:57:24 +0000261
262 MI.eraseFromParent();
263 return Legalized;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000264}
265
Tim Northover69fa84a2016-10-14 22:18:18 +0000266LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
267 unsigned TypeIdx,
268 LLT NarrowTy) {
Quentin Colombet5e60bcd2016-08-27 02:38:21 +0000269 // FIXME: Don't know how to handle secondary types yet.
Tim Northoverc2d5e6d2017-06-26 20:34:13 +0000270 if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
Quentin Colombet5e60bcd2016-08-27 02:38:21 +0000271 return UnableToLegalize;
Justin Bognerfde01042017-01-18 17:29:54 +0000272
273 MIRBuilder.setInstr(MI);
274
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000275 int64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
276 int64_t NarrowSize = NarrowTy.getSizeInBits();
277
Tim Northover9656f142016-08-04 20:54:13 +0000278 switch (MI.getOpcode()) {
279 default:
280 return UnableToLegalize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000281 case TargetOpcode::G_IMPLICIT_DEF: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000282 // FIXME: add support for when SizeOp0 isn't an exact multiple of
283 // NarrowSize.
284 if (SizeOp0 % NarrowSize != 0)
285 return UnableToLegalize;
286 int NumParts = SizeOp0 / NarrowSize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000287
288 SmallVector<unsigned, 2> DstRegs;
289 for (int i = 0; i < NumParts; ++i) {
290 unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
291 MIRBuilder.buildUndef(Dst);
292 DstRegs.push_back(Dst);
293 }
294 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
295 MI.eraseFromParent();
296 return Legalized;
297 }
Tim Northover9656f142016-08-04 20:54:13 +0000298 case TargetOpcode::G_ADD: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000299 // FIXME: add support for when SizeOp0 isn't an exact multiple of
300 // NarrowSize.
301 if (SizeOp0 % NarrowSize != 0)
302 return UnableToLegalize;
Tim Northover9656f142016-08-04 20:54:13 +0000303 // Expand in terms of carry-setting/consuming G_ADDE instructions.
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000304 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
Tim Northover9656f142016-08-04 20:54:13 +0000305
Tim Northoverb18ea162016-09-20 15:20:36 +0000306 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
Tim Northover9656f142016-08-04 20:54:13 +0000307 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
308 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
309
Tim Northover0f140c72016-09-09 11:46:34 +0000310 unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
311 MIRBuilder.buildConstant(CarryIn, 0);
Tim Northover9656f142016-08-04 20:54:13 +0000312
313 for (int i = 0; i < NumParts; ++i) {
Tim Northover0f140c72016-09-09 11:46:34 +0000314 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
315 unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Tim Northover9656f142016-08-04 20:54:13 +0000316
Tim Northover0f140c72016-09-09 11:46:34 +0000317 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
Tim Northover91c81732016-08-19 17:17:06 +0000318 Src2Regs[i], CarryIn);
Tim Northover9656f142016-08-04 20:54:13 +0000319
320 DstRegs.push_back(DstReg);
321 CarryIn = CarryOut;
322 }
Tim Northover0f140c72016-09-09 11:46:34 +0000323 unsigned DstReg = MI.getOperand(0).getReg();
Tim Northoverbf017292017-03-03 22:46:09 +0000324 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northover9656f142016-08-04 20:54:13 +0000325 MI.eraseFromParent();
326 return Legalized;
327 }
Tim Northoverc2d5e6d2017-06-26 20:34:13 +0000328 case TargetOpcode::G_EXTRACT: {
329 if (TypeIdx != 1)
330 return UnableToLegalize;
331
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000332 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
333 // FIXME: add support for when SizeOp1 isn't an exact multiple of
334 // NarrowSize.
335 if (SizeOp1 % NarrowSize != 0)
336 return UnableToLegalize;
337 int NumParts = SizeOp1 / NarrowSize;
Tim Northoverc2d5e6d2017-06-26 20:34:13 +0000338
339 SmallVector<unsigned, 2> SrcRegs, DstRegs;
340 SmallVector<uint64_t, 2> Indexes;
341 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
342
343 unsigned OpReg = MI.getOperand(0).getReg();
344 int64_t OpStart = MI.getOperand(2).getImm();
345 int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
346 for (int i = 0; i < NumParts; ++i) {
347 unsigned SrcStart = i * NarrowSize;
348
349 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
350 // No part of the extract uses this subregister, ignore it.
351 continue;
352 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
353 // The entire subregister is extracted, forward the value.
354 DstRegs.push_back(SrcRegs[i]);
355 continue;
356 }
357
358 // OpSegStart is where this destination segment would start in OpReg if it
359 // extended infinitely in both directions.
360 int64_t ExtractOffset, SegSize;
361 if (OpStart < SrcStart) {
362 ExtractOffset = 0;
363 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
364 } else {
365 ExtractOffset = OpStart - SrcStart;
366 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
367 }
368
369 unsigned SegReg = SrcRegs[i];
370 if (ExtractOffset != 0 || SegSize != NarrowSize) {
371 // A genuine extract is needed.
372 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
373 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
374 }
375
376 DstRegs.push_back(SegReg);
377 }
378
379 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
380 MI.eraseFromParent();
381 return Legalized;
382 }
Tim Northover0e6afbd2017-02-06 21:56:47 +0000383 case TargetOpcode::G_INSERT: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000384 // FIXME: add support for when SizeOp0 isn't an exact multiple of
385 // NarrowSize.
386 if (SizeOp0 % NarrowSize != 0)
Tim Northover0e6afbd2017-02-06 21:56:47 +0000387 return UnableToLegalize;
388
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000389 int NumParts = SizeOp0 / NarrowSize;
Tim Northover0e6afbd2017-02-06 21:56:47 +0000390
391 SmallVector<unsigned, 2> SrcRegs, DstRegs;
392 SmallVector<uint64_t, 2> Indexes;
393 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
394
Tim Northover75e0b912017-03-06 18:23:04 +0000395 unsigned OpReg = MI.getOperand(2).getReg();
396 int64_t OpStart = MI.getOperand(3).getImm();
397 int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
Tim Northover0e6afbd2017-02-06 21:56:47 +0000398 for (int i = 0; i < NumParts; ++i) {
399 unsigned DstStart = i * NarrowSize;
Tim Northover0e6afbd2017-02-06 21:56:47 +0000400
Tim Northover75e0b912017-03-06 18:23:04 +0000401 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
Tim Northover0e6afbd2017-02-06 21:56:47 +0000402 // No part of the insert affects this subregister, forward the original.
403 DstRegs.push_back(SrcRegs[i]);
404 continue;
Tim Northover75e0b912017-03-06 18:23:04 +0000405 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
Tim Northover0e6afbd2017-02-06 21:56:47 +0000406 // The entire subregister is defined by this insert, forward the new
407 // value.
Tim Northover75e0b912017-03-06 18:23:04 +0000408 DstRegs.push_back(OpReg);
Tim Northover0e6afbd2017-02-06 21:56:47 +0000409 continue;
410 }
411
Tim Northover2eb18d32017-03-07 21:24:33 +0000412 // OpSegStart is where this destination segment would start in OpReg if it
413 // extended infinitely in both directions.
414 int64_t ExtractOffset, InsertOffset, SegSize;
415 if (OpStart < DstStart) {
416 InsertOffset = 0;
417 ExtractOffset = DstStart - OpStart;
418 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
419 } else {
420 InsertOffset = OpStart - DstStart;
421 ExtractOffset = 0;
422 SegSize =
423 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
424 }
425
426 unsigned SegReg = OpReg;
427 if (ExtractOffset != 0 || SegSize != OpSize) {
Tim Northover75e0b912017-03-06 18:23:04 +0000428 // A genuine extract is needed.
Tim Northover2eb18d32017-03-07 21:24:33 +0000429 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
430 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
Tim Northover0e6afbd2017-02-06 21:56:47 +0000431 }
432
Tim Northover75e0b912017-03-06 18:23:04 +0000433 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Tim Northover2eb18d32017-03-07 21:24:33 +0000434 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
Tim Northover0e6afbd2017-02-06 21:56:47 +0000435 DstRegs.push_back(DstReg);
436 }
437
438 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
Tim Northoverbf017292017-03-03 22:46:09 +0000439 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
Tim Northover0e6afbd2017-02-06 21:56:47 +0000440 MI.eraseFromParent();
441 return Legalized;
442 }
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000443 case TargetOpcode::G_LOAD: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000444 // FIXME: add support for when SizeOp0 isn't an exact multiple of
445 // NarrowSize.
446 if (SizeOp0 % NarrowSize != 0)
447 return UnableToLegalize;
448 int NumParts = SizeOp0 / NarrowSize;
Daniel Sanders4e523662017-06-13 23:42:32 +0000449 LLT OffsetTy = LLT::scalar(
450 MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000451
452 SmallVector<unsigned, 2> DstRegs;
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000453 for (int i = 0; i < NumParts; ++i) {
454 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Daniel Sanders4e523662017-06-13 23:42:32 +0000455 unsigned SrcReg = 0;
456 unsigned Adjustment = i * NarrowSize / 8;
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000457
Daniel Sanders4e523662017-06-13 23:42:32 +0000458 MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
459 Adjustment);
460
Justin Bognere094cc42017-01-20 00:30:17 +0000461 // TODO: This is conservatively correct, but we probably want to split the
462 // memory operands in the future.
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000463 MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
464
465 DstRegs.push_back(DstReg);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000466 }
467 unsigned DstReg = MI.getOperand(0).getReg();
Tim Northoverbf017292017-03-03 22:46:09 +0000468 MIRBuilder.buildMerge(DstReg, DstRegs);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000469 MI.eraseFromParent();
470 return Legalized;
471 }
Justin Bognerfde01042017-01-18 17:29:54 +0000472 case TargetOpcode::G_STORE: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000473 // FIXME: add support for when SizeOp0 isn't an exact multiple of
474 // NarrowSize.
475 if (SizeOp0 % NarrowSize != 0)
476 return UnableToLegalize;
477 int NumParts = SizeOp0 / NarrowSize;
Daniel Sanders4e523662017-06-13 23:42:32 +0000478 LLT OffsetTy = LLT::scalar(
479 MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
Justin Bognerfde01042017-01-18 17:29:54 +0000480
481 SmallVector<unsigned, 2> SrcRegs;
482 extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
483
484 for (int i = 0; i < NumParts; ++i) {
Daniel Sanders4e523662017-06-13 23:42:32 +0000485 unsigned DstReg = 0;
486 unsigned Adjustment = i * NarrowSize / 8;
487
488 MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
489 Adjustment);
490
Justin Bognere094cc42017-01-20 00:30:17 +0000491 // TODO: This is conservatively correct, but we probably want to split the
492 // memory operands in the future.
Justin Bognerfde01042017-01-18 17:29:54 +0000493 MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
494 }
495 MI.eraseFromParent();
496 return Legalized;
497 }
Igor Breger29537882017-04-07 14:41:59 +0000498 case TargetOpcode::G_CONSTANT: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000499 // FIXME: add support for when SizeOp0 isn't an exact multiple of
500 // NarrowSize.
501 if (SizeOp0 % NarrowSize != 0)
502 return UnableToLegalize;
503 int NumParts = SizeOp0 / NarrowSize;
Igor Breger29537882017-04-07 14:41:59 +0000504 const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
Matthias Braunf1caa282017-12-15 22:22:58 +0000505 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Igor Breger29537882017-04-07 14:41:59 +0000506
507 SmallVector<unsigned, 2> DstRegs;
508 for (int i = 0; i < NumParts; ++i) {
509 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
510 ConstantInt *CI =
511 ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
512 MIRBuilder.buildConstant(DstReg, *CI);
513 DstRegs.push_back(DstReg);
514 }
515 unsigned DstReg = MI.getOperand(0).getReg();
516 MIRBuilder.buildMerge(DstReg, DstRegs);
517 MI.eraseFromParent();
518 return Legalized;
519 }
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000520 case TargetOpcode::G_OR: {
521 // Legalize bitwise operation:
522 // A = BinOp<Ty> B, C
523 // into:
524 // B1, ..., BN = G_UNMERGE_VALUES B
525 // C1, ..., CN = G_UNMERGE_VALUES C
526 // A1 = BinOp<Ty/N> B1, C2
527 // ...
528 // AN = BinOp<Ty/N> BN, CN
529 // A = G_MERGE_VALUES A1, ..., AN
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000530
531 // FIXME: add support for when SizeOp0 isn't an exact multiple of
532 // NarrowSize.
533 if (SizeOp0 % NarrowSize != 0)
534 return UnableToLegalize;
535 int NumParts = SizeOp0 / NarrowSize;
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000536
537 // List the registers where the destination will be scattered.
538 SmallVector<unsigned, 2> DstRegs;
539 // List the registers where the first argument will be split.
540 SmallVector<unsigned, 2> SrcsReg1;
541 // List the registers where the second argument will be split.
542 SmallVector<unsigned, 2> SrcsReg2;
543 // Create all the temporary registers.
544 for (int i = 0; i < NumParts; ++i) {
545 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
546 unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
547 unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
548
549 DstRegs.push_back(DstReg);
550 SrcsReg1.push_back(SrcReg1);
551 SrcsReg2.push_back(SrcReg2);
552 }
553 // Explode the big arguments into smaller chunks.
554 MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
555 MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
556
557 // Do the operation on each small part.
558 for (int i = 0; i < NumParts; ++i)
559 MIRBuilder.buildOr(DstRegs[i], SrcsReg1[i], SrcsReg2[i]);
560
561 // Gather the destination registers into the final destination.
562 unsigned DstReg = MI.getOperand(0).getReg();
563 MIRBuilder.buildMerge(DstReg, DstRegs);
564 MI.eraseFromParent();
565 return Legalized;
566 }
Tim Northover9656f142016-08-04 20:54:13 +0000567 }
Tim Northover33b07d62016-07-22 20:03:43 +0000568}
569
Tim Northover69fa84a2016-10-14 22:18:18 +0000570LegalizerHelper::LegalizeResult
571LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Tim Northover3c73e362016-08-23 18:20:09 +0000572 MIRBuilder.setInstr(MI);
573
Tim Northover32335812016-08-04 18:35:11 +0000574 switch (MI.getOpcode()) {
575 default:
576 return UnableToLegalize;
Tim Northover61c16142016-08-04 21:39:49 +0000577 case TargetOpcode::G_ADD:
578 case TargetOpcode::G_AND:
579 case TargetOpcode::G_MUL:
580 case TargetOpcode::G_OR:
581 case TargetOpcode::G_XOR:
Justin Bognerddb80ae2017-01-19 07:51:17 +0000582 case TargetOpcode::G_SUB:
583 case TargetOpcode::G_SHL: {
Tim Northover32335812016-08-04 18:35:11 +0000584 // Perform operation at larger width (any extension is fine here, high bits
585 // don't affect the result) and then truncate the result back to the
586 // original type.
Tim Northover0f140c72016-09-09 11:46:34 +0000587 unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
588 unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
589 MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
590 MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
Tim Northover32335812016-08-04 18:35:11 +0000591
Tim Northover0f140c72016-09-09 11:46:34 +0000592 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
593 MIRBuilder.buildInstr(MI.getOpcode())
594 .addDef(DstExt)
595 .addUse(Src1Ext)
596 .addUse(Src2Ext);
Tim Northover32335812016-08-04 18:35:11 +0000597
Tim Northover0f140c72016-09-09 11:46:34 +0000598 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
Tim Northover32335812016-08-04 18:35:11 +0000599 MI.eraseFromParent();
600 return Legalized;
601 }
Tim Northover7a753d92016-08-26 17:46:06 +0000602 case TargetOpcode::G_SDIV:
Justin Bognerddb80ae2017-01-19 07:51:17 +0000603 case TargetOpcode::G_UDIV:
Diana Picusdf4100b2017-07-18 09:08:47 +0000604 case TargetOpcode::G_SREM:
605 case TargetOpcode::G_UREM:
Justin Bognerddb80ae2017-01-19 07:51:17 +0000606 case TargetOpcode::G_ASHR:
607 case TargetOpcode::G_LSHR: {
608 unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
Diana Picusdf4100b2017-07-18 09:08:47 +0000609 MI.getOpcode() == TargetOpcode::G_SREM ||
Justin Bognerddb80ae2017-01-19 07:51:17 +0000610 MI.getOpcode() == TargetOpcode::G_ASHR
611 ? TargetOpcode::G_SEXT
612 : TargetOpcode::G_ZEXT;
Tim Northover7a753d92016-08-26 17:46:06 +0000613
Tim Northover0f140c72016-09-09 11:46:34 +0000614 unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
615 MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
616 MI.getOperand(1).getReg());
Tim Northover7a753d92016-08-26 17:46:06 +0000617
Tim Northover0f140c72016-09-09 11:46:34 +0000618 unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
619 MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
620 MI.getOperand(2).getReg());
Tim Northover7a753d92016-08-26 17:46:06 +0000621
Tim Northover0f140c72016-09-09 11:46:34 +0000622 unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
623 MIRBuilder.buildInstr(MI.getOpcode())
Tim Northover7a753d92016-08-26 17:46:06 +0000624 .addDef(ResExt)
625 .addUse(LHSExt)
626 .addUse(RHSExt);
627
Tim Northover0f140c72016-09-09 11:46:34 +0000628 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
Tim Northover7a753d92016-08-26 17:46:06 +0000629 MI.eraseFromParent();
630 return Legalized;
631 }
Tim Northover868332d2017-02-06 23:41:27 +0000632 case TargetOpcode::G_SELECT: {
633 if (TypeIdx != 0)
634 return UnableToLegalize;
635
636 // Perform operation at larger width (any extension is fine here, high bits
637 // don't affect the result) and then truncate the result back to the
638 // original type.
639 unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
640 unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
641 MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(2).getReg());
642 MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(3).getReg());
643
644 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
645 MIRBuilder.buildInstr(TargetOpcode::G_SELECT)
646 .addDef(DstExt)
647 .addReg(MI.getOperand(1).getReg())
648 .addUse(Src1Ext)
649 .addUse(Src2Ext);
650
651 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
652 MI.eraseFromParent();
653 return Legalized;
654 }
Ahmed Bougachab6137062017-01-23 21:10:14 +0000655 case TargetOpcode::G_FPTOSI:
656 case TargetOpcode::G_FPTOUI: {
657 if (TypeIdx != 0)
658 return UnableToLegalize;
659
660 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
661 MIRBuilder.buildInstr(MI.getOpcode())
662 .addDef(DstExt)
663 .addUse(MI.getOperand(1).getReg());
664
665 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
666 MI.eraseFromParent();
667 return Legalized;
668 }
Ahmed Bougachad2948232017-01-20 01:37:24 +0000669 case TargetOpcode::G_SITOFP:
670 case TargetOpcode::G_UITOFP: {
671 if (TypeIdx != 1)
672 return UnableToLegalize;
673
674 unsigned Src = MI.getOperand(1).getReg();
675 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
676
677 if (MI.getOpcode() == TargetOpcode::G_SITOFP) {
678 MIRBuilder.buildSExt(SrcExt, Src);
679 } else {
680 assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
681 MIRBuilder.buildZExt(SrcExt, Src);
682 }
683
684 MIRBuilder.buildInstr(MI.getOpcode())
685 .addDef(MI.getOperand(0).getReg())
686 .addUse(SrcExt);
687
688 MI.eraseFromParent();
689 return Legalized;
690 }
Tim Northover0e6afbd2017-02-06 21:56:47 +0000691 case TargetOpcode::G_INSERT: {
692 if (TypeIdx != 0)
693 return UnableToLegalize;
694
695 unsigned Src = MI.getOperand(1).getReg();
696 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
697 MIRBuilder.buildAnyExt(SrcExt, Src);
698
699 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
700 auto MIB = MIRBuilder.buildInsert(DstExt, SrcExt, MI.getOperand(2).getReg(),
701 MI.getOperand(3).getImm());
702 for (unsigned OpNum = 4; OpNum < MI.getNumOperands(); OpNum += 2) {
703 MIB.addReg(MI.getOperand(OpNum).getReg());
704 MIB.addImm(MI.getOperand(OpNum + 1).getImm());
705 }
706
707 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
708 MI.eraseFromParent();
709 return Legalized;
710 }
Tim Northover3c73e362016-08-23 18:20:09 +0000711 case TargetOpcode::G_LOAD: {
Rui Ueyamaa5edf652016-09-09 18:37:08 +0000712 assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
713 WideTy.getSizeInBits() &&
Tim Northover3c73e362016-08-23 18:20:09 +0000714 "illegal to increase number of bytes loaded");
715
Tim Northover0f140c72016-09-09 11:46:34 +0000716 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
717 MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
718 **MI.memoperands_begin());
719 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
Tim Northover3c73e362016-08-23 18:20:09 +0000720 MI.eraseFromParent();
721 return Legalized;
722 }
723 case TargetOpcode::G_STORE: {
Tim Northover548feee2017-03-21 22:22:05 +0000724 if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
725 WideTy != LLT::scalar(8))
726 return UnableToLegalize;
727
728 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
729 auto Content = TLI.getBooleanContents(false, false);
730
731 unsigned ExtOp = TargetOpcode::G_ANYEXT;
732 if (Content == TargetLoweringBase::ZeroOrOneBooleanContent)
733 ExtOp = TargetOpcode::G_ZEXT;
734 else if (Content == TargetLoweringBase::ZeroOrNegativeOneBooleanContent)
735 ExtOp = TargetOpcode::G_SEXT;
736 else
737 ExtOp = TargetOpcode::G_ANYEXT;
Tim Northover3c73e362016-08-23 18:20:09 +0000738
Tim Northover0f140c72016-09-09 11:46:34 +0000739 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
Tim Northover548feee2017-03-21 22:22:05 +0000740 MIRBuilder.buildInstr(ExtOp).addDef(SrcExt).addUse(
741 MI.getOperand(0).getReg());
Tim Northover0f140c72016-09-09 11:46:34 +0000742 MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
743 **MI.memoperands_begin());
Tim Northover3c73e362016-08-23 18:20:09 +0000744 MI.eraseFromParent();
745 return Legalized;
746 }
Tim Northoverea904f92016-08-19 22:40:00 +0000747 case TargetOpcode::G_CONSTANT: {
Tim Northover0f140c72016-09-09 11:46:34 +0000748 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
Tim Northover9267ac52016-12-05 21:47:07 +0000749 MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
Tim Northover0f140c72016-09-09 11:46:34 +0000750 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
Tim Northoverea904f92016-08-19 22:40:00 +0000751 MI.eraseFromParent();
752 return Legalized;
753 }
Tim Northovera11be042016-08-19 22:40:08 +0000754 case TargetOpcode::G_FCONSTANT: {
Tim Northover0f140c72016-09-09 11:46:34 +0000755 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
Amara Emerson77a5c962018-01-27 07:07:20 +0000756 const ConstantFP *CFP = MI.getOperand(1).getFPImm();
757 APFloat Val = CFP->getValueAPF();
758 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
759 auto LLT2Sem = [](LLT Ty) {
760 switch (Ty.getSizeInBits()) {
761 case 32:
762 return &APFloat::IEEEsingle();
763 break;
764 case 64:
765 return &APFloat::IEEEdouble();
766 break;
767 default:
768 llvm_unreachable("Unhandled fp widen type");
769 }
770 };
771 bool LosesInfo;
772 Val.convert(*LLT2Sem(WideTy), APFloat::rmTowardZero, &LosesInfo);
773 MIRBuilder.buildFConstant(DstExt, *ConstantFP::get(Ctx, Val));
Tim Northover0f140c72016-09-09 11:46:34 +0000774 MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
Tim Northovera11be042016-08-19 22:40:08 +0000775 MI.eraseFromParent();
776 return Legalized;
777 }
Tim Northoverb3a0be42016-08-23 21:01:20 +0000778 case TargetOpcode::G_BRCOND: {
Tim Northover0f140c72016-09-09 11:46:34 +0000779 unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
780 MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
781 MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
Tim Northoverb3a0be42016-08-23 21:01:20 +0000782 MI.eraseFromParent();
783 return Legalized;
784 }
Aditya Nandakumar02c602e2017-07-31 17:00:16 +0000785 case TargetOpcode::G_FCMP: {
786 unsigned Op0Ext, Op1Ext, DstReg;
787 unsigned Cmp1 = MI.getOperand(2).getReg();
788 unsigned Cmp2 = MI.getOperand(3).getReg();
789 if (TypeIdx == 0) {
790 Op0Ext = Cmp1;
791 Op1Ext = Cmp2;
792 DstReg = MRI.createGenericVirtualRegister(WideTy);
793 } else {
794 Op0Ext = MRI.createGenericVirtualRegister(WideTy);
795 Op1Ext = MRI.createGenericVirtualRegister(WideTy);
796 DstReg = MI.getOperand(0).getReg();
797 MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op0Ext, Cmp1);
798 MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op1Ext, Cmp2);
799 }
800 MIRBuilder.buildFCmp(
801 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
802 DstReg, Op0Ext, Op1Ext);
803 if (TypeIdx == 0)
804 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
805 DstReg);
806 MI.eraseFromParent();
807 return Legalized;
808 }
Tim Northover6cd4b232016-08-23 21:01:26 +0000809 case TargetOpcode::G_ICMP: {
Tim Northover051b8ad2016-08-26 17:46:17 +0000810 bool IsSigned = CmpInst::isSigned(
811 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
Aditya Nandakumar02c602e2017-07-31 17:00:16 +0000812 unsigned Cmp1 = MI.getOperand(2).getReg();
813 unsigned Cmp2 = MI.getOperand(3).getReg();
814 unsigned Op0Ext, Op1Ext, DstReg;
815 if (TypeIdx == 0) {
816 Op0Ext = Cmp1;
817 Op1Ext = Cmp2;
818 DstReg = MRI.createGenericVirtualRegister(WideTy);
Tim Northover6cd4b232016-08-23 21:01:26 +0000819 } else {
Aditya Nandakumar02c602e2017-07-31 17:00:16 +0000820 Op0Ext = MRI.createGenericVirtualRegister(WideTy);
821 Op1Ext = MRI.createGenericVirtualRegister(WideTy);
822 DstReg = MI.getOperand(0).getReg();
823 if (IsSigned) {
824 MIRBuilder.buildSExt(Op0Ext, Cmp1);
825 MIRBuilder.buildSExt(Op1Ext, Cmp2);
826 } else {
827 MIRBuilder.buildZExt(Op0Ext, Cmp1);
828 MIRBuilder.buildZExt(Op1Ext, Cmp2);
829 }
Tim Northover6cd4b232016-08-23 21:01:26 +0000830 }
Tim Northover051b8ad2016-08-26 17:46:17 +0000831 MIRBuilder.buildICmp(
Tim Northover051b8ad2016-08-26 17:46:17 +0000832 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
Aditya Nandakumar02c602e2017-07-31 17:00:16 +0000833 DstReg, Op0Ext, Op1Ext);
834 if (TypeIdx == 0)
835 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
836 DstReg);
Tim Northover051b8ad2016-08-26 17:46:17 +0000837 MI.eraseFromParent();
838 return Legalized;
Tim Northover6cd4b232016-08-23 21:01:26 +0000839 }
Tim Northover22d82cf2016-09-15 11:02:19 +0000840 case TargetOpcode::G_GEP: {
841 assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
842 unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
843 MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
844 MI.getOperand(2).setReg(OffsetExt);
845 return Legalized;
846 }
Aditya Nandakumar892979e2017-08-25 04:57:27 +0000847 case TargetOpcode::G_PHI: {
848 assert(TypeIdx == 0 && "Expecting only Idx 0");
David Blaikie196f53b22017-08-25 16:46:07 +0000849 auto getExtendedReg = [&](unsigned Reg, MachineBasicBlock &MBB) {
Aditya Nandakumar892979e2017-08-25 04:57:27 +0000850 auto FirstTermIt = MBB.getFirstTerminator();
851 MIRBuilder.setInsertPt(MBB, FirstTermIt);
852 MachineInstr *DefMI = MRI.getVRegDef(Reg);
853 MachineInstrBuilder MIB;
854 if (DefMI->getOpcode() == TargetOpcode::G_TRUNC)
855 MIB = MIRBuilder.buildAnyExtOrTrunc(WideTy,
856 DefMI->getOperand(1).getReg());
857 else
858 MIB = MIRBuilder.buildAnyExt(WideTy, Reg);
859 return MIB->getOperand(0).getReg();
860 };
861 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, WideTy);
862 for (auto OpIt = MI.operands_begin() + 1, OpE = MI.operands_end();
863 OpIt != OpE;) {
864 unsigned Reg = OpIt++->getReg();
865 MachineBasicBlock *OpMBB = OpIt++->getMBB();
866 MIB.addReg(getExtendedReg(Reg, *OpMBB));
867 MIB.addMBB(OpMBB);
868 }
869 auto *MBB = MI.getParent();
870 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
871 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(),
872 MIB->getOperand(0).getReg());
873 MI.eraseFromParent();
874 return Legalized;
875 }
Tim Northover32335812016-08-04 18:35:11 +0000876 }
Tim Northover33b07d62016-07-22 20:03:43 +0000877}
878
Tim Northover69fa84a2016-10-14 22:18:18 +0000879LegalizerHelper::LegalizeResult
880LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Tim Northovercecee562016-08-26 17:46:13 +0000881 using namespace TargetOpcode;
Tim Northovercecee562016-08-26 17:46:13 +0000882 MIRBuilder.setInstr(MI);
883
884 switch(MI.getOpcode()) {
885 default:
886 return UnableToLegalize;
887 case TargetOpcode::G_SREM:
888 case TargetOpcode::G_UREM: {
Tim Northover0f140c72016-09-09 11:46:34 +0000889 unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
890 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
Tim Northovercecee562016-08-26 17:46:13 +0000891 .addDef(QuotReg)
892 .addUse(MI.getOperand(1).getReg())
893 .addUse(MI.getOperand(2).getReg());
894
Tim Northover0f140c72016-09-09 11:46:34 +0000895 unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
896 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
897 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
898 ProdReg);
Tim Northovercecee562016-08-26 17:46:13 +0000899 MI.eraseFromParent();
900 return Legalized;
901 }
Tim Northover0a9b2792017-02-08 21:22:15 +0000902 case TargetOpcode::G_SMULO:
903 case TargetOpcode::G_UMULO: {
904 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
905 // result.
906 unsigned Res = MI.getOperand(0).getReg();
907 unsigned Overflow = MI.getOperand(1).getReg();
908 unsigned LHS = MI.getOperand(2).getReg();
909 unsigned RHS = MI.getOperand(3).getReg();
910
911 MIRBuilder.buildMul(Res, LHS, RHS);
912
913 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
914 ? TargetOpcode::G_SMULH
915 : TargetOpcode::G_UMULH;
916
917 unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
918 MIRBuilder.buildInstr(Opcode)
919 .addDef(HiPart)
920 .addUse(LHS)
921 .addUse(RHS);
922
923 unsigned Zero = MRI.createGenericVirtualRegister(Ty);
924 MIRBuilder.buildConstant(Zero, 0);
Amara Emerson9de62132018-01-03 04:56:56 +0000925
926 // For *signed* multiply, overflow is detected by checking:
927 // (hi != (lo >> bitwidth-1))
928 if (Opcode == TargetOpcode::G_SMULH) {
929 unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
930 unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
931 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
932 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
933 .addDef(Shifted)
934 .addUse(Res)
935 .addUse(ShiftAmt);
936 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
937 } else {
938 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
939 }
Tim Northover0a9b2792017-02-08 21:22:15 +0000940 MI.eraseFromParent();
941 return Legalized;
942 }
Volkan Keles5698b2a2017-03-08 18:09:14 +0000943 case TargetOpcode::G_FNEG: {
944 // TODO: Handle vector types once we are able to
945 // represent them.
946 if (Ty.isVector())
947 return UnableToLegalize;
948 unsigned Res = MI.getOperand(0).getReg();
949 Type *ZeroTy;
Matthias Braunf1caa282017-12-15 22:22:58 +0000950 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Volkan Keles5698b2a2017-03-08 18:09:14 +0000951 switch (Ty.getSizeInBits()) {
952 case 16:
953 ZeroTy = Type::getHalfTy(Ctx);
954 break;
955 case 32:
956 ZeroTy = Type::getFloatTy(Ctx);
957 break;
958 case 64:
959 ZeroTy = Type::getDoubleTy(Ctx);
960 break;
Amara Emersonb6ddbef2017-12-19 17:21:35 +0000961 case 128:
962 ZeroTy = Type::getFP128Ty(Ctx);
963 break;
Volkan Keles5698b2a2017-03-08 18:09:14 +0000964 default:
965 llvm_unreachable("unexpected floating-point type");
966 }
967 ConstantFP &ZeroForNegation =
968 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
969 unsigned Zero = MRI.createGenericVirtualRegister(Ty);
970 MIRBuilder.buildFConstant(Zero, ZeroForNegation);
971 MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
972 .addDef(Res)
973 .addUse(Zero)
974 .addUse(MI.getOperand(1).getReg());
975 MI.eraseFromParent();
976 return Legalized;
977 }
Volkan Keles225921a2017-03-10 21:25:09 +0000978 case TargetOpcode::G_FSUB: {
979 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
980 // First, check if G_FNEG is marked as Lower. If so, we may
981 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
Daniel Sanders9ade5592018-01-29 17:37:29 +0000982 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
Volkan Keles225921a2017-03-10 21:25:09 +0000983 return UnableToLegalize;
984 unsigned Res = MI.getOperand(0).getReg();
985 unsigned LHS = MI.getOperand(1).getReg();
986 unsigned RHS = MI.getOperand(2).getReg();
987 unsigned Neg = MRI.createGenericVirtualRegister(Ty);
988 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
989 MIRBuilder.buildInstr(TargetOpcode::G_FADD)
990 .addDef(Res)
991 .addUse(LHS)
992 .addUse(Neg);
993 MI.eraseFromParent();
994 return Legalized;
995 }
Daniel Sandersaef1dfc2017-11-30 20:11:42 +0000996 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
997 unsigned OldValRes = MI.getOperand(0).getReg();
998 unsigned SuccessRes = MI.getOperand(1).getReg();
999 unsigned Addr = MI.getOperand(2).getReg();
1000 unsigned CmpVal = MI.getOperand(3).getReg();
1001 unsigned NewVal = MI.getOperand(4).getReg();
1002 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1003 **MI.memoperands_begin());
1004 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1005 MI.eraseFromParent();
1006 return Legalized;
1007 }
Tim Northovercecee562016-08-26 17:46:13 +00001008 }
1009}
1010
Tim Northover69fa84a2016-10-14 22:18:18 +00001011LegalizerHelper::LegalizeResult
1012LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
1013 LLT NarrowTy) {
Quentin Colombet5e60bcd2016-08-27 02:38:21 +00001014 // FIXME: Don't know how to handle secondary types yet.
1015 if (TypeIdx != 0)
1016 return UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +00001017 switch (MI.getOpcode()) {
1018 default:
1019 return UnableToLegalize;
1020 case TargetOpcode::G_ADD: {
1021 unsigned NarrowSize = NarrowTy.getSizeInBits();
Tim Northover0f140c72016-09-09 11:46:34 +00001022 unsigned DstReg = MI.getOperand(0).getReg();
Kristof Beylsaf9814a2017-11-07 10:34:34 +00001023 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1024 int NumParts = Size / NarrowSize;
1025 // FIXME: Don't know how to handle the situation where the small vectors
1026 // aren't all the same size yet.
1027 if (Size % NarrowSize != 0)
1028 return UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +00001029
1030 MIRBuilder.setInstr(MI);
1031
Tim Northoverb18ea162016-09-20 15:20:36 +00001032 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
Tim Northover33b07d62016-07-22 20:03:43 +00001033 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
1034 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
1035
1036 for (int i = 0; i < NumParts; ++i) {
Tim Northover0f140c72016-09-09 11:46:34 +00001037 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
1038 MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
Tim Northover33b07d62016-07-22 20:03:43 +00001039 DstRegs.push_back(DstReg);
1040 }
1041
Tim Northoverbf017292017-03-03 22:46:09 +00001042 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northover33b07d62016-07-22 20:03:43 +00001043 MI.eraseFromParent();
1044 return Legalized;
1045 }
1046 }
1047}