blob: cdd3ca474edf69eca217b90a109046b91106ef23 [file] [log] [blame]
Nick Lewycky5fa40c32013-10-01 21:51:38 +00001//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +00006//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
Chris Lattnere70a0072010-06-29 16:40:28 +000015#include "ABIInfo.h"
Akira Hatanaka9d8ac612016-02-17 21:09:50 +000016#include "CGBlocks.h"
Chandler Carruth3a022472012-12-04 09:13:33 +000017#include "CGCXXABI.h"
David Majnemer4e52d6f2015-12-12 05:39:21 +000018#include "CGCleanup.h"
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +000019#include "CodeGenFunction.h"
Daniel Dunbarc68897d2008-09-10 00:41:16 +000020#include "CodeGenModule.h"
John McCalla729c622012-02-17 03:33:10 +000021#include "TargetInfo.h"
Reid Kleckner98031782019-12-09 16:11:56 -080022#include "clang/AST/Attr.h"
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +000023#include "clang/AST/Decl.h"
Anders Carlssonb15b55c2009-04-03 22:48:58 +000024#include "clang/AST/DeclCXX.h"
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +000025#include "clang/AST/DeclObjC.h"
Richard Trieu63688182018-12-11 03:18:39 +000026#include "clang/Basic/CodeGenOptions.h"
Eric Christopher15709992015-10-15 23:47:11 +000027#include "clang/Basic/TargetBuiltins.h"
Chandler Carruth3a022472012-12-04 09:13:33 +000028#include "clang/Basic/TargetInfo.h"
Mark Laceya8e7df32013-10-30 21:53:58 +000029#include "clang/CodeGen/CGFunctionInfo.h"
John McCall12f23522016-04-04 18:33:08 +000030#include "clang/CodeGen/SwiftCallingConv.h"
Bill Wendling706469b2013-02-28 22:49:57 +000031#include "llvm/ADT/StringExtras.h"
Nick Lewyckyd9bce502016-09-20 15:49:58 +000032#include "llvm/Analysis/ValueTracking.h"
Chandler Carruthffd55512013-01-02 11:45:17 +000033#include "llvm/IR/Attributes.h"
David Blaikief47ca252018-03-21 22:34:27 +000034#include "llvm/IR/CallingConv.h"
Chandler Carruthffd55512013-01-02 11:45:17 +000035#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InlineAsm.h"
Saleem Abdulrasool10a49722016-04-08 16:52:00 +000037#include "llvm/IR/IntrinsicInst.h"
David Blaikief47ca252018-03-21 22:34:27 +000038#include "llvm/IR/Intrinsics.h"
Reid Kleckner98031782019-12-09 16:11:56 -080039#include "llvm/Transforms/Utils/Local.h"
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +000040using namespace clang;
41using namespace CodeGen;
42
43/***/
44
Nikolay Haustov8c6538b2016-06-30 09:06:33 +000045unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
John McCallab26cfa2010-02-05 21:31:56 +000046 switch (CC) {
47 default: return llvm::CallingConv::C;
48 case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
49 case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
Erich Keane757d3172016-11-02 18:29:35 +000050 case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
Douglas Gregora941dca2010-05-18 16:57:00 +000051 case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
Martin Storsjo022e7822017-07-17 20:49:45 +000052 case CC_Win64: return llvm::CallingConv::Win64;
Charles Davisb5a214e2013-08-30 04:39:01 +000053 case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
Anton Korobeynikov231e8752011-04-14 20:06:49 +000054 case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
55 case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
Guy Benyeif0a014b2012-12-25 08:53:55 +000056 case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
Reid Klecknerd7857f02014-10-24 17:42:17 +000057 // TODO: Add support for __pascal to LLVM.
58 case CC_X86Pascal: return llvm::CallingConv::C;
59 // TODO: Add support for __vectorcall to LLVM.
Reid Kleckner80944df2014-10-31 22:00:51 +000060 case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
Sander de Smalen44a22532018-11-26 16:38:37 +000061 case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
Alexander Kornienko21de0ae2015-01-20 11:20:41 +000062 case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
Nikolay Haustov8c6538b2016-06-30 09:06:33 +000063 case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
Roman Levenstein35aa5ce2016-03-16 18:00:46 +000064 case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
65 case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
John McCall12f23522016-04-04 18:33:08 +000066 case CC_Swift: return llvm::CallingConv::Swift;
John McCallab26cfa2010-02-05 21:31:56 +000067 }
68}
69
Mikael Nilsson9d2872d2018-12-13 10:15:27 +000070/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
James Y Knightb92d2902019-02-05 16:05:50 +000071/// qualification. Either or both of RD and MD may be null. A null RD indicates
72/// that there is no meaningful 'this' type, and a null MD can occur when
73/// calling a method pointer.
74CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
75 const CXXMethodDecl *MD) {
76 QualType RecTy;
77 if (RD)
78 RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
79 else
80 RecTy = Context.VoidTy;
81
Mikael Nilsson9d2872d2018-12-13 10:15:27 +000082 if (MD)
Anastasia Stulovac61eaa52019-01-28 11:37:49 +000083 RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
John McCall2da83a32010-02-26 00:48:12 +000084 return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
Daniel Dunbar7a95ca32008-09-10 04:01:49 +000085}
86
John McCall8ee376f2010-02-24 07:14:12 +000087/// Returns the canonical formal type of the given C++ method.
John McCall2da83a32010-02-26 00:48:12 +000088static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
89 return MD->getType()->getCanonicalTypeUnqualified()
90 .getAs<FunctionProtoType>();
John McCall8ee376f2010-02-24 07:14:12 +000091}
92
93/// Returns the "extra-canonicalized" return type, which discards
94/// qualifiers on the return type. Codegen doesn't care about them,
95/// and it makes ABI code a little easier to be able to assume that
96/// all parameter and return types are top-level unqualified.
John McCall2da83a32010-02-26 00:48:12 +000097static CanQualType GetReturnType(QualType RetTy) {
98 return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
John McCall8ee376f2010-02-24 07:14:12 +000099}
100
John McCall8dda7b22012-07-07 06:41:13 +0000101/// Arrange the argument and result information for a value of the given
102/// unprototyped freestanding function type.
John McCall8ee376f2010-02-24 07:14:12 +0000103const CGFunctionInfo &
John McCall8dda7b22012-07-07 06:41:13 +0000104CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
John McCalla729c622012-02-17 03:33:10 +0000105 // When translating an unprototyped function type, always use a
106 // variadic type.
Alp Toker314cc812014-01-25 16:55:45 +0000107 return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
Peter Collingbournef7706832014-12-12 23:41:25 +0000108 /*instanceMethod=*/false,
109 /*chainCall=*/false, None,
John McCallc56a8b32016-03-11 04:30:31 +0000110 FTNP->getExtInfo(), {}, RequiredArgs(0));
John McCall8ee376f2010-02-24 07:14:12 +0000111}
112
George Burgess IVb7760212017-02-24 02:49:47 +0000113static void addExtParameterInfosForCall(
114 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
115 const FunctionProtoType *proto,
116 unsigned prefixArgs,
117 unsigned totalArgs) {
118 assert(proto->hasExtParameterInfos());
119 assert(paramInfos.size() <= prefixArgs);
120 assert(proto->getNumParams() + prefixArgs <= totalArgs);
121
122 paramInfos.reserve(totalArgs);
123
124 // Add default infos for any prefix args that don't already have infos.
125 paramInfos.resize(prefixArgs);
126
127 // Add infos for the prototype.
128 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
129 paramInfos.push_back(ParamInfo);
130 // pass_object_size params have no parameter info.
131 if (ParamInfo.hasPassObjectSize())
132 paramInfos.emplace_back();
133 }
134
135 assert(paramInfos.size() <= totalArgs &&
136 "Did we forget to insert pass_object_size args?");
137 // Add default infos for the variadic and/or suffix arguments.
138 paramInfos.resize(totalArgs);
139}
140
Hiroshi Inouec5e54dd2017-07-03 08:49:44 +0000141/// Adds the formal parameters in FPT to the given prefix. If any parameter in
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000142/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
143static void appendParameterTypes(const CodeGenTypes &CGT,
144 SmallVectorImpl<CanQualType> &prefix,
John McCallc56a8b32016-03-11 04:30:31 +0000145 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
George Burgess IVb7760212017-02-24 02:49:47 +0000146 CanQual<FunctionProtoType> FPT) {
147 // Fast path: don't touch param info if we don't need to.
148 if (!FPT->hasExtParameterInfos()) {
149 assert(paramInfos.empty() &&
150 "We have paramInfos, but the prototype doesn't?");
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000151 prefix.append(FPT->param_type_begin(), FPT->param_type_end());
152 return;
153 }
154
George Burgess IVb7760212017-02-24 02:49:47 +0000155 unsigned PrefixSize = prefix.size();
156 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000157 // parameters; the only thing that can change this is the presence of
158 // pass_object_size. So, we preallocate for the common case.
159 prefix.reserve(prefix.size() + FPT->getNumParams());
160
George Burgess IVb7760212017-02-24 02:49:47 +0000161 auto ExtInfos = FPT->getExtParameterInfos();
162 assert(ExtInfos.size() == FPT->getNumParams());
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000163 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
164 prefix.push_back(FPT->getParamType(I));
George Burgess IVb7760212017-02-24 02:49:47 +0000165 if (ExtInfos[I].hasPassObjectSize())
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000166 prefix.push_back(CGT.getContext().getSizeType());
167 }
George Burgess IVb7760212017-02-24 02:49:47 +0000168
169 addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
170 prefix.size());
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000171}
172
John McCall8dda7b22012-07-07 06:41:13 +0000173/// Arrange the LLVM function layout for a value of the given function
Alexey Samsonove5ef3ca2014-08-13 23:55:54 +0000174/// type, on top of any implicit parameters already stored.
175static const CGFunctionInfo &
Peter Collingbournef7706832014-12-12 23:41:25 +0000176arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
Alexey Samsonove5ef3ca2014-08-13 23:55:54 +0000177 SmallVectorImpl<CanQualType> &prefix,
James Y Knight916db652019-02-02 01:48:23 +0000178 CanQual<FunctionProtoType> FTP) {
John McCallc56a8b32016-03-11 04:30:31 +0000179 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
James Y Knight916db652019-02-02 01:48:23 +0000180 RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
Daniel Dunbarbf8c24a2009-02-02 23:23:47 +0000181 // FIXME: Kill copy.
George Burgess IVb7760212017-02-24 02:49:47 +0000182 appendParameterTypes(CGT, prefix, paramInfos, FTP);
Alp Toker314cc812014-01-25 16:55:45 +0000183 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
John McCallc56a8b32016-03-11 04:30:31 +0000184
Peter Collingbournef7706832014-12-12 23:41:25 +0000185 return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
186 /*chainCall=*/false, prefix,
John McCallc56a8b32016-03-11 04:30:31 +0000187 FTP->getExtInfo(), paramInfos,
George Burgess IV419996c2016-06-16 23:06:04 +0000188 Required);
John McCall8ee376f2010-02-24 07:14:12 +0000189}
190
John McCalla729c622012-02-17 03:33:10 +0000191/// Arrange the argument and result information for a value of the
John McCall8dda7b22012-07-07 06:41:13 +0000192/// given freestanding function type.
John McCall8ee376f2010-02-24 07:14:12 +0000193const CGFunctionInfo &
James Y Knight916db652019-02-02 01:48:23 +0000194CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
John McCalla729c622012-02-17 03:33:10 +0000195 SmallVector<CanQualType, 16> argTypes;
Peter Collingbournef7706832014-12-12 23:41:25 +0000196 return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
James Y Knight916db652019-02-02 01:48:23 +0000197 FTP);
Daniel Dunbar7feafc72009-09-11 22:24:53 +0000198}
199
Aaron Ballman0362a6d2013-12-18 16:23:37 +0000200static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
Daniel Dunbar7feafc72009-09-11 22:24:53 +0000201 // Set the appropriate calling convention for the Function.
202 if (D->hasAttr<StdCallAttr>())
John McCallab26cfa2010-02-05 21:31:56 +0000203 return CC_X86StdCall;
Daniel Dunbar7feafc72009-09-11 22:24:53 +0000204
205 if (D->hasAttr<FastCallAttr>())
John McCallab26cfa2010-02-05 21:31:56 +0000206 return CC_X86FastCall;
Daniel Dunbar7feafc72009-09-11 22:24:53 +0000207
Erich Keane757d3172016-11-02 18:29:35 +0000208 if (D->hasAttr<RegCallAttr>())
209 return CC_X86RegCall;
210
Douglas Gregora941dca2010-05-18 16:57:00 +0000211 if (D->hasAttr<ThisCallAttr>())
212 return CC_X86ThisCall;
213
Reid Klecknerd7857f02014-10-24 17:42:17 +0000214 if (D->hasAttr<VectorCallAttr>())
215 return CC_X86VectorCall;
216
Dawn Perchik335e16b2010-09-03 01:29:35 +0000217 if (D->hasAttr<PascalAttr>())
218 return CC_X86Pascal;
219
Anton Korobeynikov231e8752011-04-14 20:06:49 +0000220 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
221 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
222
Sander de Smalen44a22532018-11-26 16:38:37 +0000223 if (D->hasAttr<AArch64VectorPcsAttr>())
224 return CC_AArch64VectorCall;
225
Guy Benyeif0a014b2012-12-25 08:53:55 +0000226 if (D->hasAttr<IntelOclBiccAttr>())
227 return CC_IntelOclBicc;
228
Aaron Ballman0362a6d2013-12-18 16:23:37 +0000229 if (D->hasAttr<MSABIAttr>())
Martin Storsjo022e7822017-07-17 20:49:45 +0000230 return IsWindows ? CC_C : CC_Win64;
Aaron Ballman0362a6d2013-12-18 16:23:37 +0000231
232 if (D->hasAttr<SysVABIAttr>())
233 return IsWindows ? CC_X86_64SysV : CC_C;
234
Roman Levenstein35aa5ce2016-03-16 18:00:46 +0000235 if (D->hasAttr<PreserveMostAttr>())
236 return CC_PreserveMost;
237
238 if (D->hasAttr<PreserveAllAttr>())
239 return CC_PreserveAll;
240
John McCallab26cfa2010-02-05 21:31:56 +0000241 return CC_C;
Daniel Dunbar7a95ca32008-09-10 04:01:49 +0000242}
243
John McCalla729c622012-02-17 03:33:10 +0000244/// Arrange the argument and result information for a call to an
245/// unknown C++ non-static member function of the given abstract type.
James Y Knightb92d2902019-02-05 16:05:50 +0000246/// (A null RD means we don't have any meaningful "this" argument type,
Timur Iskhodzhanov88fd4392013-08-21 06:25:03 +0000247/// so fall back to a generic pointer type).
John McCalla729c622012-02-17 03:33:10 +0000248/// The member function must be an ordinary function, i.e. not a
249/// constructor or destructor.
250const CGFunctionInfo &
251CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000252 const FunctionProtoType *FTP,
253 const CXXMethodDecl *MD) {
John McCalla729c622012-02-17 03:33:10 +0000254 SmallVector<CanQualType, 16> argTypes;
John McCall8ee376f2010-02-24 07:14:12 +0000255
Anders Carlsson2ee3c012009-10-03 19:43:08 +0000256 // Add the 'this' pointer.
James Y Knightb92d2902019-02-05 16:05:50 +0000257 argTypes.push_back(DeriveThisType(RD, MD));
John McCall8ee376f2010-02-24 07:14:12 +0000258
Alexey Samsonove5ef3ca2014-08-13 23:55:54 +0000259 return ::arrangeLLVMFunctionInfo(
260 *this, true, argTypes,
James Y Knight916db652019-02-02 01:48:23 +0000261 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
Anders Carlsson2ee3c012009-10-03 19:43:08 +0000262}
263
Yaxun Liu6c10a662018-06-12 00:16:33 +0000264/// Set calling convention for CUDA/HIP kernel.
265static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
266 const FunctionDecl *FD) {
267 if (FD->hasAttr<CUDAGlobalAttr>()) {
268 const FunctionType *FT = FTy->getAs<FunctionType>();
269 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
270 FTy = FT->getCanonicalTypeUnqualified();
271 }
272}
273
John McCalla729c622012-02-17 03:33:10 +0000274/// Arrange the argument and result information for a declaration or
275/// definition of the given C++ non-static member function. The
276/// member function must be an ordinary function, i.e. not a
277/// constructor or destructor.
278const CGFunctionInfo &
279CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
Benjamin Kramer60509af2013-09-09 14:48:42 +0000280 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
John McCall0d635f52010-09-03 01:26:39 +0000281 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
282
Yaxun Liu6c10a662018-06-12 00:16:33 +0000283 CanQualType FT = GetFormalType(MD).getAs<Type>();
284 setCUDAKernelCallingConvention(FT, CGM, MD);
285 auto prototype = FT.getAs<FunctionProtoType>();
Mike Stump11289f42009-09-09 15:08:12 +0000286
John McCalla729c622012-02-17 03:33:10 +0000287 if (MD->isInstance()) {
288 // The abstract case is perfectly fine.
Mark Lacey5ea993b2013-10-02 20:35:23 +0000289 const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
George Burgess IV3e3bb95b2015-12-02 21:58:08 +0000290 return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
John McCalla729c622012-02-17 03:33:10 +0000291 }
292
James Y Knight916db652019-02-02 01:48:23 +0000293 return arrangeFreeFunctionType(prototype);
Anders Carlssonb15b55c2009-04-03 22:48:58 +0000294}
295
Richard Smith5179eb72016-06-28 19:03:57 +0000296bool CodeGenTypes::inheritingCtorHasParams(
297 const InheritedConstructor &Inherited, CXXCtorType Type) {
298 // Parameters are unnecessary if we're constructing a base class subobject
299 // and the inherited constructor lives in a virtual base.
300 return Type == Ctor_Complete ||
301 !Inherited.getShadowDecl()->constructsVirtualBase() ||
302 !Target.getCXXABI().hasConstructorVariants();
Peter Collingbourned1c5b282019-03-22 23:05:10 +0000303}
Richard Smith5179eb72016-06-28 19:03:57 +0000304
John McCalla729c622012-02-17 03:33:10 +0000305const CGFunctionInfo &
Peter Collingbourned1c5b282019-03-22 23:05:10 +0000306CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
307 auto *MD = cast<CXXMethodDecl>(GD.getDecl());
Rafael Espindola8d2a19b2014-09-08 16:01:27 +0000308
John McCalla729c622012-02-17 03:33:10 +0000309 SmallVector<CanQualType, 16> argTypes;
John McCallc56a8b32016-03-11 04:30:31 +0000310 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
James Y Knightb92d2902019-02-05 16:05:50 +0000311 argTypes.push_back(DeriveThisType(MD->getParent(), MD));
Stephen Lin9dc6eef2013-06-30 20:40:16 +0000312
Richard Smith5179eb72016-06-28 19:03:57 +0000313 bool PassParams = true;
314
Rafael Espindola8d2a19b2014-09-08 16:01:27 +0000315 if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
Richard Smith5179eb72016-06-28 19:03:57 +0000316 // A base class inheriting constructor doesn't get forwarded arguments
317 // needed to construct a virtual base (or base class thereof).
318 if (auto Inherited = CD->getInheritedConstructor())
Peter Collingbourned1c5b282019-03-22 23:05:10 +0000319 PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
Rafael Espindola8d2a19b2014-09-08 16:01:27 +0000320 }
Anders Carlsson82ba57c2009-11-25 03:15:49 +0000321
Rafael Espindola8d2a19b2014-09-08 16:01:27 +0000322 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
John McCall5d865c322010-08-31 07:33:07 +0000323
324 // Add the formal parameters.
Richard Smith5179eb72016-06-28 19:03:57 +0000325 if (PassParams)
George Burgess IVb7760212017-02-24 02:49:47 +0000326 appendParameterTypes(*this, argTypes, paramInfos, FTP);
John McCall5d865c322010-08-31 07:33:07 +0000327
George Burgess IV75b34a92017-02-22 22:38:25 +0000328 CGCXXABI::AddedStructorArgs AddedArgs =
Peter Collingbourned1c5b282019-03-22 23:05:10 +0000329 TheCXXABI.buildStructorSignature(GD, argTypes);
George Burgess IV75b34a92017-02-22 22:38:25 +0000330 if (!paramInfos.empty()) {
331 // Note: prefix implies after the first param.
332 if (AddedArgs.Prefix)
333 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
334 FunctionProtoType::ExtParameterInfo{});
335 if (AddedArgs.Suffix)
336 paramInfos.append(AddedArgs.Suffix,
337 FunctionProtoType::ExtParameterInfo{});
338 }
Reid Kleckner89077a12013-12-17 19:46:40 +0000339
340 RequiredArgs required =
Richard Smith5179eb72016-06-28 19:03:57 +0000341 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
342 : RequiredArgs::All);
Reid Kleckner89077a12013-12-17 19:46:40 +0000343
John McCall8dda7b22012-07-07 06:41:13 +0000344 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
David Majnemer0c0b6d92014-10-31 20:09:12 +0000345 CanQualType resultType = TheCXXABI.HasThisReturn(GD)
346 ? argTypes.front()
347 : TheCXXABI.hasMostDerivedReturn(GD)
348 ? CGM.getContext().VoidPtrTy
349 : Context.VoidTy;
Peter Collingbournef7706832014-12-12 23:41:25 +0000350 return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
351 /*chainCall=*/false, argTypes, extInfo,
John McCallc56a8b32016-03-11 04:30:31 +0000352 paramInfos, required);
353}
354
355static SmallVector<CanQualType, 16>
356getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
357 SmallVector<CanQualType, 16> argTypes;
358 for (auto &arg : args)
359 argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
360 return argTypes;
361}
362
363static SmallVector<CanQualType, 16>
364getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
365 SmallVector<CanQualType, 16> argTypes;
366 for (auto &arg : args)
367 argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
368 return argTypes;
369}
370
John McCallc56a8b32016-03-11 04:30:31 +0000371static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
372getExtParameterInfosForCall(const FunctionProtoType *proto,
373 unsigned prefixArgs, unsigned totalArgs) {
374 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
375 if (proto->hasExtParameterInfos()) {
376 addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
377 }
378 return result;
Anders Carlsson82ba57c2009-11-25 03:15:49 +0000379}
380
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000381/// Arrange a call to a C++ method, passing the given arguments.
George Burgess IVd0a9e802017-02-23 22:07:35 +0000382///
383/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
384/// parameter.
385/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
386/// args.
387/// PassProtoArgs indicates whether `args` has args for the parameters in the
388/// given CXXConstructorDecl.
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000389const CGFunctionInfo &
390CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
391 const CXXConstructorDecl *D,
392 CXXCtorType CtorKind,
George Burgess IVd0a9e802017-02-23 22:07:35 +0000393 unsigned ExtraPrefixArgs,
394 unsigned ExtraSuffixArgs,
395 bool PassProtoArgs) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000396 // FIXME: Kill copy.
397 SmallVector<CanQualType, 16> ArgTypes;
Alexey Samsonov3551e312014-08-13 20:06:24 +0000398 for (const auto &Arg : args)
399 ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000400
George Burgess IVd0a9e802017-02-23 22:07:35 +0000401 // +1 for implicit this, which should always be args[0].
402 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
403
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000404 CanQual<FunctionProtoType> FPT = GetFormalType(D);
James Y Knightb92d2902019-02-05 16:05:50 +0000405 RequiredArgs Required = PassProtoArgs
406 ? RequiredArgs::forPrototypePlus(
407 FPT, TotalPrefixArgs + ExtraSuffixArgs)
408 : RequiredArgs::All;
409
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000410 GlobalDecl GD(D, CtorKind);
David Majnemer0c0b6d92014-10-31 20:09:12 +0000411 CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
412 ? ArgTypes.front()
413 : TheCXXABI.hasMostDerivedReturn(GD)
414 ? CGM.getContext().VoidPtrTy
415 : Context.VoidTy;
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000416
417 FunctionType::ExtInfo Info = FPT->getExtInfo();
George Burgess IVd0a9e802017-02-23 22:07:35 +0000418 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
419 // If the prototype args are elided, we should only have ABI-specific args,
420 // which never have param info.
421 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
422 // ABI-specific suffix arguments are treated the same as variadic arguments.
423 addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
424 ArgTypes.size());
425 }
Peter Collingbournef7706832014-12-12 23:41:25 +0000426 return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
427 /*chainCall=*/false, ArgTypes, Info,
John McCallc56a8b32016-03-11 04:30:31 +0000428 ParamInfos, Required);
Reid Kleckner314ef7b2014-02-01 00:04:45 +0000429}
430
John McCalla729c622012-02-17 03:33:10 +0000431/// Arrange the argument and result information for the declaration or
432/// definition of the given function.
433const CGFunctionInfo &
434CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
Chris Lattnerbea5b622009-05-12 20:27:19 +0000435 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
Anders Carlssonb15b55c2009-04-03 22:48:58 +0000436 if (MD->isInstance())
John McCalla729c622012-02-17 03:33:10 +0000437 return arrangeCXXMethodDeclaration(MD);
Mike Stump11289f42009-09-09 15:08:12 +0000438
John McCall2da83a32010-02-26 00:48:12 +0000439 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
John McCalla729c622012-02-17 03:33:10 +0000440
John McCall2da83a32010-02-26 00:48:12 +0000441 assert(isa<FunctionType>(FTy));
Yaxun Liu6c10a662018-06-12 00:16:33 +0000442 setCUDAKernelCallingConvention(FTy, CGM, FD);
John McCalla729c622012-02-17 03:33:10 +0000443
444 // When declaring a function without a prototype, always use a
445 // non-variadic type.
George Burgess IV35cfca22017-01-06 19:10:48 +0000446 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
Peter Collingbournef7706832014-12-12 23:41:25 +0000447 return arrangeLLVMFunctionInfo(
448 noProto->getReturnType(), /*instanceMethod=*/false,
John McCallc56a8b32016-03-11 04:30:31 +0000449 /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
John McCalla729c622012-02-17 03:33:10 +0000450 }
451
James Y Knight916db652019-02-02 01:48:23 +0000452 return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +0000453}
454
John McCalla729c622012-02-17 03:33:10 +0000455/// Arrange the argument and result information for the declaration or
456/// definition of an Objective-C method.
457const CGFunctionInfo &
458CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
459 // It happens that this is the same as a call with no optional
460 // arguments, except also using the formal 'self' type.
461 return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
462}
463
464/// Arrange the argument and result information for the function type
465/// through which to perform a send to the given Objective-C method,
466/// using the given receiver type. The receiver type is not always
467/// the 'self' type of the method or even an Objective-C pointer type.
468/// This is *not* the right method for actually performing such a
469/// message send, due to the possibility of optional arguments.
470const CGFunctionInfo &
471CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
472 QualType receiverType) {
473 SmallVector<CanQualType, 16> argTys;
Akira Hatanaka98a49332017-09-22 00:41:05 +0000474 SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
John McCalla729c622012-02-17 03:33:10 +0000475 argTys.push_back(Context.getCanonicalParamType(receiverType));
476 argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
Daniel Dunbarbf8c24a2009-02-02 23:23:47 +0000477 // FIXME: Kill copy?
David Majnemer59f77922016-06-24 04:05:48 +0000478 for (const auto *I : MD->parameters()) {
Aaron Ballman43b68be2014-03-07 17:50:17 +0000479 argTys.push_back(Context.getCanonicalParamType(I->getType()));
Akira Hatanaka98a49332017-09-22 00:41:05 +0000480 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
481 I->hasAttr<NoEscapeAttr>());
482 extParamInfos.push_back(extParamInfo);
John McCall8ee376f2010-02-24 07:14:12 +0000483 }
John McCall31168b02011-06-15 23:02:42 +0000484
485 FunctionType::ExtInfo einfo;
Aaron Ballman0362a6d2013-12-18 16:23:37 +0000486 bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
487 einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
John McCall31168b02011-06-15 23:02:42 +0000488
David Blaikiebbafb8a2012-03-11 07:00:24 +0000489 if (getContext().getLangOpts().ObjCAutoRefCount &&
John McCall31168b02011-06-15 23:02:42 +0000490 MD->hasAttr<NSReturnsRetainedAttr>())
491 einfo = einfo.withProducesResult(true);
492
John McCalla729c622012-02-17 03:33:10 +0000493 RequiredArgs required =
494 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
495
Peter Collingbournef7706832014-12-12 23:41:25 +0000496 return arrangeLLVMFunctionInfo(
497 GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
Akira Hatanaka98a49332017-09-22 00:41:05 +0000498 /*chainCall=*/false, argTys, einfo, extParamInfos, required);
John McCallc56a8b32016-03-11 04:30:31 +0000499}
500
501const CGFunctionInfo &
502CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
503 const CallArgList &args) {
504 auto argTypes = getArgTypesForCall(Context, args);
505 FunctionType::ExtInfo einfo;
506
507 return arrangeLLVMFunctionInfo(
508 GetReturnType(returnType), /*instanceMethod=*/false,
509 /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
Daniel Dunbar3d7c90b2008-09-08 21:33:45 +0000510}
511
John McCalla729c622012-02-17 03:33:10 +0000512const CGFunctionInfo &
513CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
Anders Carlsson6710c532010-02-06 02:44:09 +0000514 // FIXME: Do we need to handle ObjCMethodDecl?
515 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +0000516
Peter Collingbourned1c5b282019-03-22 23:05:10 +0000517 if (isa<CXXConstructorDecl>(GD.getDecl()) ||
518 isa<CXXDestructorDecl>(GD.getDecl()))
519 return arrangeCXXStructorDeclaration(GD);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +0000520
John McCalla729c622012-02-17 03:33:10 +0000521 return arrangeFunctionDeclaration(FD);
Anders Carlsson6710c532010-02-06 02:44:09 +0000522}
523
Reid Klecknerc3473512014-08-29 21:43:29 +0000524/// Arrange a thunk that takes 'this' as the first parameter followed by
525/// varargs. Return a void pointer, regardless of the actual return type.
526/// The body of the thunk will end in a musttail call to a function of the
527/// correct type, and the caller will bitcast the function to the correct
528/// prototype.
529const CGFunctionInfo &
Reid Kleckner399d96e2018-04-02 20:20:33 +0000530CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
531 assert(MD->isVirtual() && "only methods have thunks");
Reid Klecknerc3473512014-08-29 21:43:29 +0000532 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
James Y Knightb92d2902019-02-05 16:05:50 +0000533 CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
Peter Collingbournef7706832014-12-12 23:41:25 +0000534 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
535 /*chainCall=*/false, ArgTys,
John McCallc56a8b32016-03-11 04:30:31 +0000536 FTP->getExtInfo(), {}, RequiredArgs(1));
Reid Klecknerc3473512014-08-29 21:43:29 +0000537}
538
David Majnemerdfa6d202015-03-11 18:36:39 +0000539const CGFunctionInfo &
David Majnemer37fd66e2015-03-13 22:36:55 +0000540CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
541 CXXCtorType CT) {
542 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
543
David Majnemerdfa6d202015-03-11 18:36:39 +0000544 CanQual<FunctionProtoType> FTP = GetFormalType(CD);
545 SmallVector<CanQualType, 2> ArgTys;
546 const CXXRecordDecl *RD = CD->getParent();
James Y Knightb92d2902019-02-05 16:05:50 +0000547 ArgTys.push_back(DeriveThisType(RD, CD));
David Majnemer37fd66e2015-03-13 22:36:55 +0000548 if (CT == Ctor_CopyingClosure)
549 ArgTys.push_back(*FTP->param_type_begin());
David Majnemerdfa6d202015-03-11 18:36:39 +0000550 if (RD->getNumVBases() > 0)
551 ArgTys.push_back(Context.IntTy);
552 CallingConv CC = Context.getDefaultCallingConvention(
553 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
554 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
555 /*chainCall=*/false, ArgTys,
John McCallc56a8b32016-03-11 04:30:31 +0000556 FunctionType::ExtInfo(CC), {},
557 RequiredArgs::All);
David Majnemerdfa6d202015-03-11 18:36:39 +0000558}
559
John McCallc818bbb2012-12-07 07:03:17 +0000560/// Arrange a call as unto a free function, except possibly with an
561/// additional number of formal parameters considered required.
562static const CGFunctionInfo &
563arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
Mark Lacey23455752013-10-10 20:57:00 +0000564 CodeGenModule &CGM,
John McCallc818bbb2012-12-07 07:03:17 +0000565 const CallArgList &args,
566 const FunctionType *fnType,
Peter Collingbournef7706832014-12-12 23:41:25 +0000567 unsigned numExtraRequiredArgs,
568 bool chainCall) {
John McCallc818bbb2012-12-07 07:03:17 +0000569 assert(args.size() >= numExtraRequiredArgs);
570
John McCallc56a8b32016-03-11 04:30:31 +0000571 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
572
John McCallc818bbb2012-12-07 07:03:17 +0000573 // In most cases, there are no optional arguments.
574 RequiredArgs required = RequiredArgs::All;
575
576 // If we have a variadic prototype, the required arguments are the
577 // extra prefix plus the arguments in the prototype.
578 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
579 if (proto->isVariadic())
James Y Knightb92d2902019-02-05 16:05:50 +0000580 required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
John McCallc818bbb2012-12-07 07:03:17 +0000581
John McCallc56a8b32016-03-11 04:30:31 +0000582 if (proto->hasExtParameterInfos())
583 addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
584 args.size());
585
John McCallc818bbb2012-12-07 07:03:17 +0000586 // If we don't have a prototype at all, but we're supposed to
587 // explicitly use the variadic convention for unprototyped calls,
588 // treat all of the arguments as required but preserve the nominal
589 // possibility of variadics.
Mark Lacey23455752013-10-10 20:57:00 +0000590 } else if (CGM.getTargetCodeGenInfo()
591 .isNoProtoCallVariadic(args,
592 cast<FunctionNoProtoType>(fnType))) {
John McCallc818bbb2012-12-07 07:03:17 +0000593 required = RequiredArgs(args.size());
594 }
595
Peter Collingbournef7706832014-12-12 23:41:25 +0000596 // FIXME: Kill copy.
597 SmallVector<CanQualType, 16> argTypes;
598 for (const auto &arg : args)
599 argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
600 return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
601 /*instanceMethod=*/false, chainCall,
John McCallc56a8b32016-03-11 04:30:31 +0000602 argTypes, fnType->getExtInfo(), paramInfos,
603 required);
John McCallc818bbb2012-12-07 07:03:17 +0000604}
605
John McCalla729c622012-02-17 03:33:10 +0000606/// Figure out the rules for calling a function with the given formal
607/// type using the given arguments. The arguments are necessary
608/// because the function might be unprototyped, in which case it's
609/// target-dependent in crazy ways.
610const CGFunctionInfo &
John McCall8dda7b22012-07-07 06:41:13 +0000611CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
Peter Collingbournef7706832014-12-12 23:41:25 +0000612 const FunctionType *fnType,
613 bool chainCall) {
614 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
615 chainCall ? 1 : 0, chainCall);
John McCallc818bbb2012-12-07 07:03:17 +0000616}
John McCalla729c622012-02-17 03:33:10 +0000617
John McCallc56a8b32016-03-11 04:30:31 +0000618/// A block function is essentially a free function with an
John McCallc818bbb2012-12-07 07:03:17 +0000619/// extra implicit argument.
620const CGFunctionInfo &
621CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
622 const FunctionType *fnType) {
Peter Collingbournef7706832014-12-12 23:41:25 +0000623 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
624 /*chainCall=*/false);
John McCalla729c622012-02-17 03:33:10 +0000625}
626
627const CGFunctionInfo &
John McCallc56a8b32016-03-11 04:30:31 +0000628CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
629 const FunctionArgList &params) {
630 auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
631 auto argTypes = getArgTypesForDeclaration(Context, params);
632
James Y Knight916db652019-02-02 01:48:23 +0000633 return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
634 /*instanceMethod*/ false, /*chainCall*/ false,
635 argTypes, proto->getExtInfo(), paramInfos,
636 RequiredArgs::forPrototypePlus(proto, 1));
John McCallc56a8b32016-03-11 04:30:31 +0000637}
638
639const CGFunctionInfo &
640CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
641 const CallArgList &args) {
Daniel Dunbarbf8c24a2009-02-02 23:23:47 +0000642 // FIXME: Kill copy.
John McCalla729c622012-02-17 03:33:10 +0000643 SmallVector<CanQualType, 16> argTypes;
Alexey Samsonov3551e312014-08-13 20:06:24 +0000644 for (const auto &Arg : args)
645 argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
Peter Collingbournef7706832014-12-12 23:41:25 +0000646 return arrangeLLVMFunctionInfo(
647 GetReturnType(resultType), /*instanceMethod=*/false,
John McCallc56a8b32016-03-11 04:30:31 +0000648 /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
649 /*paramInfos=*/ {}, RequiredArgs::All);
John McCall8dda7b22012-07-07 06:41:13 +0000650}
651
John McCallc56a8b32016-03-11 04:30:31 +0000652const CGFunctionInfo &
653CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
654 const FunctionArgList &args) {
655 auto argTypes = getArgTypesForDeclaration(Context, args);
656
657 return arrangeLLVMFunctionInfo(
658 GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
659 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
660}
661
662const CGFunctionInfo &
663CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
664 ArrayRef<CanQualType> argTypes) {
665 return arrangeLLVMFunctionInfo(
666 resultType, /*instanceMethod=*/false, /*chainCall=*/false,
667 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
668}
669
John McCall8dda7b22012-07-07 06:41:13 +0000670/// Arrange a call to a C++ method, passing the given arguments.
George Burgess IVd0a9e802017-02-23 22:07:35 +0000671///
672/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
673/// does not count `this`.
John McCall8dda7b22012-07-07 06:41:13 +0000674const CGFunctionInfo &
675CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
John McCallc56a8b32016-03-11 04:30:31 +0000676 const FunctionProtoType *proto,
George Burgess IVd0a9e802017-02-23 22:07:35 +0000677 RequiredArgs required,
678 unsigned numPrefixArgs) {
679 assert(numPrefixArgs + 1 <= args.size() &&
680 "Emitting a call with less args than the required prefix?");
681 // Add one to account for `this`. It's a bit awkward here, but we don't count
682 // `this` in similar places elsewhere.
John McCallc56a8b32016-03-11 04:30:31 +0000683 auto paramInfos =
George Burgess IVd0a9e802017-02-23 22:07:35 +0000684 getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
John McCallc56a8b32016-03-11 04:30:31 +0000685
John McCall8dda7b22012-07-07 06:41:13 +0000686 // FIXME: Kill copy.
John McCallc56a8b32016-03-11 04:30:31 +0000687 auto argTypes = getArgTypesForCall(Context, args);
John McCall8dda7b22012-07-07 06:41:13 +0000688
John McCallc56a8b32016-03-11 04:30:31 +0000689 FunctionType::ExtInfo info = proto->getExtInfo();
Peter Collingbournef7706832014-12-12 23:41:25 +0000690 return arrangeLLVMFunctionInfo(
John McCallc56a8b32016-03-11 04:30:31 +0000691 GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
692 /*chainCall=*/false, argTypes, info, paramInfos, required);
Daniel Dunbarbf8c24a2009-02-02 23:23:47 +0000693}
694
John McCalla729c622012-02-17 03:33:10 +0000695const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
Peter Collingbournef7706832014-12-12 23:41:25 +0000696 return arrangeLLVMFunctionInfo(
697 getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
John McCallc56a8b32016-03-11 04:30:31 +0000698 None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
699}
700
701const CGFunctionInfo &
702CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
703 const CallArgList &args) {
704 assert(signature.arg_size() <= args.size());
705 if (signature.arg_size() == args.size())
706 return signature;
707
708 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
709 auto sigParamInfos = signature.getExtParameterInfos();
710 if (!sigParamInfos.empty()) {
711 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
712 paramInfos.resize(args.size());
713 }
714
715 auto argTypes = getArgTypesForCall(Context, args);
716
717 assert(signature.getRequiredArgs().allowsOptionalArgs());
718 return arrangeLLVMFunctionInfo(signature.getReturnType(),
719 signature.isInstanceMethod(),
720 signature.isChainCall(),
721 argTypes,
722 signature.getExtInfo(),
723 paramInfos,
724 signature.getRequiredArgs());
John McCalla738c252011-03-09 04:27:21 +0000725}
726
Pekka Jaaskelainenfc2629a2017-06-01 07:18:49 +0000727namespace clang {
728namespace CodeGen {
729void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
730}
731}
732
John McCalla729c622012-02-17 03:33:10 +0000733/// Arrange the argument and result information for an abstract value
734/// of a given function type. This is the method which all of the
735/// above functions ultimately defer to.
736const CGFunctionInfo &
John McCall8dda7b22012-07-07 06:41:13 +0000737CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
Peter Collingbournef7706832014-12-12 23:41:25 +0000738 bool instanceMethod,
739 bool chainCall,
John McCall8dda7b22012-07-07 06:41:13 +0000740 ArrayRef<CanQualType> argTypes,
741 FunctionType::ExtInfo info,
John McCallc56a8b32016-03-11 04:30:31 +0000742 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
John McCall8dda7b22012-07-07 06:41:13 +0000743 RequiredArgs required) {
Fangrui Song3117b172018-10-20 17:53:42 +0000744 assert(llvm::all_of(argTypes,
745 [](CanQualType T) { return T.isCanonicalAsParam(); }));
John McCall2da83a32010-02-26 00:48:12 +0000746
Daniel Dunbare0be8292009-02-03 00:07:12 +0000747 // Lookup or create unique function info.
748 llvm::FoldingSetNodeID ID;
John McCallc56a8b32016-03-11 04:30:31 +0000749 CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
750 required, resultType, argTypes);
Daniel Dunbare0be8292009-02-03 00:07:12 +0000751
Craig Topper8a13c412014-05-21 05:09:00 +0000752 void *insertPos = nullptr;
John McCalla729c622012-02-17 03:33:10 +0000753 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
Daniel Dunbare0be8292009-02-03 00:07:12 +0000754 if (FI)
755 return *FI;
756
John McCallc56a8b32016-03-11 04:30:31 +0000757 unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
758
John McCalla729c622012-02-17 03:33:10 +0000759 // Construct the function info. We co-allocate the ArgInfos.
Peter Collingbournef7706832014-12-12 23:41:25 +0000760 FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
John McCallc56a8b32016-03-11 04:30:31 +0000761 paramInfos, resultType, argTypes, required);
John McCalla729c622012-02-17 03:33:10 +0000762 FunctionInfos.InsertNode(FI, insertPos);
Daniel Dunbar313321e2009-02-03 05:31:23 +0000763
David Blaikie82e95a32014-11-19 07:49:47 +0000764 bool inserted = FunctionsBeingProcessed.insert(FI).second;
765 (void)inserted;
John McCalla729c622012-02-17 03:33:10 +0000766 assert(inserted && "Recursively being processed?");
Pekka Jaaskelainenfc2629a2017-06-01 07:18:49 +0000767
Daniel Dunbar313321e2009-02-03 05:31:23 +0000768 // Compute ABI information.
Pekka Jaaskelainenfc2629a2017-06-01 07:18:49 +0000769 if (CC == llvm::CallingConv::SPIR_KERNEL) {
770 // Force target independent argument handling for the host visible
771 // kernel functions.
772 computeSPIRKernelABIInfo(CGM, *FI);
773 } else if (info.getCC() == CC_Swift) {
John McCall12f23522016-04-04 18:33:08 +0000774 swiftcall::computeABIInfo(CGM, *FI);
Pekka Jaaskelainenfc2629a2017-06-01 07:18:49 +0000775 } else {
776 getABIInfo().computeInfo(*FI);
John McCall12f23522016-04-04 18:33:08 +0000777 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +0000778
Chris Lattnerfe34c1d2010-07-29 06:26:06 +0000779 // Loop over all of the computed argument and return value info. If any of
780 // them are direct or extend without a specified coerce type, specify the
781 // default now.
John McCalla729c622012-02-17 03:33:10 +0000782 ABIArgInfo &retInfo = FI->getReturnInfo();
Craig Topper8a13c412014-05-21 05:09:00 +0000783 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
John McCalla729c622012-02-17 03:33:10 +0000784 retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +0000785
Aaron Ballmanec47bc22014-03-17 18:10:01 +0000786 for (auto &I : FI->arguments())
Craig Topper8a13c412014-05-21 05:09:00 +0000787 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
Aaron Ballmanec47bc22014-03-17 18:10:01 +0000788 I.info.setCoerceToType(ConvertType(I.type));
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +0000789
John McCalla729c622012-02-17 03:33:10 +0000790 bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
791 assert(erased && "Not in set?");
Fangrui Song6907ce22018-07-30 19:24:48 +0000792
Daniel Dunbare0be8292009-02-03 00:07:12 +0000793 return *FI;
Daniel Dunbarbf8c24a2009-02-02 23:23:47 +0000794}
795
John McCalla729c622012-02-17 03:33:10 +0000796CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
Peter Collingbournef7706832014-12-12 23:41:25 +0000797 bool instanceMethod,
798 bool chainCall,
John McCalla729c622012-02-17 03:33:10 +0000799 const FunctionType::ExtInfo &info,
John McCallc56a8b32016-03-11 04:30:31 +0000800 ArrayRef<ExtParameterInfo> paramInfos,
John McCalla729c622012-02-17 03:33:10 +0000801 CanQualType resultType,
802 ArrayRef<CanQualType> argTypes,
803 RequiredArgs required) {
John McCallc56a8b32016-03-11 04:30:31 +0000804 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
James Y Knightb92d2902019-02-05 16:05:50 +0000805 assert(!required.allowsOptionalArgs() ||
806 required.getNumRequiredArgs() <= argTypes.size());
John McCallc56a8b32016-03-11 04:30:31 +0000807
808 void *buffer =
809 operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
810 argTypes.size() + 1, paramInfos.size()));
811
John McCalla729c622012-02-17 03:33:10 +0000812 CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
813 FI->CallingConvention = llvmCC;
814 FI->EffectiveCallingConvention = llvmCC;
815 FI->ASTCallingConvention = info.getCC();
Peter Collingbournef7706832014-12-12 23:41:25 +0000816 FI->InstanceMethod = instanceMethod;
817 FI->ChainCall = chainCall;
John McCalla729c622012-02-17 03:33:10 +0000818 FI->NoReturn = info.getNoReturn();
819 FI->ReturnsRetained = info.getProducesResult();
Oren Ben Simhon318a6ea2017-04-27 12:01:00 +0000820 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
Oren Ben Simhon220671a2018-03-17 13:31:35 +0000821 FI->NoCfCheck = info.getNoCfCheck();
John McCalla729c622012-02-17 03:33:10 +0000822 FI->Required = required;
823 FI->HasRegParm = info.getHasRegParm();
824 FI->RegParm = info.getRegParm();
Craig Topper8a13c412014-05-21 05:09:00 +0000825 FI->ArgStruct = nullptr;
John McCall7f416cc2015-09-08 08:05:57 +0000826 FI->ArgStructAlign = 0;
John McCalla729c622012-02-17 03:33:10 +0000827 FI->NumArgs = argTypes.size();
John McCallc56a8b32016-03-11 04:30:31 +0000828 FI->HasExtParameterInfos = !paramInfos.empty();
John McCalla729c622012-02-17 03:33:10 +0000829 FI->getArgsBuffer()[0].type = resultType;
830 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
831 FI->getArgsBuffer()[i + 1].type = argTypes[i];
John McCallc56a8b32016-03-11 04:30:31 +0000832 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
833 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
John McCalla729c622012-02-17 03:33:10 +0000834 return FI;
Daniel Dunbar313321e2009-02-03 05:31:23 +0000835}
836
837/***/
838
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000839namespace {
840// ABIArgInfo::Expand implementation.
841
842// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
843struct TypeExpansion {
844 enum TypeExpansionKind {
845 // Elements of constant arrays are expanded recursively.
846 TEK_ConstantArray,
847 // Record fields are expanded recursively (but if record is a union, only
848 // the field with the largest size is expanded).
849 TEK_Record,
850 // For complex types, real and imaginary parts are expanded recursively.
851 TEK_Complex,
852 // All other types are not expandable.
853 TEK_None
854 };
855
856 const TypeExpansionKind Kind;
857
858 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
Angel Garcia Gomez637d1e62015-10-20 13:23:58 +0000859 virtual ~TypeExpansion() {}
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000860};
861
862struct ConstantArrayExpansion : TypeExpansion {
863 QualType EltTy;
864 uint64_t NumElts;
865
866 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
867 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
868 static bool classof(const TypeExpansion *TE) {
869 return TE->Kind == TEK_ConstantArray;
870 }
871};
872
873struct RecordExpansion : TypeExpansion {
Reid Klecknere9f6a712014-10-31 17:10:41 +0000874 SmallVector<const CXXBaseSpecifier *, 1> Bases;
875
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000876 SmallVector<const FieldDecl *, 1> Fields;
877
Reid Klecknere9f6a712014-10-31 17:10:41 +0000878 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
879 SmallVector<const FieldDecl *, 1> &&Fields)
Benjamin Kramer0bb97742016-02-13 16:00:13 +0000880 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
881 Fields(std::move(Fields)) {}
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000882 static bool classof(const TypeExpansion *TE) {
883 return TE->Kind == TEK_Record;
884 }
885};
886
887struct ComplexExpansion : TypeExpansion {
888 QualType EltTy;
889
890 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
891 static bool classof(const TypeExpansion *TE) {
892 return TE->Kind == TEK_Complex;
893 }
894};
895
896struct NoExpansion : TypeExpansion {
897 NoExpansion() : TypeExpansion(TEK_None) {}
898 static bool classof(const TypeExpansion *TE) {
899 return TE->Kind == TEK_None;
900 }
901};
902} // namespace
903
904static std::unique_ptr<TypeExpansion>
905getTypeExpansion(QualType Ty, const ASTContext &Context) {
906 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
Jonas Devlieghere2b3d49b2019-08-14 23:04:18 +0000907 return std::make_unique<ConstantArrayExpansion>(
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000908 AT->getElementType(), AT->getSize().getZExtValue());
909 }
910 if (const RecordType *RT = Ty->getAs<RecordType>()) {
Reid Klecknere9f6a712014-10-31 17:10:41 +0000911 SmallVector<const CXXBaseSpecifier *, 1> Bases;
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000912 SmallVector<const FieldDecl *, 1> Fields;
Bob Wilsone826a2a2011-08-03 05:58:22 +0000913 const RecordDecl *RD = RT->getDecl();
914 assert(!RD->hasFlexibleArrayMember() &&
915 "Cannot expand structure with flexible array.");
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000916 if (RD->isUnion()) {
917 // Unions can be here only in degenerative cases - all the fields are same
918 // after flattening. Thus we have to use the "largest" field.
Craig Topper8a13c412014-05-21 05:09:00 +0000919 const FieldDecl *LargestFD = nullptr;
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000920 CharUnits UnionSize = CharUnits::Zero();
921
Aaron Ballmane8a8bae2014-03-08 20:12:42 +0000922 for (const auto *FD : RD->fields()) {
Richard Smith866dee42018-04-02 18:29:43 +0000923 if (FD->isZeroLengthBitField(Context))
Reid Kleckner80944df2014-10-31 22:00:51 +0000924 continue;
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000925 assert(!FD->isBitField() &&
926 "Cannot expand structure with bit-field members.");
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000927 CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000928 if (UnionSize < FieldSize) {
929 UnionSize = FieldSize;
930 LargestFD = FD;
931 }
932 }
933 if (LargestFD)
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000934 Fields.push_back(LargestFD);
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000935 } else {
Reid Klecknere9f6a712014-10-31 17:10:41 +0000936 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
937 assert(!CXXRD->isDynamicClass() &&
938 "cannot expand vtable pointers in dynamic classes");
939 for (const CXXBaseSpecifier &BS : CXXRD->bases())
940 Bases.push_back(&BS);
941 }
942
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000943 for (const auto *FD : RD->fields()) {
Richard Smith866dee42018-04-02 18:29:43 +0000944 if (FD->isZeroLengthBitField(Context))
Reid Kleckner80944df2014-10-31 22:00:51 +0000945 continue;
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000946 assert(!FD->isBitField() &&
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000947 "Cannot expand structure with bit-field members.");
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000948 Fields.push_back(FD);
Anton Korobeynikov4215ca72012-04-13 11:22:00 +0000949 }
Bob Wilsone826a2a2011-08-03 05:58:22 +0000950 }
Jonas Devlieghere2b3d49b2019-08-14 23:04:18 +0000951 return std::make_unique<RecordExpansion>(std::move(Bases),
Reid Klecknere9f6a712014-10-31 17:10:41 +0000952 std::move(Fields));
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000953 }
954 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
Jonas Devlieghere2b3d49b2019-08-14 23:04:18 +0000955 return std::make_unique<ComplexExpansion>(CT->getElementType());
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000956 }
Jonas Devlieghere2b3d49b2019-08-14 23:04:18 +0000957 return std::make_unique<NoExpansion>();
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000958}
959
Alexey Samsonov52c0f6a2014-09-29 20:30:22 +0000960static int getExpansionSize(QualType Ty, const ASTContext &Context) {
961 auto Exp = getTypeExpansion(Ty, Context);
962 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
963 return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
964 }
965 if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
966 int Res = 0;
Reid Klecknere9f6a712014-10-31 17:10:41 +0000967 for (auto BS : RExp->Bases)
968 Res += getExpansionSize(BS->getType(), Context);
Alexey Samsonov52c0f6a2014-09-29 20:30:22 +0000969 for (auto FD : RExp->Fields)
970 Res += getExpansionSize(FD->getType(), Context);
971 return Res;
972 }
973 if (isa<ComplexExpansion>(Exp.get()))
974 return 2;
975 assert(isa<NoExpansion>(Exp.get()));
976 return 1;
977}
978
Alexey Samsonov153004f2014-09-29 22:08:00 +0000979void
980CodeGenTypes::getExpandedTypes(QualType Ty,
981 SmallVectorImpl<llvm::Type *>::iterator &TI) {
982 auto Exp = getTypeExpansion(Ty, Context);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000983 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
984 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
Alexey Samsonov153004f2014-09-29 22:08:00 +0000985 getExpandedTypes(CAExp->EltTy, TI);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000986 }
987 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
Reid Klecknere9f6a712014-10-31 17:10:41 +0000988 for (auto BS : RExp->Bases)
989 getExpandedTypes(BS->getType(), TI);
990 for (auto FD : RExp->Fields)
Alexey Samsonov153004f2014-09-29 22:08:00 +0000991 getExpandedTypes(FD->getType(), TI);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000992 } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
993 llvm::Type *EltTy = ConvertType(CExp->EltTy);
Alexey Samsonov153004f2014-09-29 22:08:00 +0000994 *TI++ = EltTy;
995 *TI++ = EltTy;
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000996 } else {
997 assert(isa<NoExpansion>(Exp.get()));
Alexey Samsonov153004f2014-09-29 22:08:00 +0000998 *TI++ = ConvertType(Ty);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +0000999 }
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00001000}
1001
John McCall7f416cc2015-09-08 08:05:57 +00001002static void forConstantArrayExpansion(CodeGenFunction &CGF,
1003 ConstantArrayExpansion *CAE,
1004 Address BaseAddr,
1005 llvm::function_ref<void(Address)> Fn) {
1006 CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1007 CharUnits EltAlign =
1008 BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1009
1010 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1011 llvm::Value *EltAddr =
1012 CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
1013 Fn(Address(EltAddr, EltAlign));
1014 }
1015}
1016
Alexey Samsonov91cf4552014-08-22 01:06:06 +00001017void CodeGenFunction::ExpandTypeFromArgs(
John McCall12f23522016-04-04 18:33:08 +00001018 QualType Ty, LValue LV, SmallVectorImpl<llvm::Value *>::iterator &AI) {
Mike Stump11289f42009-09-09 15:08:12 +00001019 assert(LV.isSimple() &&
1020 "Unexpected non-simple lvalue during struct expansion.");
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00001021
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001022 auto Exp = getTypeExpansion(Ty, getContext());
1023 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001024 forConstantArrayExpansion(
1025 *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1026 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1027 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1028 });
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001029 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001030 Address This = LV.getAddress(*this);
Reid Klecknere9f6a712014-10-31 17:10:41 +00001031 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1032 // Perform a single step derived-to-base conversion.
John McCall7f416cc2015-09-08 08:05:57 +00001033 Address Base =
Reid Klecknere9f6a712014-10-31 17:10:41 +00001034 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1035 /*NullCheckValue=*/false, SourceLocation());
1036 LValue SubLV = MakeAddrLValue(Base, BS->getType());
1037
1038 // Recurse onto bases.
1039 ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1040 }
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001041 for (auto FD : RExp->Fields) {
1042 // FIXME: What are the right qualifiers here?
Reid Kleckner9d031092016-05-02 22:42:34 +00001043 LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001044 ExpandTypeFromArgs(FD->getType(), SubLV, AI);
Bob Wilsone826a2a2011-08-03 05:58:22 +00001045 }
John McCall7f416cc2015-09-08 08:05:57 +00001046 } else if (isa<ComplexExpansion>(Exp.get())) {
1047 auto realValue = *AI++;
1048 auto imagValue = *AI++;
1049 EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001050 } else {
Akira Hatanakad8136f12019-11-29 09:56:02 -08001051 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1052 // primitive store.
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001053 assert(isa<NoExpansion>(Exp.get()));
Akira Hatanakad8136f12019-11-29 09:56:02 -08001054 if (LV.isBitField())
1055 EmitStoreThroughLValue(RValue::get(*AI++), LV);
1056 else
1057 EmitStoreOfScalar(*AI++, LV);
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00001058 }
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001059}
1060
1061void CodeGenFunction::ExpandTypeToArgs(
Yaxun Liu5b330e82018-03-15 15:25:19 +00001062 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001063 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1064 auto Exp = getTypeExpansion(Ty, getContext());
1065 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001066 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00001067 : Arg.getKnownRValue().getAggregateAddress();
1068 forConstantArrayExpansion(
1069 *this, CAExp, Addr, [&](Address EltAddr) {
1070 CallArg EltArg = CallArg(
1071 convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1072 CAExp->EltTy);
1073 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1074 IRCallArgPos);
1075 });
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001076 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001077 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00001078 : Arg.getKnownRValue().getAggregateAddress();
Reid Klecknere9f6a712014-10-31 17:10:41 +00001079 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1080 // Perform a single step derived-to-base conversion.
John McCall7f416cc2015-09-08 08:05:57 +00001081 Address Base =
Reid Klecknere9f6a712014-10-31 17:10:41 +00001082 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1083 /*NullCheckValue=*/false, SourceLocation());
Yaxun Liu5b330e82018-03-15 15:25:19 +00001084 CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
Reid Klecknere9f6a712014-10-31 17:10:41 +00001085
1086 // Recurse onto bases.
Yaxun Liu5b330e82018-03-15 15:25:19 +00001087 ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
Reid Klecknere9f6a712014-10-31 17:10:41 +00001088 IRCallArgPos);
1089 }
1090
1091 LValue LV = MakeAddrLValue(This, Ty);
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001092 for (auto FD : RExp->Fields) {
Yaxun Liu5b330e82018-03-15 15:25:19 +00001093 CallArg FldArg =
1094 CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1095 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001096 IRCallArgPos);
1097 }
1098 } else if (isa<ComplexExpansion>(Exp.get())) {
Yaxun Liu5b330e82018-03-15 15:25:19 +00001099 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001100 IRCallArgs[IRCallArgPos++] = CV.first;
1101 IRCallArgs[IRCallArgPos++] = CV.second;
1102 } else {
1103 assert(isa<NoExpansion>(Exp.get()));
Yaxun Liu5b330e82018-03-15 15:25:19 +00001104 auto RV = Arg.getKnownRValue();
Alexey Samsonov8a0bad02014-09-29 18:41:28 +00001105 assert(RV.isScalar() &&
1106 "Unexpected non-scalar rvalue during struct expansion.");
1107
1108 // Insert a bitcast as needed.
1109 llvm::Value *V = RV.getScalarVal();
1110 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1111 V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1112 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1113
1114 IRCallArgs[IRCallArgPos++] = V;
1115 }
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00001116}
1117
John McCall7f416cc2015-09-08 08:05:57 +00001118/// Create a temporary allocation for the purposes of coercion.
1119static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1120 CharUnits MinAlign) {
1121 // Don't use an alignment that's worse than what LLVM would prefer.
1122 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1123 CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1124
1125 return CGF.CreateTempAlloca(Ty, Align);
1126}
1127
Chris Lattner895c52b2010-06-27 06:04:18 +00001128/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
Chris Lattner1cd66982010-06-27 05:56:15 +00001129/// accessing some number of bytes out of it, try to gep into the struct to get
1130/// at its inner goodness. Dive as deep as possible without entering an element
1131/// with an in-memory size smaller than DstSize.
John McCall7f416cc2015-09-08 08:05:57 +00001132static Address
1133EnterStructPointerForCoercedAccess(Address SrcPtr,
Chris Lattner2192fe52011-07-18 04:24:23 +00001134 llvm::StructType *SrcSTy,
Chris Lattner895c52b2010-06-27 06:04:18 +00001135 uint64_t DstSize, CodeGenFunction &CGF) {
Chris Lattner1cd66982010-06-27 05:56:15 +00001136 // We can't dive into a zero-element struct.
1137 if (SrcSTy->getNumElements() == 0) return SrcPtr;
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001138
Chris Lattner2192fe52011-07-18 04:24:23 +00001139 llvm::Type *FirstElt = SrcSTy->getElementType(0);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001140
Chris Lattner1cd66982010-06-27 05:56:15 +00001141 // If the first elt is at least as large as what we're looking for, or if the
James Molloy90d61012014-08-29 10:17:52 +00001142 // first element is the same size as the whole struct, we can enter it. The
1143 // comparison must be made on the store size and not the alloca size. Using
1144 // the alloca size may overstate the size of the load.
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001145 uint64_t FirstEltSize =
James Molloy90d61012014-08-29 10:17:52 +00001146 CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001147 if (FirstEltSize < DstSize &&
James Molloy90d61012014-08-29 10:17:52 +00001148 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
Chris Lattner1cd66982010-06-27 05:56:15 +00001149 return SrcPtr;
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001150
Chris Lattner1cd66982010-06-27 05:56:15 +00001151 // GEP into the first element.
James Y Knight751fe282019-02-09 22:22:28 +00001152 SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001153
Chris Lattner1cd66982010-06-27 05:56:15 +00001154 // If the first element is a struct, recurse.
John McCall7f416cc2015-09-08 08:05:57 +00001155 llvm::Type *SrcTy = SrcPtr.getElementType();
Chris Lattner2192fe52011-07-18 04:24:23 +00001156 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
Chris Lattner895c52b2010-06-27 06:04:18 +00001157 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
Chris Lattner1cd66982010-06-27 05:56:15 +00001158
1159 return SrcPtr;
1160}
1161
Chris Lattner055097f2010-06-27 06:26:04 +00001162/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1163/// are either integers or pointers. This does a truncation of the value if it
1164/// is too large or a zero extension if it is too small.
Jakob Stoklund Olesen36af2522013-06-05 03:00:13 +00001165///
1166/// This behaves as if the value were coerced through memory, so on big-endian
1167/// targets the high bits are preserved in a truncation, while little-endian
1168/// targets preserve the low bits.
Chris Lattner055097f2010-06-27 06:26:04 +00001169static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
Chris Lattner2192fe52011-07-18 04:24:23 +00001170 llvm::Type *Ty,
Chris Lattner055097f2010-06-27 06:26:04 +00001171 CodeGenFunction &CGF) {
1172 if (Val->getType() == Ty)
1173 return Val;
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001174
Chris Lattner055097f2010-06-27 06:26:04 +00001175 if (isa<llvm::PointerType>(Val->getType())) {
1176 // If this is Pointer->Pointer avoid conversion to and from int.
1177 if (isa<llvm::PointerType>(Ty))
1178 return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001179
Chris Lattner055097f2010-06-27 06:26:04 +00001180 // Convert the pointer to an integer so we can play with its width.
Chris Lattner5e016ae2010-06-27 07:15:29 +00001181 Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
Chris Lattner055097f2010-06-27 06:26:04 +00001182 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001183
Chris Lattner2192fe52011-07-18 04:24:23 +00001184 llvm::Type *DestIntTy = Ty;
Chris Lattner055097f2010-06-27 06:26:04 +00001185 if (isa<llvm::PointerType>(DestIntTy))
Chris Lattner5e016ae2010-06-27 07:15:29 +00001186 DestIntTy = CGF.IntPtrTy;
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001187
Jakob Stoklund Olesen36af2522013-06-05 03:00:13 +00001188 if (Val->getType() != DestIntTy) {
1189 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1190 if (DL.isBigEndian()) {
1191 // Preserve the high bits on big-endian targets.
1192 // That is what memory coercion does.
James Molloy491cefb2014-05-07 17:41:15 +00001193 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1194 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1195
Jakob Stoklund Olesen36af2522013-06-05 03:00:13 +00001196 if (SrcSize > DstSize) {
1197 Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1198 Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1199 } else {
1200 Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1201 Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1202 }
1203 } else {
1204 // Little-endian targets preserve the low bits. No shifts required.
1205 Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1206 }
1207 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001208
Chris Lattner055097f2010-06-27 06:26:04 +00001209 if (isa<llvm::PointerType>(Ty))
1210 Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1211 return Val;
1212}
1213
Chris Lattner1cd66982010-06-27 05:56:15 +00001214
1215
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001216/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
Ulrich Weigand6e2cea62015-07-10 11:31:43 +00001217/// a pointer to an object of type \arg Ty, known to be aligned to
1218/// \arg SrcAlign bytes.
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001219///
1220/// This safely handles the case when the src type is smaller than the
1221/// destination type; in this situation the values of bits which not
1222/// present in the src are undefined.
John McCall7f416cc2015-09-08 08:05:57 +00001223static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001224 CodeGenFunction &CGF) {
John McCall7f416cc2015-09-08 08:05:57 +00001225 llvm::Type *SrcTy = Src.getElementType();
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001226
Chris Lattnerd200eda2010-06-28 22:51:39 +00001227 // If SrcTy and Ty are the same, just do a load.
1228 if (SrcTy == Ty)
John McCall7f416cc2015-09-08 08:05:57 +00001229 return CGF.Builder.CreateLoad(Src);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001230
Micah Villmowdd31ca12012-10-08 16:25:52 +00001231 uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001232
Chris Lattner2192fe52011-07-18 04:24:23 +00001233 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
John McCall7f416cc2015-09-08 08:05:57 +00001234 Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF);
1235 SrcTy = Src.getType()->getElementType();
Chris Lattner1cd66982010-06-27 05:56:15 +00001236 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001237
Micah Villmowdd31ca12012-10-08 16:25:52 +00001238 uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001239
Chris Lattner055097f2010-06-27 06:26:04 +00001240 // If the source and destination are integer or pointer types, just do an
1241 // extension or truncation to the desired type.
1242 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1243 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
John McCall7f416cc2015-09-08 08:05:57 +00001244 llvm::Value *Load = CGF.Builder.CreateLoad(Src);
Chris Lattner055097f2010-06-27 06:26:04 +00001245 return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1246 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001247
Daniel Dunbarb52d0772009-02-03 05:59:18 +00001248 // If load is legal, just bitcast the src pointer.
Daniel Dunbarffdb8432009-05-13 18:54:26 +00001249 if (SrcSize >= DstSize) {
Mike Stump18bb9282009-05-16 07:57:57 +00001250 // Generally SrcSize is never greater than DstSize, since this means we are
1251 // losing bits. However, this can happen in cases where the structure has
1252 // additional padding, for example due to a user specified alignment.
Daniel Dunbarffdb8432009-05-13 18:54:26 +00001253 //
Mike Stump18bb9282009-05-16 07:57:57 +00001254 // FIXME: Assert that we aren't truncating non-padding bits when have access
1255 // to that information.
Matt Arsenault7a124f32017-08-01 20:36:57 +00001256 Src = CGF.Builder.CreateBitCast(Src,
1257 Ty->getPointerTo(Src.getAddressSpace()));
John McCall7f416cc2015-09-08 08:05:57 +00001258 return CGF.Builder.CreateLoad(Src);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001259 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001260
John McCall7f416cc2015-09-08 08:05:57 +00001261 // Otherwise do coercion through memory. This is stupid, but simple.
1262 Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment());
Yaxun Liu4bbdebc2018-11-08 16:55:46 +00001263 Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1264 Address SrcCasted = CGF.Builder.CreateElementBitCast(Src,CGF.Int8Ty);
Manman Ren84b921f2012-11-28 22:08:52 +00001265 CGF.Builder.CreateMemCpy(Casted, SrcCasted,
1266 llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
John McCall7f416cc2015-09-08 08:05:57 +00001267 false);
1268 return CGF.Builder.CreateLoad(Tmp);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001269}
1270
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001271// Function to store a first-class aggregate into memory. We prefer to
1272// store the elements rather than the aggregate to be more friendly to
1273// fast-isel.
1274// FIXME: Do we need to recurse here?
1275static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
John McCall7f416cc2015-09-08 08:05:57 +00001276 Address Dest, bool DestIsVolatile) {
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001277 // Prefer scalar stores to first-class aggregate stores.
Chris Lattner2192fe52011-07-18 04:24:23 +00001278 if (llvm::StructType *STy =
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001279 dyn_cast<llvm::StructType>(Val->getType())) {
1280 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
James Y Knight751fe282019-02-09 22:22:28 +00001281 Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i);
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001282 llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
John McCall7f416cc2015-09-08 08:05:57 +00001283 CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001284 }
1285 } else {
John McCall7f416cc2015-09-08 08:05:57 +00001286 CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
Eli Friedmanaf9b3252011-05-17 21:08:01 +00001287 }
1288}
1289
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001290/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
Ulrich Weigand6e2cea62015-07-10 11:31:43 +00001291/// where the source and destination may have different types. The
1292/// destination is known to be aligned to \arg DstAlign bytes.
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001293///
1294/// This safely handles the case when the src type is larger than the
1295/// destination type; the upper bits of the src will be lost.
1296static void CreateCoercedStore(llvm::Value *Src,
John McCall7f416cc2015-09-08 08:05:57 +00001297 Address Dst,
Anders Carlsson17490832009-12-24 20:40:36 +00001298 bool DstIsVolatile,
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001299 CodeGenFunction &CGF) {
Chris Lattner2192fe52011-07-18 04:24:23 +00001300 llvm::Type *SrcTy = Src->getType();
John McCall7f416cc2015-09-08 08:05:57 +00001301 llvm::Type *DstTy = Dst.getType()->getElementType();
Chris Lattnerd200eda2010-06-28 22:51:39 +00001302 if (SrcTy == DstTy) {
John McCall7f416cc2015-09-08 08:05:57 +00001303 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
Chris Lattnerd200eda2010-06-28 22:51:39 +00001304 return;
1305 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001306
Micah Villmowdd31ca12012-10-08 16:25:52 +00001307 uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001308
Chris Lattner2192fe52011-07-18 04:24:23 +00001309 if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
John McCall7f416cc2015-09-08 08:05:57 +00001310 Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF);
1311 DstTy = Dst.getType()->getElementType();
Chris Lattner895c52b2010-06-27 06:04:18 +00001312 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001313
Michael Liao15140e42019-11-04 11:41:07 -05001314 llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1315 llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1316 if (SrcPtrTy && DstPtrTy &&
1317 SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1318 Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1319 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1320 return;
1321 }
1322
Chris Lattner055097f2010-06-27 06:26:04 +00001323 // If the source and destination are integer or pointer types, just do an
1324 // extension or truncation to the desired type.
1325 if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1326 (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1327 Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
John McCall7f416cc2015-09-08 08:05:57 +00001328 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
Chris Lattner055097f2010-06-27 06:26:04 +00001329 return;
1330 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001331
Micah Villmowdd31ca12012-10-08 16:25:52 +00001332 uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001333
Daniel Dunbar313321e2009-02-03 05:31:23 +00001334 // If store is legal, just bitcast the src pointer.
Daniel Dunbar4be99ff2009-06-05 07:58:54 +00001335 if (SrcSize <= DstSize) {
Yaxun Liue9e5c4f2017-06-29 18:47:45 +00001336 Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
John McCall7f416cc2015-09-08 08:05:57 +00001337 BuildAggStore(CGF, Src, Dst, DstIsVolatile);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001338 } else {
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001339 // Otherwise do coercion through memory. This is stupid, but
1340 // simple.
Daniel Dunbar4be99ff2009-06-05 07:58:54 +00001341
1342 // Generally SrcSize is never greater than DstSize, since this means we are
1343 // losing bits. However, this can happen in cases where the structure has
1344 // additional padding, for example due to a user specified alignment.
1345 //
1346 // FIXME: Assert that we aren't truncating non-padding bits when have access
1347 // to that information.
John McCall7f416cc2015-09-08 08:05:57 +00001348 Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1349 CGF.Builder.CreateStore(Src, Tmp);
Yaxun Liu4bbdebc2018-11-08 16:55:46 +00001350 Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1351 Address DstCasted = CGF.Builder.CreateElementBitCast(Dst,CGF.Int8Ty);
Manman Ren84b921f2012-11-28 22:08:52 +00001352 CGF.Builder.CreateMemCpy(DstCasted, Casted,
1353 llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
John McCall7f416cc2015-09-08 08:05:57 +00001354 false);
Daniel Dunbarf5589ac2009-02-02 19:06:38 +00001355 }
1356}
1357
John McCall7f416cc2015-09-08 08:05:57 +00001358static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
Fangrui Song6907ce22018-07-30 19:24:48 +00001359 const ABIArgInfo &info) {
John McCall7f416cc2015-09-08 08:05:57 +00001360 if (unsigned offset = info.getDirectOffset()) {
1361 addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1362 addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1363 CharUnits::fromQuantity(offset));
1364 addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1365 }
1366 return addr;
1367}
1368
Alexey Samsonov153004f2014-09-29 22:08:00 +00001369namespace {
1370
1371/// Encapsulates information about the way function arguments from
1372/// CGFunctionInfo should be passed to actual LLVM IR function.
1373class ClangToLLVMArgMapping {
1374 static const unsigned InvalidIndex = ~0U;
1375 unsigned InallocaArgNo;
1376 unsigned SRetArgNo;
1377 unsigned TotalIRArgs;
1378
1379 /// Arguments of LLVM IR function corresponding to single Clang argument.
1380 struct IRArgs {
1381 unsigned PaddingArgIndex;
1382 // Argument is expanded to IR arguments at positions
1383 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1384 unsigned FirstArgIndex;
1385 unsigned NumberOfArgs;
1386
1387 IRArgs()
1388 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1389 NumberOfArgs(0) {}
1390 };
1391
1392 SmallVector<IRArgs, 8> ArgInfo;
1393
1394public:
1395 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1396 bool OnlyRequiredArgs = false)
1397 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1398 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1399 construct(Context, FI, OnlyRequiredArgs);
1400 }
1401
1402 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1403 unsigned getInallocaArgNo() const {
1404 assert(hasInallocaArg());
1405 return InallocaArgNo;
1406 }
1407
1408 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1409 unsigned getSRetArgNo() const {
1410 assert(hasSRetArg());
1411 return SRetArgNo;
1412 }
1413
1414 unsigned totalIRArgs() const { return TotalIRArgs; }
1415
1416 bool hasPaddingArg(unsigned ArgNo) const {
1417 assert(ArgNo < ArgInfo.size());
1418 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1419 }
1420 unsigned getPaddingArgNo(unsigned ArgNo) const {
1421 assert(hasPaddingArg(ArgNo));
1422 return ArgInfo[ArgNo].PaddingArgIndex;
1423 }
1424
1425 /// Returns index of first IR argument corresponding to ArgNo, and their
1426 /// quantity.
1427 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1428 assert(ArgNo < ArgInfo.size());
1429 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1430 ArgInfo[ArgNo].NumberOfArgs);
1431 }
1432
1433private:
1434 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1435 bool OnlyRequiredArgs);
1436};
1437
1438void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1439 const CGFunctionInfo &FI,
1440 bool OnlyRequiredArgs) {
1441 unsigned IRArgNo = 0;
1442 bool SwapThisWithSRet = false;
1443 const ABIArgInfo &RetAI = FI.getReturnInfo();
1444
1445 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1446 SwapThisWithSRet = RetAI.isSRetAfterThis();
1447 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1448 }
1449
1450 unsigned ArgNo = 0;
1451 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1452 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1453 ++I, ++ArgNo) {
1454 assert(I != FI.arg_end());
1455 QualType ArgType = I->type;
1456 const ABIArgInfo &AI = I->info;
1457 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1458 auto &IRArgs = ArgInfo[ArgNo];
1459
1460 if (AI.getPaddingType())
1461 IRArgs.PaddingArgIndex = IRArgNo++;
1462
1463 switch (AI.getKind()) {
1464 case ABIArgInfo::Extend:
1465 case ABIArgInfo::Direct: {
1466 // FIXME: handle sseregparm someday...
1467 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1468 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1469 IRArgs.NumberOfArgs = STy->getNumElements();
1470 } else {
1471 IRArgs.NumberOfArgs = 1;
1472 }
1473 break;
1474 }
1475 case ABIArgInfo::Indirect:
1476 IRArgs.NumberOfArgs = 1;
1477 break;
1478 case ABIArgInfo::Ignore:
1479 case ABIArgInfo::InAlloca:
1480 // ignore and inalloca doesn't have matching LLVM parameters.
1481 IRArgs.NumberOfArgs = 0;
1482 break;
John McCallf26e73d2016-03-11 04:30:43 +00001483 case ABIArgInfo::CoerceAndExpand:
1484 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1485 break;
1486 case ABIArgInfo::Expand:
Alexey Samsonov153004f2014-09-29 22:08:00 +00001487 IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1488 break;
1489 }
Alexey Samsonov153004f2014-09-29 22:08:00 +00001490
1491 if (IRArgs.NumberOfArgs > 0) {
1492 IRArgs.FirstArgIndex = IRArgNo;
1493 IRArgNo += IRArgs.NumberOfArgs;
1494 }
1495
1496 // Skip over the sret parameter when it comes second. We already handled it
1497 // above.
1498 if (IRArgNo == 1 && SwapThisWithSRet)
1499 IRArgNo++;
1500 }
1501 assert(ArgNo == ArgInfo.size());
1502
1503 if (FI.usesInAlloca())
1504 InallocaArgNo = IRArgNo++;
1505
1506 TotalIRArgs = IRArgNo;
1507}
1508} // namespace
1509
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00001510/***/
1511
Daniel Dunbar6f2e8392010-07-14 23:39:36 +00001512bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
Saleem Abdulrasoolf181f1a2018-02-28 20:16:12 +00001513 const auto &RI = FI.getReturnInfo();
1514 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
Daniel Dunbar7633cbf2009-02-02 21:43:58 +00001515}
1516
Tim Northovere77cc392014-03-29 13:28:05 +00001517bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1518 return ReturnTypeUsesSRet(FI) &&
1519 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1520}
1521
Daniel Dunbar6f2e8392010-07-14 23:39:36 +00001522bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1523 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1524 switch (BT->getKind()) {
1525 default:
1526 return false;
1527 case BuiltinType::Float:
John McCallc8e01702013-04-16 22:48:15 +00001528 return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
Daniel Dunbar6f2e8392010-07-14 23:39:36 +00001529 case BuiltinType::Double:
John McCallc8e01702013-04-16 22:48:15 +00001530 return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
Daniel Dunbar6f2e8392010-07-14 23:39:36 +00001531 case BuiltinType::LongDouble:
John McCallc8e01702013-04-16 22:48:15 +00001532 return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
Daniel Dunbar6f2e8392010-07-14 23:39:36 +00001533 }
1534 }
1535
1536 return false;
1537}
1538
Anders Carlsson2f1a6c32011-10-31 16:27:11 +00001539bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1540 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1541 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1542 if (BT->getKind() == BuiltinType::LongDouble)
John McCallc8e01702013-04-16 22:48:15 +00001543 return getTarget().useObjCFP2RetForComplexLongDouble();
Anders Carlsson2f1a6c32011-10-31 16:27:11 +00001544 }
1545 }
1546
1547 return false;
1548}
1549
Chris Lattnera5f58b02011-07-09 17:41:47 +00001550llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
John McCalla729c622012-02-17 03:33:10 +00001551 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1552 return GetFunctionType(FI);
John McCallf8ff7b92010-02-23 00:48:20 +00001553}
1554
Chris Lattnera5f58b02011-07-09 17:41:47 +00001555llvm::FunctionType *
John McCalla729c622012-02-17 03:33:10 +00001556CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
Alexey Samsonov153004f2014-09-29 22:08:00 +00001557
David Blaikie82e95a32014-11-19 07:49:47 +00001558 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1559 (void)Inserted;
Chris Lattner6fb0ccf2011-07-15 05:16:14 +00001560 assert(Inserted && "Recursively being processed?");
Daniel Dunbar7a95ca32008-09-10 04:01:49 +00001561
Alexey Samsonov153004f2014-09-29 22:08:00 +00001562 llvm::Type *resultType = nullptr;
John McCall85dd2c52011-05-15 02:19:42 +00001563 const ABIArgInfo &retAI = FI.getReturnInfo();
1564 switch (retAI.getKind()) {
Daniel Dunbard3674e62008-09-11 01:48:57 +00001565 case ABIArgInfo::Expand:
John McCall85dd2c52011-05-15 02:19:42 +00001566 llvm_unreachable("Invalid ABI kind for return argument");
Daniel Dunbard3674e62008-09-11 01:48:57 +00001567
Anton Korobeynikov18adbf52009-06-06 09:36:29 +00001568 case ABIArgInfo::Extend:
Daniel Dunbar67dace892009-02-03 06:17:37 +00001569 case ABIArgInfo::Direct:
John McCall85dd2c52011-05-15 02:19:42 +00001570 resultType = retAI.getCoerceToType();
Daniel Dunbar67dace892009-02-03 06:17:37 +00001571 break;
1572
Reid Kleckner314ef7b2014-02-01 00:04:45 +00001573 case ABIArgInfo::InAlloca:
Reid Klecknerfab1e892014-02-25 00:59:14 +00001574 if (retAI.getInAllocaSRet()) {
1575 // sret things on win32 aren't void, they return the sret pointer.
1576 QualType ret = FI.getReturnType();
1577 llvm::Type *ty = ConvertType(ret);
1578 unsigned addressSpace = Context.getTargetAddressSpace(ret);
1579 resultType = llvm::PointerType::get(ty, addressSpace);
1580 } else {
1581 resultType = llvm::Type::getVoidTy(getLLVMContext());
1582 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00001583 break;
1584
John McCall7f416cc2015-09-08 08:05:57 +00001585 case ABIArgInfo::Indirect:
Daniel Dunbar94a6f252009-01-26 21:26:08 +00001586 case ABIArgInfo::Ignore:
John McCall85dd2c52011-05-15 02:19:42 +00001587 resultType = llvm::Type::getVoidTy(getLLVMContext());
Daniel Dunbar94a6f252009-01-26 21:26:08 +00001588 break;
John McCallf26e73d2016-03-11 04:30:43 +00001589
1590 case ABIArgInfo::CoerceAndExpand:
1591 resultType = retAI.getUnpaddedCoerceAndExpandType();
1592 break;
Daniel Dunbar7a95ca32008-09-10 04:01:49 +00001593 }
Mike Stump11289f42009-09-09 15:08:12 +00001594
Alexey Samsonov153004f2014-09-29 22:08:00 +00001595 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1596 SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1597
1598 // Add type for sret argument.
1599 if (IRFunctionArgs.hasSRetArg()) {
1600 QualType Ret = FI.getReturnType();
1601 llvm::Type *Ty = ConvertType(Ret);
1602 unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1603 ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1604 llvm::PointerType::get(Ty, AddressSpace);
1605 }
1606
1607 // Add type for inalloca argument.
1608 if (IRFunctionArgs.hasInallocaArg()) {
1609 auto ArgStruct = FI.getArgStruct();
1610 assert(ArgStruct);
1611 ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1612 }
1613
John McCallc818bbb2012-12-07 07:03:17 +00001614 // Add in all of the required arguments.
Alexey Samsonov153004f2014-09-29 22:08:00 +00001615 unsigned ArgNo = 0;
Alexey Samsonov34625dd2014-09-29 21:21:48 +00001616 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1617 ie = it + FI.getNumRequiredArgs();
Alexey Samsonov153004f2014-09-29 22:08:00 +00001618 for (; it != ie; ++it, ++ArgNo) {
1619 const ABIArgInfo &ArgInfo = it->info;
Mike Stump11289f42009-09-09 15:08:12 +00001620
Rafael Espindolafad28de2012-10-24 01:59:00 +00001621 // Insert a padding type to ensure proper alignment.
Alexey Samsonov153004f2014-09-29 22:08:00 +00001622 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1623 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1624 ArgInfo.getPaddingType();
Rafael Espindolafad28de2012-10-24 01:59:00 +00001625
Alexey Samsonov153004f2014-09-29 22:08:00 +00001626 unsigned FirstIRArg, NumIRArgs;
1627 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1628
1629 switch (ArgInfo.getKind()) {
Daniel Dunbar94a6f252009-01-26 21:26:08 +00001630 case ABIArgInfo::Ignore:
Reid Kleckner314ef7b2014-02-01 00:04:45 +00001631 case ABIArgInfo::InAlloca:
Alexey Samsonov153004f2014-09-29 22:08:00 +00001632 assert(NumIRArgs == 0);
Daniel Dunbar94a6f252009-01-26 21:26:08 +00001633 break;
1634
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00001635 case ABIArgInfo::Indirect: {
Alexey Samsonov153004f2014-09-29 22:08:00 +00001636 assert(NumIRArgs == 1);
Yaxun Liud7523282017-04-17 20:10:44 +00001637 // indirect arguments are always on the stack, which is alloca addr space.
Chris Lattner2192fe52011-07-18 04:24:23 +00001638 llvm::Type *LTy = ConvertTypeForMem(it->type);
Yaxun Liud7523282017-04-17 20:10:44 +00001639 ArgTypes[FirstIRArg] = LTy->getPointerTo(
1640 CGM.getDataLayout().getAllocaAddrSpace());
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00001641 break;
1642 }
1643
1644 case ABIArgInfo::Extend:
Chris Lattner2cdfda42010-07-29 06:44:09 +00001645 case ABIArgInfo::Direct: {
Oliver Stannard2bfdc5b2014-08-27 10:43:15 +00001646 // Fast-isel and the optimizer generally like scalar values better than
1647 // FCAs, so we flatten them if this is safe to do for this argument.
Alexey Samsonov153004f2014-09-29 22:08:00 +00001648 llvm::Type *argType = ArgInfo.getCoerceToType();
James Molloy6f244b62014-05-09 16:21:39 +00001649 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
Alexey Samsonov153004f2014-09-29 22:08:00 +00001650 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1651 assert(NumIRArgs == st->getNumElements());
John McCall85dd2c52011-05-15 02:19:42 +00001652 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
Alexey Samsonov153004f2014-09-29 22:08:00 +00001653 ArgTypes[FirstIRArg + i] = st->getElementType(i);
Chris Lattner3dd716c2010-06-28 23:44:11 +00001654 } else {
Alexey Samsonov153004f2014-09-29 22:08:00 +00001655 assert(NumIRArgs == 1);
1656 ArgTypes[FirstIRArg] = argType;
Chris Lattner3dd716c2010-06-28 23:44:11 +00001657 }
Daniel Dunbar2f219b02009-02-03 19:12:28 +00001658 break;
Chris Lattner2cdfda42010-07-29 06:44:09 +00001659 }
Mike Stump11289f42009-09-09 15:08:12 +00001660
John McCallf26e73d2016-03-11 04:30:43 +00001661 case ABIArgInfo::CoerceAndExpand: {
1662 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1663 for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1664 *ArgTypesIter++ = EltTy;
1665 }
1666 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1667 break;
1668 }
1669
Daniel Dunbard3674e62008-09-11 01:48:57 +00001670 case ABIArgInfo::Expand:
Alexey Samsonov153004f2014-09-29 22:08:00 +00001671 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1672 getExpandedTypes(it->type, ArgTypesIter);
1673 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
Daniel Dunbard3674e62008-09-11 01:48:57 +00001674 break;
1675 }
Daniel Dunbar7a95ca32008-09-10 04:01:49 +00001676 }
1677
Chris Lattner6fb0ccf2011-07-15 05:16:14 +00001678 bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1679 assert(Erased && "Not in set?");
Alexey Samsonov153004f2014-09-29 22:08:00 +00001680
1681 return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
Daniel Dunbar81cf67f2008-09-09 23:48:28 +00001682}
1683
Chris Lattner2192fe52011-07-18 04:24:23 +00001684llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
John McCall5d865c322010-08-31 07:33:07 +00001685 const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
Anders Carlsson64457732009-11-24 05:08:52 +00001686 const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00001687
Chris Lattner8806e322011-07-10 00:18:59 +00001688 if (!isFuncTypeConvertible(FPT))
1689 return llvm::StructType::get(getLLVMContext());
Fangrui Song6907ce22018-07-30 19:24:48 +00001690
Peter Collingbourned1c5b282019-03-22 23:05:10 +00001691 return GetFunctionType(GD);
Anders Carlsson64457732009-11-24 05:08:52 +00001692}
1693
Samuel Antao798f11c2015-11-23 22:04:44 +00001694static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1695 llvm::AttrBuilder &FuncAttrs,
1696 const FunctionProtoType *FPT) {
1697 if (!FPT)
1698 return;
1699
1700 if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
Richard Smitheaf11ad2018-05-03 03:58:32 +00001701 FPT->isNothrow())
Samuel Antao798f11c2015-11-23 22:04:44 +00001702 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1703}
1704
Justin Lebarb080b632017-01-25 21:29:48 +00001705void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
1706 bool AttrOnCallSite,
1707 llvm::AttrBuilder &FuncAttrs) {
1708 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1709 if (!HasOptnone) {
1710 if (CodeGenOpts.OptimizeSize)
1711 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1712 if (CodeGenOpts.OptimizeSize == 2)
1713 FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1714 }
1715
1716 if (CodeGenOpts.DisableRedZone)
1717 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
Kristina Brooks7f569b72018-10-18 14:07:02 +00001718 if (CodeGenOpts.IndirectTlsSegRefs)
1719 FuncAttrs.addAttribute("indirect-tls-seg-refs");
Justin Lebarb080b632017-01-25 21:29:48 +00001720 if (CodeGenOpts.NoImplicitFloat)
1721 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1722
1723 if (AttrOnCallSite) {
1724 // Attributes that should go on the call site only.
1725 if (!CodeGenOpts.SimplifyLibCalls ||
1726 CodeGenOpts.isNoBuiltinFunc(Name.data()))
1727 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1728 if (!CodeGenOpts.TrapFuncName.empty())
1729 FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1730 } else {
Yuanfang Chenff22ec32019-07-20 22:50:50 +00001731 StringRef FpKind;
1732 switch (CodeGenOpts.getFramePointer()) {
1733 case CodeGenOptions::FramePointerKind::None:
1734 FpKind = "none";
1735 break;
1736 case CodeGenOptions::FramePointerKind::NonLeaf:
1737 FpKind = "non-leaf";
1738 break;
1739 case CodeGenOptions::FramePointerKind::All:
1740 FpKind = "all";
1741 break;
Justin Lebarb080b632017-01-25 21:29:48 +00001742 }
Yuanfang Chenff22ec32019-07-20 22:50:50 +00001743 FuncAttrs.addAttribute("frame-pointer", FpKind);
Justin Lebarb080b632017-01-25 21:29:48 +00001744
1745 FuncAttrs.addAttribute("less-precise-fpmad",
1746 llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1747
Manoj Guptada08f6a2018-07-19 00:44:52 +00001748 if (CodeGenOpts.NullPointerIsValid)
1749 FuncAttrs.addAttribute("null-pointer-is-valid", "true");
Matt Arsenaulta4451d82019-11-01 17:57:29 -07001750
1751 // TODO: Omit attribute when the default is IEEE.
Matt Arsenaulta3c814d2019-11-06 17:10:52 -08001752 if (CodeGenOpts.FPDenormalMode.isValid())
Matt Arsenault7fe94352019-10-29 16:16:05 -07001753 FuncAttrs.addAttribute("denormal-fp-math",
Matt Arsenaulta3c814d2019-11-06 17:10:52 -08001754 CodeGenOpts.FPDenormalMode.str());
1755 if (CodeGenOpts.FP32DenormalMode.isValid()) {
Matt Arsenaulta4451d82019-11-01 17:57:29 -07001756 FuncAttrs.addAttribute(
1757 "denormal-fp-math-f32",
Matt Arsenaulta3c814d2019-11-06 17:10:52 -08001758 CodeGenOpts.FP32DenormalMode.str());
1759 }
Matt Arsenaulta4451d82019-11-01 17:57:29 -07001760
Justin Lebarb080b632017-01-25 21:29:48 +00001761 FuncAttrs.addAttribute("no-trapping-math",
1762 llvm::toStringRef(CodeGenOpts.NoTrappingMath));
1763
Sanjay Patelc81450e2018-04-30 18:19:03 +00001764 // Strict (compliant) code is the default, so only add this attribute to
1765 // indicate that we are trying to workaround a problem case.
1766 if (!CodeGenOpts.StrictFloatCastOverflow)
1767 FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
Sanjay Pateld1754762018-04-27 14:22:48 +00001768
Justin Lebarb080b632017-01-25 21:29:48 +00001769 // TODO: Are these all needed?
1770 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1771 FuncAttrs.addAttribute("no-infs-fp-math",
1772 llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
1773 FuncAttrs.addAttribute("no-nans-fp-math",
1774 llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
1775 FuncAttrs.addAttribute("unsafe-fp-math",
1776 llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
1777 FuncAttrs.addAttribute("use-soft-float",
1778 llvm::toStringRef(CodeGenOpts.SoftFloat));
1779 FuncAttrs.addAttribute("stack-protector-buffer-size",
1780 llvm::utostr(CodeGenOpts.SSPBufferSize));
1781 FuncAttrs.addAttribute("no-signed-zeros-fp-math",
1782 llvm::toStringRef(CodeGenOpts.NoSignedZeros));
1783 FuncAttrs.addAttribute(
1784 "correctly-rounded-divide-sqrt-fp-math",
1785 llvm::toStringRef(CodeGenOpts.CorrectlyRoundedDivSqrt));
1786
1787 // TODO: Reciprocal estimate codegen options should apply to instructions?
Craig Topper402b4312017-11-20 17:09:22 +00001788 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
Justin Lebarb080b632017-01-25 21:29:48 +00001789 if (!Recips.empty())
1790 FuncAttrs.addAttribute("reciprocal-estimates",
Erich Keane857ac592017-10-27 18:45:06 +00001791 llvm::join(Recips, ","));
Justin Lebarb080b632017-01-25 21:29:48 +00001792
Craig Topper9a724aa2017-12-11 21:09:19 +00001793 if (!CodeGenOpts.PreferVectorWidth.empty() &&
1794 CodeGenOpts.PreferVectorWidth != "none")
1795 FuncAttrs.addAttribute("prefer-vector-width",
1796 CodeGenOpts.PreferVectorWidth);
1797
Justin Lebarb080b632017-01-25 21:29:48 +00001798 if (CodeGenOpts.StackRealignment)
1799 FuncAttrs.addAttribute("stackrealign");
1800 if (CodeGenOpts.Backchain)
1801 FuncAttrs.addAttribute("backchain");
Chandler Carruth664aa862018-09-04 12:38:00 +00001802
1803 if (CodeGenOpts.SpeculativeLoadHardening)
1804 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
Justin Lebarb080b632017-01-25 21:29:48 +00001805 }
1806
Matt Arsenaulta1cf61b2017-10-06 19:34:40 +00001807 if (getLangOpts().assumeFunctionsAreConvergent()) {
1808 // Conservatively, mark all functions and calls in CUDA and OpenCL as
1809 // convergent (meaning, they may call an intrinsically convergent op, such
1810 // as __syncthreads() / barrier(), and so can't have certain optimizations
1811 // applied around them). LLVM will remove this attribute where it safely
1812 // can.
Justin Lebarb080b632017-01-25 21:29:48 +00001813 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
Matt Arsenaulta1cf61b2017-10-06 19:34:40 +00001814 }
Justin Lebarb080b632017-01-25 21:29:48 +00001815
Matt Arsenaulta1cf61b2017-10-06 19:34:40 +00001816 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
Justin Lebarb080b632017-01-25 21:29:48 +00001817 // Exceptions aren't supported in CUDA device code.
1818 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
Justin Lebarb080b632017-01-25 21:29:48 +00001819 }
Peter Collingbourne87f477b2019-01-04 19:27:04 +00001820
1821 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1822 StringRef Var, Value;
1823 std::tie(Var, Value) = Attr.split('=');
1824 FuncAttrs.addAttribute(Var, Value);
1825 }
Justin Lebarb080b632017-01-25 21:29:48 +00001826}
1827
1828void CodeGenModule::AddDefaultFnAttrs(llvm::Function &F) {
1829 llvm::AttrBuilder FuncAttrs;
Evandro Menezes85bd3972019-04-04 22:40:06 +00001830 ConstructDefaultFnAttrList(F.getName(), F.hasOptNone(),
Rui Ueyama49a3ad22019-07-16 04:46:31 +00001831 /* AttrOnCallSite = */ false, FuncAttrs);
Reid Kleckneree4930b2017-05-02 22:07:37 +00001832 F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
Justin Lebarb080b632017-01-25 21:29:48 +00001833}
1834
Francis Visoiu Mistrih4e799ad2020-01-27 10:40:14 -08001835static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1836 const LangOptions &LangOpts,
1837 const NoBuiltinAttr *NBA = nullptr) {
1838 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1839 SmallString<32> AttributeName;
1840 AttributeName += "no-builtin-";
1841 AttributeName += BuiltinName;
1842 FuncAttrs.addAttribute(AttributeName);
1843 };
1844
Francis Visoiu Mistrihb1a81892020-01-28 15:23:28 -08001845 // First, handle the language options passed through -fno-builtin.
Francis Visoiu Mistrih4e799ad2020-01-27 10:40:14 -08001846 if (LangOpts.NoBuiltin) {
1847 // -fno-builtin disables them all.
1848 FuncAttrs.addAttribute("no-builtins");
1849 return;
1850 }
1851
1852 // Then, add attributes for builtins specified through -fno-builtin-<name>.
1853 llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1854
1855 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1856 // the source.
1857 if (!NBA)
1858 return;
1859
1860 // If there is a wildcard in the builtin names specified through the
1861 // attribute, disable them all.
1862 if (llvm::is_contained(NBA->builtinNames(), "*")) {
1863 FuncAttrs.addAttribute("no-builtins");
1864 return;
1865 }
1866
1867 // And last, add the rest of the builtin names.
1868 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1869}
1870
Chad Rosier7dbc9cf2016-01-06 14:35:46 +00001871void CodeGenModule::ConstructAttributeList(
1872 StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
Reid Klecknercdd26792017-04-18 23:50:03 +00001873 llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
Bill Wendlinga514ebc2012-10-15 20:36:26 +00001874 llvm::AttrBuilder FuncAttrs;
1875 llvm::AttrBuilder RetAttrs;
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00001876
Daniel Dunbar0ef34792009-09-12 00:59:20 +00001877 CallingConv = FI.getEffectiveCallingConvention();
John McCallab26cfa2010-02-05 21:31:56 +00001878 if (FI.isNoReturn())
Bill Wendling207f0532012-12-20 19:27:06 +00001879 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
John McCallab26cfa2010-02-05 21:31:56 +00001880
Samuel Antao798f11c2015-11-23 22:04:44 +00001881 // If we have information about the function prototype, we can learn
Craig Topper13d759f2018-04-30 22:02:48 +00001882 // attributes from there.
Samuel Antao798f11c2015-11-23 22:04:44 +00001883 AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
1884 CalleeInfo.getCalleeFunctionProtoType());
1885
Erich Keanede6480a32018-11-13 15:48:08 +00001886 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
Samuel Antao798f11c2015-11-23 22:04:44 +00001887
Justin Lebarb080b632017-01-25 21:29:48 +00001888 bool HasOptnone = false;
Francis Visoiu Mistrih4e799ad2020-01-27 10:40:14 -08001889 // The NoBuiltinAttr attached to a TargetDecl (only allowed on FunctionDecls).
1890 const NoBuiltinAttr *NBA = nullptr;
Anton Korobeynikovc8478242009-04-04 00:49:24 +00001891 // FIXME: handle sseregparm someday...
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00001892 if (TargetDecl) {
Rafael Espindola2d21ab02011-10-12 19:51:18 +00001893 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
Bill Wendling207f0532012-12-20 19:27:06 +00001894 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
Argyrios Kyrtzidisb4b64ca2009-06-30 02:34:44 +00001895 if (TargetDecl->hasAttr<NoThrowAttr>())
Bill Wendling207f0532012-12-20 19:27:06 +00001896 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
Richard Smithdebc59d2013-01-30 05:45:05 +00001897 if (TargetDecl->hasAttr<NoReturnAttr>())
1898 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
Xinliang David Li4ec36062017-06-13 21:14:07 +00001899 if (TargetDecl->hasAttr<ColdAttr>())
1900 FuncAttrs.addAttribute(llvm::Attribute::Cold);
Aaron Ballman7c19ab12014-02-22 16:59:24 +00001901 if (TargetDecl->hasAttr<NoDuplicateAttr>())
1902 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
Yaxun Liu7d07ae72016-11-01 18:45:32 +00001903 if (TargetDecl->hasAttr<ConvergentAttr>())
1904 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
Richard Smithdebc59d2013-01-30 05:45:05 +00001905
Chandler Carruth45bbe012017-03-24 09:11:57 +00001906 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
Samuel Antao798f11c2015-11-23 22:04:44 +00001907 AddAttributesFromFunctionProtoType(
Chandler Carruth45bbe012017-03-24 09:11:57 +00001908 getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
Chandler Carruth45bbe012017-03-24 09:11:57 +00001909 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
Guillaume Chatelet98f31512019-09-25 11:31:28 +02001910 const bool IsVirtualCall = MD && MD->isVirtual();
1911 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
1912 // virtual function. These attributes are not inherited by overloads.
1913 if (!(AttrOnCallSite && IsVirtualCall)) {
1914 if (Fn->isNoReturn())
1915 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
Francis Visoiu Mistrih4e799ad2020-01-27 10:40:14 -08001916 NBA = Fn->getAttr<NoBuiltinAttr>();
Guillaume Chatelet98f31512019-09-25 11:31:28 +02001917 }
John McCallbe349de2010-07-08 06:48:12 +00001918 }
1919
David Majnemer1bf0f8e2015-07-20 22:51:52 +00001920 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
Eric Christopherbf005ec2011-08-15 22:38:22 +00001921 if (TargetDecl->hasAttr<ConstAttr>()) {
Bill Wendling207f0532012-12-20 19:27:06 +00001922 FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1923 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
Eric Christopherbf005ec2011-08-15 22:38:22 +00001924 } else if (TargetDecl->hasAttr<PureAttr>()) {
Bill Wendling207f0532012-12-20 19:27:06 +00001925 FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1926 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
David Majnemer1bf0f8e2015-07-20 22:51:52 +00001927 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
1928 FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
1929 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
Eric Christopherbf005ec2011-08-15 22:38:22 +00001930 }
David Majnemer631a90b2015-02-04 07:23:21 +00001931 if (TargetDecl->hasAttr<RestrictAttr>())
Bill Wendling207f0532012-12-20 19:27:06 +00001932 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
Manoj Guptada08f6a2018-07-19 00:44:52 +00001933 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
1934 !CodeGenOpts.NullPointerIsValid)
Hal Finkeld8442b12014-07-12 04:51:04 +00001935 RetAttrs.addAttribute(llvm::Attribute::NonNull);
Oren Ben Simhon318a6ea2017-04-27 12:01:00 +00001936 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
1937 FuncAttrs.addAttribute("no_caller_saved_registers");
Oren Ben Simhon220671a2018-03-17 13:31:35 +00001938 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
1939 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
Paul Robinson08556952014-12-11 20:14:04 +00001940
1941 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
George Burgess IVe3763372016-12-22 02:50:20 +00001942 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
1943 Optional<unsigned> NumElemsParam;
Joel E. Denny81508102018-03-13 14:51:22 +00001944 if (AllocSize->getNumElemsParam().isValid())
1945 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
1946 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
George Burgess IVe3763372016-12-22 02:50:20 +00001947 NumElemsParam);
1948 }
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00001949 }
1950
Francis Visoiu Mistrih4e799ad2020-01-27 10:40:14 -08001951 // Attach "no-builtins" attributes to:
1952 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
1953 // * definitions: "no-builtins" or "no-builtin-<name>" only.
1954 // The attributes can come from:
1955 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
1956 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
1957 addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
1958
Justin Lebarb080b632017-01-25 21:29:48 +00001959 ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
Paul Robinson08556952014-12-11 20:14:04 +00001960
Zola Bridges826ef592019-01-18 17:20:46 +00001961 // This must run after constructing the default function attribute list
1962 // to ensure that the speculative load hardening attribute is removed
1963 // in the case where the -mspeculative-load-hardening flag was passed.
1964 if (TargetDecl) {
1965 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
1966 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
1967 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
1968 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1969 }
1970
Peter Collingbourneb4728c12014-05-19 22:14:34 +00001971 if (CodeGenOpts.EnableSegmentedStacks &&
1972 !(TargetDecl && TargetDecl->hasAttr<NoSplitStackAttr>()))
Reid Klecknerfb873af2014-04-10 22:59:13 +00001973 FuncAttrs.addAttribute("split-stack");
Devang Patel6e467b12009-06-04 23:32:02 +00001974
Sriraman Tallam5c651482017-11-07 19:37:51 +00001975 // Add NonLazyBind attribute to function declarations when -fno-plt
1976 // is used.
1977 if (TargetDecl && CodeGenOpts.NoPLT) {
1978 if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1979 if (!Fn->isDefined() && !AttrOnCallSite) {
1980 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
1981 }
1982 }
1983 }
1984
Alexey Sotkin20f65922018-02-22 11:54:14 +00001985 if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>()) {
1986 if (getLangOpts().OpenCLVersion <= 120) {
1987 // OpenCL v1.2 Work groups are always uniform
1988 FuncAttrs.addAttribute("uniform-work-group-size", "true");
1989 } else {
1990 // OpenCL v2.0 Work groups may be whether uniform or not.
1991 // '-cl-uniform-work-group-size' compile option gets a hint
1992 // to the compiler that the global work-size be a multiple of
1993 // the work-group size specified to clEnqueueNDRangeKernel
1994 // (i.e. work groups are uniform).
1995 FuncAttrs.addAttribute("uniform-work-group-size",
1996 llvm::toStringRef(CodeGenOpts.UniformWGSize));
1997 }
1998 }
1999
Justin Lebarb080b632017-01-25 21:29:48 +00002000 if (!AttrOnCallSite) {
Akira Hatanaka627586b2018-03-02 01:53:15 +00002001 bool DisableTailCalls = false;
2002
2003 if (CodeGenOpts.DisableTailCalls)
2004 DisableTailCalls = true;
2005 else if (TargetDecl) {
2006 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2007 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2008 DisableTailCalls = true;
2009 else if (CodeGenOpts.NoEscapingBlockTailCalls) {
2010 if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2011 if (!BD->doesNotEscape())
2012 DisableTailCalls = true;
2013 }
2014 }
2015
Justin Lebarb080b632017-01-25 21:29:48 +00002016 FuncAttrs.addAttribute("disable-tail-calls",
2017 llvm::toStringRef(DisableTailCalls));
Erich Keanede6480a32018-11-13 15:48:08 +00002018 GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
Bill Wendling985d1c52013-02-15 21:30:01 +00002019 }
2020
Alexey Samsonov153004f2014-09-29 22:08:00 +00002021 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002022
Daniel Dunbar3668cb22009-02-02 23:43:58 +00002023 QualType RetTy = FI.getReturnType();
Daniel Dunbarb52d0772009-02-03 05:59:18 +00002024 const ABIArgInfo &RetAI = FI.getReturnInfo();
Daniel Dunbar7a95ca32008-09-10 04:01:49 +00002025 switch (RetAI.getKind()) {
Anton Korobeynikov18adbf52009-06-06 09:36:29 +00002026 case ABIArgInfo::Extend:
Alex Bradburye41a5e22018-01-12 20:08:16 +00002027 if (RetAI.isSignExt())
Jakob Stoklund Olesend7bf2932013-05-29 03:57:23 +00002028 RetAttrs.addAttribute(llvm::Attribute::SExt);
Alex Bradburye41a5e22018-01-12 20:08:16 +00002029 else
Jakob Stoklund Olesend7bf2932013-05-29 03:57:23 +00002030 RetAttrs.addAttribute(llvm::Attribute::ZExt);
Adrian Prantlf3b3ccd2017-12-19 22:06:11 +00002031 LLVM_FALLTHROUGH;
Daniel Dunbar67dace892009-02-03 06:17:37 +00002032 case ABIArgInfo::Direct:
Jakob Stoklund Olesena3661142013-06-05 03:00:09 +00002033 if (RetAI.getInReg())
2034 RetAttrs.addAttribute(llvm::Attribute::InReg);
2035 break;
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002036 case ABIArgInfo::Ignore:
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002037 break;
2038
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002039 case ABIArgInfo::InAlloca:
Rafael Espindola06b2b4a2012-07-31 02:44:24 +00002040 case ABIArgInfo::Indirect: {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002041 // inalloca and sret disable readnone and readonly
Bill Wendling207f0532012-12-20 19:27:06 +00002042 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2043 .removeAttribute(llvm::Attribute::ReadNone);
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002044 break;
Rafael Espindola06b2b4a2012-07-31 02:44:24 +00002045 }
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002046
John McCallf26e73d2016-03-11 04:30:43 +00002047 case ABIArgInfo::CoerceAndExpand:
2048 break;
2049
Daniel Dunbard3674e62008-09-11 01:48:57 +00002050 case ABIArgInfo::Expand:
David Blaikie83d382b2011-09-23 05:06:16 +00002051 llvm_unreachable("Invalid ABI kind for return argument");
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00002052 }
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002053
Hal Finkela2347ba2014-07-18 15:52:10 +00002054 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2055 QualType PTy = RefTy->getPointeeType();
David Majnemer9df56372015-09-10 21:52:00 +00002056 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
Richard Smith0130b6c2020-01-31 19:06:21 -08002057 RetAttrs.addDereferenceableAttr(
2058 getMinimumObjectSize(PTy).getQuantity());
Manoj Guptada08f6a2018-07-19 00:44:52 +00002059 else if (getContext().getTargetAddressSpace(PTy) == 0 &&
2060 !CodeGenOpts.NullPointerIsValid)
Hal Finkela2347ba2014-07-18 15:52:10 +00002061 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2062 }
Nick Lewycky9b46eb82014-05-28 09:56:42 +00002063
John McCall12f23522016-04-04 18:33:08 +00002064 bool hasUsedSRet = false;
Reid Klecknercdd26792017-04-18 23:50:03 +00002065 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
John McCall12f23522016-04-04 18:33:08 +00002066
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002067 // Attach attributes to sret.
2068 if (IRFunctionArgs.hasSRetArg()) {
2069 llvm::AttrBuilder SRETAttrs;
Mandeep Singh Grang85a0f8f2019-05-03 21:12:24 +00002070 SRETAttrs.addAttribute(llvm::Attribute::StructRet);
John McCall12f23522016-04-04 18:33:08 +00002071 hasUsedSRet = true;
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002072 if (RetAI.getInReg())
2073 SRETAttrs.addAttribute(llvm::Attribute::InReg);
Reid Klecknercdd26792017-04-18 23:50:03 +00002074 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2075 llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002076 }
2077
2078 // Attach attributes to inalloca argument.
2079 if (IRFunctionArgs.hasInallocaArg()) {
2080 llvm::AttrBuilder Attrs;
2081 Attrs.addAttribute(llvm::Attribute::InAlloca);
Reid Klecknercdd26792017-04-18 23:50:03 +00002082 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2083 llvm::AttributeSet::get(getLLVMContext(), Attrs);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002084 }
2085
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002086 unsigned ArgNo = 0;
2087 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2088 E = FI.arg_end();
2089 I != E; ++I, ++ArgNo) {
2090 QualType ParamType = I->type;
2091 const ABIArgInfo &AI = I->info;
Bill Wendlinga514ebc2012-10-15 20:36:26 +00002092 llvm::AttrBuilder Attrs;
Anton Korobeynikovc8478242009-04-04 00:49:24 +00002093
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002094 // Add attribute for padding argument, if necessary.
2095 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
Reid Klecknercdd26792017-04-18 23:50:03 +00002096 if (AI.getPaddingInReg()) {
2097 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2098 llvm::AttributeSet::get(
2099 getLLVMContext(),
2100 llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2101 }
Rafael Espindolafad28de2012-10-24 01:59:00 +00002102 }
2103
John McCall39ec71f2010-03-27 00:47:27 +00002104 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2105 // have the corresponding parameter variable. It doesn't make
Daniel Dunbarcb2b3d02011-02-10 18:10:07 +00002106 // sense to do it here because parameters are so messed up.
Daniel Dunbard3674e62008-09-11 01:48:57 +00002107 switch (AI.getKind()) {
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002108 case ABIArgInfo::Extend:
Alex Bradburye41a5e22018-01-12 20:08:16 +00002109 if (AI.isSignExt())
Bill Wendling207f0532012-12-20 19:27:06 +00002110 Attrs.addAttribute(llvm::Attribute::SExt);
Alex Bradburye41a5e22018-01-12 20:08:16 +00002111 else
2112 Attrs.addAttribute(llvm::Attribute::ZExt);
Adrian Prantlf3b3ccd2017-12-19 22:06:11 +00002113 LLVM_FALLTHROUGH;
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002114 case ABIArgInfo::Direct:
Peter Collingbournef7706832014-12-12 23:41:25 +00002115 if (ArgNo == 0 && FI.isChainCall())
2116 Attrs.addAttribute(llvm::Attribute::Nest);
2117 else if (AI.getInReg())
Bill Wendling207f0532012-12-20 19:27:06 +00002118 Attrs.addAttribute(llvm::Attribute::InReg);
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002119 break;
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002120
James Y Knight71608572015-08-21 18:19:06 +00002121 case ABIArgInfo::Indirect: {
Rafael Espindola703c47f2012-10-19 05:04:37 +00002122 if (AI.getInReg())
Bill Wendling207f0532012-12-20 19:27:06 +00002123 Attrs.addAttribute(llvm::Attribute::InReg);
Rafael Espindola703c47f2012-10-19 05:04:37 +00002124
Anders Carlsson20759ad2009-09-16 15:53:40 +00002125 if (AI.getIndirectByVal())
Tim Northoverc46827c2019-06-05 21:12:14 +00002126 Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
Anders Carlsson20759ad2009-09-16 15:53:40 +00002127
John McCall7f416cc2015-09-08 08:05:57 +00002128 CharUnits Align = AI.getIndirectAlign();
James Y Knight71608572015-08-21 18:19:06 +00002129
2130 // In a byval argument, it is important that the required
2131 // alignment of the type is honored, as LLVM might be creating a
2132 // *new* stack object, and needs to know what alignment to give
2133 // it. (Sometimes it can deduce a sensible alignment on its own,
2134 // but not if clang decides it must emit a packed struct, or the
2135 // user specifies increased alignment requirements.)
2136 //
2137 // This is different from indirect *not* byval, where the object
2138 // exists already, and the align attribute is purely
2139 // informative.
John McCall7f416cc2015-09-08 08:05:57 +00002140 assert(!Align.isZero());
James Y Knight71608572015-08-21 18:19:06 +00002141
John McCall7f416cc2015-09-08 08:05:57 +00002142 // For now, only add this when we have a byval argument.
2143 // TODO: be less lazy about updating test cases.
2144 if (AI.getIndirectByVal())
2145 Attrs.addAlignmentAttr(Align.getQuantity());
Bill Wendlinga7912f82012-10-10 07:36:56 +00002146
Daniel Dunbarc2304432009-03-18 19:51:01 +00002147 // byval disables readnone and readonly.
Bill Wendling207f0532012-12-20 19:27:06 +00002148 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2149 .removeAttribute(llvm::Attribute::ReadNone);
Daniel Dunbard3674e62008-09-11 01:48:57 +00002150 break;
James Y Knight71608572015-08-21 18:19:06 +00002151 }
Daniel Dunbar94a6f252009-01-26 21:26:08 +00002152 case ABIArgInfo::Ignore:
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002153 case ABIArgInfo::Expand:
John McCallf26e73d2016-03-11 04:30:43 +00002154 case ABIArgInfo::CoerceAndExpand:
2155 break;
Daniel Dunbar94a6f252009-01-26 21:26:08 +00002156
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002157 case ABIArgInfo::InAlloca:
2158 // inalloca disables readnone and readonly.
2159 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2160 .removeAttribute(llvm::Attribute::ReadNone);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002161 continue;
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00002162 }
Mike Stump11289f42009-09-09 15:08:12 +00002163
Hal Finkela2347ba2014-07-18 15:52:10 +00002164 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2165 QualType PTy = RefTy->getPointeeType();
David Majnemer9df56372015-09-10 21:52:00 +00002166 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
Richard Smith0130b6c2020-01-31 19:06:21 -08002167 Attrs.addDereferenceableAttr(
2168 getMinimumObjectSize(PTy).getQuantity());
Manoj Guptada08f6a2018-07-19 00:44:52 +00002169 else if (getContext().getTargetAddressSpace(PTy) == 0 &&
2170 !CodeGenOpts.NullPointerIsValid)
Hal Finkela2347ba2014-07-18 15:52:10 +00002171 Attrs.addAttribute(llvm::Attribute::NonNull);
2172 }
Nick Lewycky9b46eb82014-05-28 09:56:42 +00002173
John McCall12f23522016-04-04 18:33:08 +00002174 switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2175 case ParameterABI::Ordinary:
2176 break;
2177
2178 case ParameterABI::SwiftIndirectResult: {
2179 // Add 'sret' if we haven't already used it for something, but
2180 // only if the result is void.
2181 if (!hasUsedSRet && RetTy->isVoidType()) {
2182 Attrs.addAttribute(llvm::Attribute::StructRet);
2183 hasUsedSRet = true;
2184 }
2185
2186 // Add 'noalias' in either case.
2187 Attrs.addAttribute(llvm::Attribute::NoAlias);
2188
2189 // Add 'dereferenceable' and 'alignment'.
2190 auto PTy = ParamType->getPointeeType();
2191 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2192 auto info = getContext().getTypeInfoInChars(PTy);
2193 Attrs.addDereferenceableAttr(info.first.getQuantity());
Guillaume Chateletb65fa482019-10-15 12:56:24 +00002194 Attrs.addAttribute(llvm::Attribute::getWithAlignment(
2195 getLLVMContext(), info.second.getAsAlign()));
John McCall12f23522016-04-04 18:33:08 +00002196 }
2197 break;
2198 }
2199
2200 case ParameterABI::SwiftErrorResult:
2201 Attrs.addAttribute(llvm::Attribute::SwiftError);
2202 break;
2203
2204 case ParameterABI::SwiftContext:
2205 Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2206 break;
2207 }
2208
Akira Hatanaka98a49332017-09-22 00:41:05 +00002209 if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2210 Attrs.addAttribute(llvm::Attribute::NoCapture);
2211
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002212 if (Attrs.hasAttributes()) {
2213 unsigned FirstIRArg, NumIRArgs;
2214 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2215 for (unsigned i = 0; i < NumIRArgs; i++)
Reid Klecknercdd26792017-04-18 23:50:03 +00002216 ArgAttrs[FirstIRArg + i] =
2217 llvm::AttributeSet::get(getLLVMContext(), Attrs);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002218 }
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00002219 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002220 assert(ArgNo == FI.arg_size());
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002221
Reid Klecknercdd26792017-04-18 23:50:03 +00002222 AttrList = llvm::AttributeList::get(
2223 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2224 llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
Daniel Dunbar76c8eb72008-09-10 00:32:18 +00002225}
2226
John McCalla738c252011-03-09 04:27:21 +00002227/// An argument came in as a promoted argument; demote it back to its
2228/// declared type.
2229static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2230 const VarDecl *var,
2231 llvm::Value *value) {
Chris Lattner2192fe52011-07-18 04:24:23 +00002232 llvm::Type *varType = CGF.ConvertType(var->getType());
John McCalla738c252011-03-09 04:27:21 +00002233
2234 // This can happen with promotions that actually don't change the
2235 // underlying type, like the enum promotions.
2236 if (value->getType() == varType) return value;
2237
2238 assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2239 && "unexpected promotion type");
2240
2241 if (isa<llvm::IntegerType>(varType))
2242 return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2243
2244 return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2245}
2246
Chandler Carruth45bbe012017-03-24 09:11:57 +00002247/// Returns the attribute (either parameter attribute, or function
2248/// attribute), which declares argument ArgNo to be non-null.
2249static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2250 QualType ArgType, unsigned ArgNo) {
2251 // FIXME: __attribute__((nonnull)) can also be applied to:
2252 // - references to pointers, where the pointee is known to be
2253 // nonnull (apparently a Clang extension)
2254 // - transparent unions containing pointers
2255 // In the former case, LLVM IR cannot represent the constraint. In
2256 // the latter case, we have no guarantee that the transparent union
2257 // is in fact passed as a pointer.
2258 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
2259 return nullptr;
2260 // First, check attribute on parameter itself.
2261 if (PVD) {
2262 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2263 return ParmNNAttr;
2264 }
2265 // Check function attributes.
2266 if (!FD)
2267 return nullptr;
2268 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2269 if (NNAttr->isNonNull(ArgNo))
2270 return NNAttr;
2271 }
2272 return nullptr;
2273}
2274
John McCall12f23522016-04-04 18:33:08 +00002275namespace {
2276 struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2277 Address Temp;
2278 Address Arg;
2279 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2280 void Emit(CodeGenFunction &CGF, Flags flags) override {
2281 llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2282 CGF.Builder.CreateStore(errorValue, Arg);
2283 }
2284 };
2285}
2286
Daniel Dunbard931a872009-02-02 22:03:45 +00002287void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2288 llvm::Function *Fn,
Daniel Dunbar613855c2008-09-09 23:27:19 +00002289 const FunctionArgList &Args) {
Hans Wennborgd71907d2014-09-04 22:16:33 +00002290 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
2291 // Naked functions don't have prologues.
2292 return;
2293
John McCallcaa19452009-07-28 01:00:58 +00002294 // If this is an implicit-return-zero function, go ahead and
2295 // initialize the return value. TODO: it might be nice to have
2296 // a more general mechanism for this that didn't require synthesized
2297 // return statements.
John McCalldec348f72013-05-03 07:33:41 +00002298 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
John McCallcaa19452009-07-28 01:00:58 +00002299 if (FD->hasImplicitReturnZero()) {
Alp Toker314cc812014-01-25 16:55:45 +00002300 QualType RetTy = FD->getReturnType().getUnqualifiedType();
Chris Lattner2192fe52011-07-18 04:24:23 +00002301 llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
Owen Anderson0b75f232009-07-31 20:28:54 +00002302 llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
John McCallcaa19452009-07-28 01:00:58 +00002303 Builder.CreateStore(Zero, ReturnValue);
2304 }
2305 }
2306
Mike Stump18bb9282009-05-16 07:57:57 +00002307 // FIXME: We no longer need the types from FunctionArgList; lift up and
2308 // simplify.
Daniel Dunbar5a0acdc92009-02-03 06:02:10 +00002309
Alexey Samsonov153004f2014-09-29 22:08:00 +00002310 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002311 // Flattened function arguments.
John McCall12f23522016-04-04 18:33:08 +00002312 SmallVector<llvm::Value *, 16> FnArgs;
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002313 FnArgs.reserve(IRFunctionArgs.totalIRArgs());
2314 for (auto &Arg : Fn->args()) {
2315 FnArgs.push_back(&Arg);
2316 }
2317 assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
Mike Stump11289f42009-09-09 15:08:12 +00002318
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002319 // If we're using inalloca, all the memory arguments are GEPs off of the last
2320 // parameter, which is a pointer to the complete memory area.
John McCall7f416cc2015-09-08 08:05:57 +00002321 Address ArgStruct = Address::invalid();
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002322 if (IRFunctionArgs.hasInallocaArg()) {
John McCall7f416cc2015-09-08 08:05:57 +00002323 ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()],
2324 FI.getArgStructAlignment());
2325
2326 assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002327 }
2328
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002329 // Name the struct return parameter.
2330 if (IRFunctionArgs.hasSRetArg()) {
John McCall12f23522016-04-04 18:33:08 +00002331 auto AI = cast<llvm::Argument>(FnArgs[IRFunctionArgs.getSRetArgNo()]);
Daniel Dunbar613855c2008-09-09 23:27:19 +00002332 AI->setName("agg.result");
Reid Klecknercdd26792017-04-18 23:50:03 +00002333 AI->addAttr(llvm::Attribute::NoAlias);
Daniel Dunbar613855c2008-09-09 23:27:19 +00002334 }
Mike Stump11289f42009-09-09 15:08:12 +00002335
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002336 // Track if we received the parameter as a pointer (indirect, byval, or
2337 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
2338 // into a local alloca for us.
John McCall7f416cc2015-09-08 08:05:57 +00002339 SmallVector<ParamValue, 16> ArgVals;
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002340 ArgVals.reserve(Args.size());
2341
Reid Kleckner739756c2013-12-04 19:23:12 +00002342 // Create a pointer value for every parameter declaration. This usually
2343 // entails copying one or more LLVM IR arguments into an alloca. Don't push
2344 // any cleanups or do anything that might unwind. We do that separately, so
2345 // we can push the cleanups in the correct order for the ABI.
Daniel Dunbara45bdbb2009-02-04 21:17:21 +00002346 assert(FI.arg_size() == Args.size() &&
2347 "Mismatch between function signature & arguments.");
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002348 unsigned ArgNo = 0;
Daniel Dunbarb52d0772009-02-03 05:59:18 +00002349 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002350 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
Devang Patel68a15252011-03-03 20:13:15 +00002351 i != e; ++i, ++info_it, ++ArgNo) {
John McCalla738c252011-03-09 04:27:21 +00002352 const VarDecl *Arg = *i;
Daniel Dunbarb52d0772009-02-03 05:59:18 +00002353 const ABIArgInfo &ArgI = info_it->info;
Daniel Dunbard3674e62008-09-11 01:48:57 +00002354
John McCalla738c252011-03-09 04:27:21 +00002355 bool isPromoted =
2356 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
Volodymyr Sapsai17ebdb22018-01-22 22:29:24 +00002357 // We are converting from ABIArgInfo type to VarDecl type directly, unless
2358 // the parameter is promoted. In this case we convert to
2359 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2360 QualType Ty = isPromoted ? info_it->type : Arg->getType();
2361 assert(hasScalarEvaluationKind(Ty) ==
2362 hasScalarEvaluationKind(Arg->getType()));
John McCalla738c252011-03-09 04:27:21 +00002363
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002364 unsigned FirstIRArg, NumIRArgs;
2365 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
Rafael Espindolafad28de2012-10-24 01:59:00 +00002366
Daniel Dunbard3674e62008-09-11 01:48:57 +00002367 switch (ArgI.getKind()) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002368 case ABIArgInfo::InAlloca: {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002369 assert(NumIRArgs == 0);
John McCall7f416cc2015-09-08 08:05:57 +00002370 auto FieldIndex = ArgI.getInAllocaFieldIndex();
James Y Knight751fe282019-02-09 22:22:28 +00002371 Address V =
2372 Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
John McCall7f416cc2015-09-08 08:05:57 +00002373 ArgVals.push_back(ParamValue::forIndirect(V));
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002374 break;
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002375 }
2376
Daniel Dunbar747865a2009-02-05 09:16:39 +00002377 case ABIArgInfo::Indirect: {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002378 assert(NumIRArgs == 1);
John McCall7f416cc2015-09-08 08:05:57 +00002379 Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign());
Daniel Dunbar7b7c2932010-09-16 20:42:02 +00002380
John McCall47fb9502013-03-07 21:37:08 +00002381 if (!hasScalarEvaluationKind(Ty)) {
Daniel Dunbar7b7c2932010-09-16 20:42:02 +00002382 // Aggregates and complex variables are accessed by reference. All we
John McCall7f416cc2015-09-08 08:05:57 +00002383 // need to do is realign the value, if requested.
2384 Address V = ParamAddr;
Daniel Dunbar7b7c2932010-09-16 20:42:02 +00002385 if (ArgI.getIndirectRealign()) {
John McCall7f416cc2015-09-08 08:05:57 +00002386 Address AlignedTemp = CreateMemTemp(Ty, "coerce");
Daniel Dunbar7b7c2932010-09-16 20:42:02 +00002387
2388 // Copy from the incoming argument pointer to the temporary with the
2389 // appropriate alignment.
2390 //
2391 // FIXME: We should have a common utility for generating an aggregate
2392 // copy.
Ken Dyck705ba072011-01-19 01:58:38 +00002393 CharUnits Size = getContext().getTypeSizeInChars(Ty);
John McCall7f416cc2015-09-08 08:05:57 +00002394 auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity());
2395 Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy);
2396 Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy);
2397 Builder.CreateMemCpy(Dst, Src, SizeVal, false);
Daniel Dunbar7b7c2932010-09-16 20:42:02 +00002398 V = AlignedTemp;
2399 }
John McCall7f416cc2015-09-08 08:05:57 +00002400 ArgVals.push_back(ParamValue::forIndirect(V));
Daniel Dunbar747865a2009-02-05 09:16:39 +00002401 } else {
2402 // Load scalar value from indirect argument.
John McCall7f416cc2015-09-08 08:05:57 +00002403 llvm::Value *V =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002404 EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
John McCalla738c252011-03-09 04:27:21 +00002405
2406 if (isPromoted)
2407 V = emitArgumentDemotion(*this, Arg, V);
John McCall7f416cc2015-09-08 08:05:57 +00002408 ArgVals.push_back(ParamValue::forDirect(V));
Daniel Dunbar747865a2009-02-05 09:16:39 +00002409 }
Daniel Dunbar747865a2009-02-05 09:16:39 +00002410 break;
2411 }
Anton Korobeynikov18adbf52009-06-06 09:36:29 +00002412
2413 case ABIArgInfo::Extend:
Daniel Dunbar67dace892009-02-03 06:17:37 +00002414 case ABIArgInfo::Direct: {
Akira Hatanaka18334dd2012-01-09 19:08:06 +00002415
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002416 // If we have the trivial case, handle it with no muss and fuss.
2417 if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
Volodymyr Sapsai22b00ec2017-12-21 20:52:59 +00002418 ArgI.getCoerceToType() == ConvertType(Ty) &&
Chris Lattner8a2f3c72010-07-30 04:02:24 +00002419 ArgI.getDirectOffset() == 0) {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002420 assert(NumIRArgs == 1);
John McCall12f23522016-04-04 18:33:08 +00002421 llvm::Value *V = FnArgs[FirstIRArg];
2422 auto AI = cast<llvm::Argument>(V);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002423
Hal Finkel48d53e22014-07-19 01:41:07 +00002424 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00002425 if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
Manoj Guptada08f6a2018-07-19 00:44:52 +00002426 PVD->getFunctionScopeIndex()) &&
2427 !CGM.getCodeGenOpts().NullPointerIsValid)
Reid Klecknercdd26792017-04-18 23:50:03 +00002428 AI->addAttr(llvm::Attribute::NonNull);
Hal Finkel82504f02014-07-11 17:35:21 +00002429
Hal Finkel48d53e22014-07-19 01:41:07 +00002430 QualType OTy = PVD->getOriginalType();
2431 if (const auto *ArrTy =
2432 getContext().getAsConstantArrayType(OTy)) {
2433 // A C99 array parameter declaration with the static keyword also
2434 // indicates dereferenceability, and if the size is constant we can
2435 // use the dereferenceable attribute (which requires the size in
2436 // bytes).
Hal Finkel16e394a2014-07-19 02:13:40 +00002437 if (ArrTy->getSizeModifier() == ArrayType::Static) {
Hal Finkel48d53e22014-07-19 01:41:07 +00002438 QualType ETy = ArrTy->getElementType();
2439 uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2440 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2441 ArrSize) {
2442 llvm::AttrBuilder Attrs;
2443 Attrs.addDereferenceableAttr(
2444 getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
Reid Kleckner9d16fa02017-04-19 17:28:52 +00002445 AI->addAttrs(Attrs);
Manoj Guptada08f6a2018-07-19 00:44:52 +00002446 } else if (getContext().getTargetAddressSpace(ETy) == 0 &&
2447 !CGM.getCodeGenOpts().NullPointerIsValid) {
Reid Klecknercdd26792017-04-18 23:50:03 +00002448 AI->addAttr(llvm::Attribute::NonNull);
Hal Finkel48d53e22014-07-19 01:41:07 +00002449 }
2450 }
2451 } else if (const auto *ArrTy =
2452 getContext().getAsVariableArrayType(OTy)) {
2453 // For C99 VLAs with the static keyword, we don't know the size so
2454 // we can't use the dereferenceable attribute, but in addrspace(0)
2455 // we know that it must be nonnull.
2456 if (ArrTy->getSizeModifier() == VariableArrayType::Static &&
Manoj Guptada08f6a2018-07-19 00:44:52 +00002457 !getContext().getTargetAddressSpace(ArrTy->getElementType()) &&
2458 !CGM.getCodeGenOpts().NullPointerIsValid)
Reid Klecknercdd26792017-04-18 23:50:03 +00002459 AI->addAttr(llvm::Attribute::NonNull);
Hal Finkel48d53e22014-07-19 01:41:07 +00002460 }
Hal Finkel1b0d24e2014-10-02 21:21:25 +00002461
2462 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2463 if (!AVAttr)
2464 if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2465 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
Roman Lebedevbd1c0872019-01-15 09:44:25 +00002466 if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {
2467 // If alignment-assumption sanitizer is enabled, we do *not* add
2468 // alignment attribute here, but emit normal alignment assumption,
2469 // so the UBSAN check could function.
Hal Finkel1b0d24e2014-10-02 21:21:25 +00002470 llvm::Value *AlignmentValue =
2471 EmitScalarExpr(AVAttr->getAlignment());
2472 llvm::ConstantInt *AlignmentCI =
2473 cast<llvm::ConstantInt>(AlignmentValue);
Roman Lebedev6b2f8202020-01-21 21:18:29 +03002474 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(llvm::MaybeAlign(
2475 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))));
Hal Finkel1b0d24e2014-10-02 21:21:25 +00002476 }
Hal Finkel48d53e22014-07-19 01:41:07 +00002477 }
2478
Bill Wendling507c3512012-10-16 05:23:44 +00002479 if (Arg->getType().isRestrictQualified())
Reid Klecknercdd26792017-04-18 23:50:03 +00002480 AI->addAttr(llvm::Attribute::NoAlias);
John McCall39ec71f2010-03-27 00:47:27 +00002481
John McCall12f23522016-04-04 18:33:08 +00002482 // LLVM expects swifterror parameters to be used in very restricted
2483 // ways. Copy the value into a less-restricted temporary.
2484 if (FI.getExtParameterInfo(ArgNo).getABI()
2485 == ParameterABI::SwiftErrorResult) {
2486 QualType pointeeTy = Ty->getPointeeType();
2487 assert(pointeeTy->isPointerType());
2488 Address temp =
2489 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2490 Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2491 llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2492 Builder.CreateStore(incomingErrorValue, temp);
2493 V = temp.getPointer();
2494
2495 // Push a cleanup to copy the value back at the end of the function.
2496 // The convention does not guarantee that the value will be written
2497 // back if the function exits with an unwind exception.
2498 EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2499 }
2500
Chris Lattner7369c142011-07-20 06:29:00 +00002501 // Ensure the argument is the correct type.
2502 if (V->getType() != ArgI.getCoerceToType())
2503 V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2504
John McCalla738c252011-03-09 04:27:21 +00002505 if (isPromoted)
2506 V = emitArgumentDemotion(*this, Arg, V);
Rafael Espindola8778c282012-11-29 16:09:03 +00002507
2508 // Because of merging of function types from multiple decls it is
2509 // possible for the type of an argument to not match the corresponding
2510 // type in the function type. Since we are codegening the callee
2511 // in here, add a cast to the argument type.
2512 llvm::Type *LTy = ConvertType(Arg->getType());
2513 if (V->getType() != LTy)
2514 V = Builder.CreateBitCast(V, LTy);
2515
John McCall7f416cc2015-09-08 08:05:57 +00002516 ArgVals.push_back(ParamValue::forDirect(V));
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002517 break;
Daniel Dunbard5f1f552009-02-10 00:06:49 +00002518 }
Mike Stump11289f42009-09-09 15:08:12 +00002519
Volodymyr Sapsai22b00ec2017-12-21 20:52:59 +00002520 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2521 Arg->getName());
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002522
John McCall7f416cc2015-09-08 08:05:57 +00002523 // Pointer to store into.
2524 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002525
Oliver Stannard2bfdc5b2014-08-27 10:43:15 +00002526 // Fast-isel and the optimizer generally like scalar values better than
2527 // FCAs, so we flatten them if this is safe to do for this argument.
Evgeniy Stepanov3fae4ae2012-02-10 09:30:15 +00002528 llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
Oliver Stannard2bfdc5b2014-08-27 10:43:15 +00002529 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
2530 STy->getNumElements() > 1) {
Micah Villmowdd31ca12012-10-08 16:25:52 +00002531 uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
John McCall7f416cc2015-09-08 08:05:57 +00002532 llvm::Type *DstTy = Ptr.getElementType();
Micah Villmowdd31ca12012-10-08 16:25:52 +00002533 uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002534
John McCall7f416cc2015-09-08 08:05:57 +00002535 Address AddrToStoreInto = Address::invalid();
Evgeniy Stepanov3fae4ae2012-02-10 09:30:15 +00002536 if (SrcSize <= DstSize) {
Yaxun Liue9e5c4f2017-06-29 18:47:45 +00002537 AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
Evgeniy Stepanov3fae4ae2012-02-10 09:30:15 +00002538 } else {
John McCall7f416cc2015-09-08 08:05:57 +00002539 AddrToStoreInto =
2540 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
Chris Lattner15ec3612010-06-29 00:06:42 +00002541 }
John McCall7f416cc2015-09-08 08:05:57 +00002542
2543 assert(STy->getNumElements() == NumIRArgs);
2544 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
2545 auto AI = FnArgs[FirstIRArg + i];
2546 AI->setName(Arg->getName() + ".coerce" + Twine(i));
James Y Knight751fe282019-02-09 22:22:28 +00002547 Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
John McCall7f416cc2015-09-08 08:05:57 +00002548 Builder.CreateStore(AI, EltPtr);
2549 }
2550
2551 if (SrcSize > DstSize) {
2552 Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2553 }
2554
Chris Lattner15ec3612010-06-29 00:06:42 +00002555 } else {
2556 // Simple case, just do a coerced store of the argument into the alloca.
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002557 assert(NumIRArgs == 1);
2558 auto AI = FnArgs[FirstIRArg];
Chris Lattner9e748e92010-06-29 00:14:52 +00002559 AI->setName(Arg->getName() + ".coerce");
Rui Ueyama49a3ad22019-07-16 04:46:31 +00002560 CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
Chris Lattner15ec3612010-06-29 00:06:42 +00002561 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002562
Daniel Dunbar2f219b02009-02-03 19:12:28 +00002563 // Match to what EmitParmDecl is expecting for this type.
Volodymyr Sapsai22b00ec2017-12-21 20:52:59 +00002564 if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
John McCall7f416cc2015-09-08 08:05:57 +00002565 llvm::Value *V =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002566 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
John McCalla738c252011-03-09 04:27:21 +00002567 if (isPromoted)
2568 V = emitArgumentDemotion(*this, Arg, V);
John McCall7f416cc2015-09-08 08:05:57 +00002569 ArgVals.push_back(ParamValue::forDirect(V));
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002570 } else {
John McCall7f416cc2015-09-08 08:05:57 +00002571 ArgVals.push_back(ParamValue::forIndirect(Alloca));
Daniel Dunbar6e3b7df2009-02-04 07:22:24 +00002572 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002573 break;
Daniel Dunbar2f219b02009-02-03 19:12:28 +00002574 }
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002575
John McCallf26e73d2016-03-11 04:30:43 +00002576 case ABIArgInfo::CoerceAndExpand: {
2577 // Reconstruct into a temporary.
2578 Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2579 ArgVals.push_back(ParamValue::forIndirect(alloca));
2580
2581 auto coercionType = ArgI.getCoerceAndExpandType();
2582 alloca = Builder.CreateElementBitCast(alloca, coercionType);
John McCallf26e73d2016-03-11 04:30:43 +00002583
2584 unsigned argIndex = FirstIRArg;
2585 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
2586 llvm::Type *eltType = coercionType->getElementType(i);
2587 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2588 continue;
2589
James Y Knight751fe282019-02-09 22:22:28 +00002590 auto eltAddr = Builder.CreateStructGEP(alloca, i);
John McCallf26e73d2016-03-11 04:30:43 +00002591 auto elt = FnArgs[argIndex++];
2592 Builder.CreateStore(elt, eltAddr);
2593 }
2594 assert(argIndex == FirstIRArg + NumIRArgs);
2595 break;
2596 }
2597
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002598 case ABIArgInfo::Expand: {
2599 // If this structure was expanded into multiple arguments then
2600 // we need to create a temporary and reconstruct it from the
2601 // arguments.
John McCall7f416cc2015-09-08 08:05:57 +00002602 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2603 LValue LV = MakeAddrLValue(Alloca, Ty);
2604 ArgVals.push_back(ParamValue::forIndirect(Alloca));
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002605
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002606 auto FnArgIter = FnArgs.begin() + FirstIRArg;
2607 ExpandTypeFromArgs(Ty, LV, FnArgIter);
2608 assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
2609 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
2610 auto AI = FnArgs[FirstIRArg + i];
2611 AI->setName(Arg->getName() + "." + Twine(i));
2612 }
2613 break;
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002614 }
2615
2616 case ABIArgInfo::Ignore:
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002617 assert(NumIRArgs == 0);
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002618 // Initialize the local variable appropriately.
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002619 if (!hasScalarEvaluationKind(Ty)) {
John McCall7f416cc2015-09-08 08:05:57 +00002620 ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002621 } else {
2622 llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
John McCall7f416cc2015-09-08 08:05:57 +00002623 ArgVals.push_back(ParamValue::forDirect(U));
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002624 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00002625 break;
Daniel Dunbard3674e62008-09-11 01:48:57 +00002626 }
Daniel Dunbar613855c2008-09-09 23:27:19 +00002627 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002628
Reid Kleckner739756c2013-12-04 19:23:12 +00002629 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2630 for (int I = Args.size() - 1; I >= 0; --I)
John McCall7f416cc2015-09-08 08:05:57 +00002631 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
Reid Kleckner739756c2013-12-04 19:23:12 +00002632 } else {
2633 for (unsigned I = 0, E = Args.size(); I != E; ++I)
John McCall7f416cc2015-09-08 08:05:57 +00002634 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
Reid Kleckner739756c2013-12-04 19:23:12 +00002635 }
Daniel Dunbar613855c2008-09-09 23:27:19 +00002636}
2637
John McCallffa2c1a2012-01-29 07:46:59 +00002638static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2639 while (insn->use_empty()) {
2640 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2641 if (!bitcast) return;
2642
2643 // This is "safe" because we would have used a ConstantExpr otherwise.
2644 insn = cast<llvm::Instruction>(bitcast->getOperand(0));
2645 bitcast->eraseFromParent();
2646 }
2647}
2648
John McCall31168b02011-06-15 23:02:42 +00002649/// Try to emit a fused autorelease of a return result.
2650static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
2651 llvm::Value *result) {
2652 // We must be immediately followed the cast.
2653 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
Craig Topper8a13c412014-05-21 05:09:00 +00002654 if (BB->empty()) return nullptr;
2655 if (&BB->back() != result) return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002656
Chris Lattner2192fe52011-07-18 04:24:23 +00002657 llvm::Type *resultType = result->getType();
John McCall31168b02011-06-15 23:02:42 +00002658
2659 // result is in a BasicBlock and is therefore an Instruction.
2660 llvm::Instruction *generator = cast<llvm::Instruction>(result);
2661
Justin Bogner882f8612016-08-18 21:46:54 +00002662 SmallVector<llvm::Instruction *, 4> InstsToKill;
John McCall31168b02011-06-15 23:02:42 +00002663
2664 // Look for:
2665 // %generator = bitcast %type1* %generator2 to %type2*
2666 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2667 // We would have emitted this as a constant if the operand weren't
2668 // an Instruction.
2669 generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2670
2671 // Require the generator to be immediately followed by the cast.
2672 if (generator->getNextNode() != bitcast)
Craig Topper8a13c412014-05-21 05:09:00 +00002673 return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002674
Justin Bogner882f8612016-08-18 21:46:54 +00002675 InstsToKill.push_back(bitcast);
John McCall31168b02011-06-15 23:02:42 +00002676 }
2677
2678 // Look for:
2679 // %generator = call i8* @objc_retain(i8* %originalResult)
2680 // or
2681 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2682 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
Craig Topper8a13c412014-05-21 05:09:00 +00002683 if (!call) return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002684
2685 bool doRetainAutorelease;
2686
John McCallb04ecb72015-10-21 18:06:43 +00002687 if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) {
John McCall31168b02011-06-15 23:02:42 +00002688 doRetainAutorelease = true;
John McCallb04ecb72015-10-21 18:06:43 +00002689 } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints()
John McCall31168b02011-06-15 23:02:42 +00002690 .objc_retainAutoreleasedReturnValue) {
2691 doRetainAutorelease = false;
2692
John McCallcfa4e9b2012-09-07 23:30:50 +00002693 // If we emitted an assembly marker for this call (and the
2694 // ARCEntrypoints field should have been set if so), go looking
2695 // for that call. If we can't find it, we can't do this
2696 // optimization. But it should always be the immediately previous
2697 // instruction, unless we needed bitcasts around the call.
John McCallb04ecb72015-10-21 18:06:43 +00002698 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
John McCallcfa4e9b2012-09-07 23:30:50 +00002699 llvm::Instruction *prev = call->getPrevNode();
2700 assert(prev);
2701 if (isa<llvm::BitCastInst>(prev)) {
2702 prev = prev->getPrevNode();
2703 assert(prev);
2704 }
2705 assert(isa<llvm::CallInst>(prev));
2706 assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
John McCallb04ecb72015-10-21 18:06:43 +00002707 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
Justin Bogner882f8612016-08-18 21:46:54 +00002708 InstsToKill.push_back(prev);
John McCallcfa4e9b2012-09-07 23:30:50 +00002709 }
John McCall31168b02011-06-15 23:02:42 +00002710 } else {
Craig Topper8a13c412014-05-21 05:09:00 +00002711 return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002712 }
2713
2714 result = call->getArgOperand(0);
Justin Bogner882f8612016-08-18 21:46:54 +00002715 InstsToKill.push_back(call);
John McCall31168b02011-06-15 23:02:42 +00002716
2717 // Keep killing bitcasts, for sanity. Note that we no longer care
2718 // about precise ordering as long as there's exactly one use.
2719 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2720 if (!bitcast->hasOneUse()) break;
Justin Bogner882f8612016-08-18 21:46:54 +00002721 InstsToKill.push_back(bitcast);
John McCall31168b02011-06-15 23:02:42 +00002722 result = bitcast->getOperand(0);
2723 }
2724
2725 // Delete all the unnecessary instructions, from latest to earliest.
Justin Bogner882f8612016-08-18 21:46:54 +00002726 for (auto *I : InstsToKill)
Saleem Abdulrasoolbe25c482016-08-18 21:40:06 +00002727 I->eraseFromParent();
John McCall31168b02011-06-15 23:02:42 +00002728
2729 // Do the fused retain/autorelease if we were asked to.
2730 if (doRetainAutorelease)
2731 result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2732
2733 // Cast back to the result type.
2734 return CGF.Builder.CreateBitCast(result, resultType);
2735}
2736
John McCallffa2c1a2012-01-29 07:46:59 +00002737/// If this is a +1 of the value of an immutable 'self', remove it.
2738static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2739 llvm::Value *result) {
2740 // This is only applicable to a method with an immutable 'self'.
John McCallff755cd2012-07-31 00:33:55 +00002741 const ObjCMethodDecl *method =
2742 dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
Craig Topper8a13c412014-05-21 05:09:00 +00002743 if (!method) return nullptr;
John McCallffa2c1a2012-01-29 07:46:59 +00002744 const VarDecl *self = method->getSelfDecl();
Craig Topper8a13c412014-05-21 05:09:00 +00002745 if (!self->getType().isConstQualified()) return nullptr;
John McCallffa2c1a2012-01-29 07:46:59 +00002746
2747 // Look for a retain call.
2748 llvm::CallInst *retainCall =
2749 dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2750 if (!retainCall ||
John McCallb04ecb72015-10-21 18:06:43 +00002751 retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain)
Craig Topper8a13c412014-05-21 05:09:00 +00002752 return nullptr;
John McCallffa2c1a2012-01-29 07:46:59 +00002753
2754 // Look for an ordinary load of 'self'.
2755 llvm::Value *retainedValue = retainCall->getArgOperand(0);
2756 llvm::LoadInst *load =
2757 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
Fangrui Song6907ce22018-07-30 19:24:48 +00002758 if (!load || load->isAtomic() || load->isVolatile() ||
John McCall7f416cc2015-09-08 08:05:57 +00002759 load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
Craig Topper8a13c412014-05-21 05:09:00 +00002760 return nullptr;
John McCallffa2c1a2012-01-29 07:46:59 +00002761
2762 // Okay! Burn it all down. This relies for correctness on the
2763 // assumption that the retain is emitted as part of the return and
2764 // that thereafter everything is used "linearly".
2765 llvm::Type *resultType = result->getType();
2766 eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2767 assert(retainCall->use_empty());
2768 retainCall->eraseFromParent();
2769 eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2770
2771 return CGF.Builder.CreateBitCast(load, resultType);
2772}
2773
John McCall31168b02011-06-15 23:02:42 +00002774/// Emit an ARC autorelease of the result of a function.
John McCallffa2c1a2012-01-29 07:46:59 +00002775///
2776/// \return the value to actually return from the function
John McCall31168b02011-06-15 23:02:42 +00002777static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2778 llvm::Value *result) {
John McCallffa2c1a2012-01-29 07:46:59 +00002779 // If we're returning 'self', kill the initial retain. This is a
2780 // heuristic attempt to "encourage correctness" in the really unfortunate
2781 // case where we have a return of self during a dealloc and we desperately
2782 // need to avoid the possible autorelease.
2783 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2784 return self;
2785
John McCall31168b02011-06-15 23:02:42 +00002786 // At -O0, try to emit a fused retain/autorelease.
2787 if (CGF.shouldUseFusedARCCalls())
2788 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2789 return fused;
2790
2791 return CGF.EmitARCAutoreleaseReturnValue(result);
2792}
2793
John McCall6e1c0122012-01-29 02:35:02 +00002794/// Heuristically search for a dominating store to the return-value slot.
2795static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
Jakub Kuderskif50ab0f2015-09-08 10:36:42 +00002796 // Check if a User is a store which pointerOperand is the ReturnValue.
2797 // We are looking for stores to the ReturnValue, not for stores of the
2798 // ReturnValue to some other location.
2799 auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
2800 auto *SI = dyn_cast<llvm::StoreInst>(U);
2801 if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer())
2802 return nullptr;
2803 // These aren't actually possible for non-coerced returns, and we
2804 // only care about non-coerced returns on this code path.
2805 assert(!SI->isAtomic() && !SI->isVolatile());
2806 return SI;
2807 };
John McCall6e1c0122012-01-29 02:35:02 +00002808 // If there are multiple uses of the return-value slot, just check
2809 // for something immediately preceding the IP. Sometimes this can
2810 // happen with how we generate implicit-returns; it can also happen
2811 // with noreturn cleanups.
John McCall7f416cc2015-09-08 08:05:57 +00002812 if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
John McCall6e1c0122012-01-29 02:35:02 +00002813 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
Craig Topper8a13c412014-05-21 05:09:00 +00002814 if (IP->empty()) return nullptr;
David Majnemerdc012fa2015-04-22 21:38:15 +00002815 llvm::Instruction *I = &IP->back();
2816
2817 // Skip lifetime markers
2818 for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
2819 IE = IP->rend();
2820 II != IE; ++II) {
2821 if (llvm::IntrinsicInst *Intrinsic =
2822 dyn_cast<llvm::IntrinsicInst>(&*II)) {
2823 if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
2824 const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
2825 ++II;
Alexey Samsonov10544202015-06-12 21:05:32 +00002826 if (II == IE)
2827 break;
2828 if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
2829 continue;
David Majnemerdc012fa2015-04-22 21:38:15 +00002830 }
2831 }
2832 I = &*II;
2833 break;
2834 }
2835
Jakub Kuderskif50ab0f2015-09-08 10:36:42 +00002836 return GetStoreIfValid(I);
John McCall6e1c0122012-01-29 02:35:02 +00002837 }
2838
2839 llvm::StoreInst *store =
Jakub Kuderskif50ab0f2015-09-08 10:36:42 +00002840 GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
Craig Topper8a13c412014-05-21 05:09:00 +00002841 if (!store) return nullptr;
John McCall6e1c0122012-01-29 02:35:02 +00002842
John McCall6e1c0122012-01-29 02:35:02 +00002843 // Now do a first-and-dirty dominance check: just walk up the
2844 // single-predecessors chain from the current insertion point.
2845 llvm::BasicBlock *StoreBB = store->getParent();
2846 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2847 while (IP != StoreBB) {
2848 if (!(IP = IP->getSinglePredecessor()))
Craig Topper8a13c412014-05-21 05:09:00 +00002849 return nullptr;
John McCall6e1c0122012-01-29 02:35:02 +00002850 }
2851
2852 // Okay, the store's basic block dominates the insertion point; we
2853 // can do our thing.
2854 return store;
2855}
2856
Adrian Prantl3be10542013-05-02 17:30:20 +00002857void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
Nick Lewycky2d84e842013-10-02 02:29:49 +00002858 bool EmitRetDbgLoc,
2859 SourceLocation EndLoc) {
Vedant Kumar09b5bfd2017-12-21 00:10:25 +00002860 if (FI.isNoReturn()) {
2861 // Noreturn functions don't return.
2862 EmitUnreachable(EndLoc);
2863 return;
2864 }
2865
Hans Wennborgd71907d2014-09-04 22:16:33 +00002866 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
2867 // Naked functions don't have epilogues.
2868 Builder.CreateUnreachable();
2869 return;
2870 }
2871
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002872 // Functions with no result always return void.
John McCall7f416cc2015-09-08 08:05:57 +00002873 if (!ReturnValue.isValid()) {
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002874 Builder.CreateRetVoid();
Chris Lattner726b3d02010-06-26 23:13:19 +00002875 return;
Daniel Dunbara72d4ae2008-09-10 02:41:04 +00002876 }
Daniel Dunbar6696e222010-06-30 21:27:58 +00002877
Dan Gohman481e40c2010-07-20 20:13:52 +00002878 llvm::DebugLoc RetDbgLoc;
Craig Topper8a13c412014-05-21 05:09:00 +00002879 llvm::Value *RV = nullptr;
Chris Lattner726b3d02010-06-26 23:13:19 +00002880 QualType RetTy = FI.getReturnType();
2881 const ABIArgInfo &RetAI = FI.getReturnInfo();
2882
2883 switch (RetAI.getKind()) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002884 case ABIArgInfo::InAlloca:
Reid Klecknerfab1e892014-02-25 00:59:14 +00002885 // Aggregrates get evaluated directly into the destination. Sometimes we
2886 // need to return the sret value in a register, though.
2887 assert(hasAggregateEvaluationKind(RetTy));
2888 if (RetAI.getInAllocaSRet()) {
2889 llvm::Function::arg_iterator EI = CurFn->arg_end();
2890 --EI;
Duncan P. N. Exon Smith9f5260a2015-11-06 23:00:41 +00002891 llvm::Value *ArgStruct = &*EI;
David Blaikie2e804282015-04-05 22:47:07 +00002892 llvm::Value *SRet = Builder.CreateStructGEP(
2893 nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
John McCall7f416cc2015-09-08 08:05:57 +00002894 RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
Reid Klecknerfab1e892014-02-25 00:59:14 +00002895 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00002896 break;
2897
Daniel Dunbar03816342010-08-21 02:24:36 +00002898 case ABIArgInfo::Indirect: {
Reid Kleckner37abaca2014-05-09 22:46:15 +00002899 auto AI = CurFn->arg_begin();
2900 if (RetAI.isSRetAfterThis())
2901 ++AI;
John McCall47fb9502013-03-07 21:37:08 +00002902 switch (getEvaluationKind(RetTy)) {
2903 case TEK_Complex: {
2904 ComplexPairTy RT =
John McCall7f416cc2015-09-08 08:05:57 +00002905 EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
Duncan P. N. Exon Smith9f5260a2015-11-06 23:00:41 +00002906 EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
John McCall47fb9502013-03-07 21:37:08 +00002907 /*isInit*/ true);
2908 break;
2909 }
2910 case TEK_Aggregate:
Chris Lattner726b3d02010-06-26 23:13:19 +00002911 // Do nothing; aggregrates get evaluated directly into the destination.
John McCall47fb9502013-03-07 21:37:08 +00002912 break;
2913 case TEK_Scalar:
2914 EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
Duncan P. N. Exon Smith9f5260a2015-11-06 23:00:41 +00002915 MakeNaturalAlignAddrLValue(&*AI, RetTy),
John McCall47fb9502013-03-07 21:37:08 +00002916 /*isInit*/ true);
2917 break;
Chris Lattner726b3d02010-06-26 23:13:19 +00002918 }
2919 break;
Daniel Dunbar03816342010-08-21 02:24:36 +00002920 }
Chris Lattner726b3d02010-06-26 23:13:19 +00002921
2922 case ABIArgInfo::Extend:
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002923 case ABIArgInfo::Direct:
Chris Lattner8a2f3c72010-07-30 04:02:24 +00002924 if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
2925 RetAI.getDirectOffset() == 0) {
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002926 // The internal return value temp always will have pointer-to-return-type
2927 // type, just do a load.
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002928
John McCall6e1c0122012-01-29 02:35:02 +00002929 // If there is a dominating store to ReturnValue, we can elide
2930 // the load, zap the store, and usually zap the alloca.
David Majnemerdc012fa2015-04-22 21:38:15 +00002931 if (llvm::StoreInst *SI =
2932 findDominatingStoreToReturnValue(*this)) {
Adrian Prantl4c9a38a2013-05-30 18:12:23 +00002933 // Reuse the debug location from the store unless there is
2934 // cleanup code to be emitted between the store and return
2935 // instruction.
2936 if (EmitRetDbgLoc && !AutoreleaseResult)
Adrian Prantl3be10542013-05-02 17:30:20 +00002937 RetDbgLoc = SI->getDebugLoc();
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002938 // Get the stored value and nuke the now-dead store.
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002939 RV = SI->getValueOperand();
2940 SI->eraseFromParent();
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002941
John McCall6e1c0122012-01-29 02:35:02 +00002942 // Otherwise, we have to do a simple load.
2943 } else {
2944 RV = Builder.CreateLoad(ReturnValue);
Chris Lattner3fcc7902010-06-27 01:06:27 +00002945 }
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002946 } else {
Chris Lattner8a2f3c72010-07-30 04:02:24 +00002947 // If the value is offset in memory, apply the offset now.
John McCall7f416cc2015-09-08 08:05:57 +00002948 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00002949
John McCall7f416cc2015-09-08 08:05:57 +00002950 RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
Chris Lattner3fcc7902010-06-27 01:06:27 +00002951 }
John McCall31168b02011-06-15 23:02:42 +00002952
2953 // In ARC, end functions that return a retainable type with a call
2954 // to objc_autoreleaseReturnValue.
2955 if (AutoreleaseResult) {
Akira Hatanaka9d8ac612016-02-17 21:09:50 +00002956#ifndef NDEBUG
2957 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
2958 // been stripped of the typedefs, so we cannot use RetTy here. Get the
2959 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
2960 // CurCodeDecl or BlockInfo.
2961 QualType RT;
2962
2963 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
2964 RT = FD->getReturnType();
2965 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
2966 RT = MD->getReturnType();
2967 else if (isa<BlockDecl>(CurCodeDecl))
2968 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
2969 else
2970 llvm_unreachable("Unexpected function/method type");
2971
David Blaikiebbafb8a2012-03-11 07:00:24 +00002972 assert(getLangOpts().ObjCAutoRefCount &&
John McCall31168b02011-06-15 23:02:42 +00002973 !FI.isReturnsRetained() &&
Akira Hatanaka9d8ac612016-02-17 21:09:50 +00002974 RT->isObjCRetainableType());
2975#endif
John McCall31168b02011-06-15 23:02:42 +00002976 RV = emitAutoreleaseOfResult(*this, RV);
2977 }
2978
Chris Lattner726b3d02010-06-26 23:13:19 +00002979 break;
Chris Lattner726b3d02010-06-26 23:13:19 +00002980
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00002981 case ABIArgInfo::Ignore:
Chris Lattner726b3d02010-06-26 23:13:19 +00002982 break;
2983
John McCallf26e73d2016-03-11 04:30:43 +00002984 case ABIArgInfo::CoerceAndExpand: {
2985 auto coercionType = RetAI.getCoerceAndExpandType();
John McCallf26e73d2016-03-11 04:30:43 +00002986
2987 // Load all of the coerced elements out into results.
2988 llvm::SmallVector<llvm::Value*, 4> results;
2989 Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
2990 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
2991 auto coercedEltType = coercionType->getElementType(i);
2992 if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
2993 continue;
2994
James Y Knight751fe282019-02-09 22:22:28 +00002995 auto eltAddr = Builder.CreateStructGEP(addr, i);
John McCallf26e73d2016-03-11 04:30:43 +00002996 auto elt = Builder.CreateLoad(eltAddr);
2997 results.push_back(elt);
2998 }
2999
3000 // If we have one result, it's the single direct result type.
3001 if (results.size() == 1) {
3002 RV = results[0];
3003
3004 // Otherwise, we need to make a first-class aggregate.
3005 } else {
3006 // Construct a return type that lacks padding elements.
3007 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3008
3009 RV = llvm::UndefValue::get(returnType);
3010 for (unsigned i = 0, e = results.size(); i != e; ++i) {
3011 RV = Builder.CreateInsertValue(RV, results[i], i);
3012 }
3013 }
3014 break;
3015 }
3016
Chris Lattner726b3d02010-06-26 23:13:19 +00003017 case ABIArgInfo::Expand:
David Blaikie83d382b2011-09-23 05:06:16 +00003018 llvm_unreachable("Invalid ABI kind for return argument");
Chris Lattner726b3d02010-06-26 23:13:19 +00003019 }
3020
Alexey Samsonovde443c52014-08-13 00:26:40 +00003021 llvm::Instruction *Ret;
3022 if (RV) {
Vedant Kumarc34d3432017-06-23 21:32:38 +00003023 EmitReturnValueCheck(RV);
Alexey Samsonovde443c52014-08-13 00:26:40 +00003024 Ret = Builder.CreateRet(RV);
3025 } else {
3026 Ret = Builder.CreateRetVoid();
3027 }
3028
Duncan P. N. Exon Smith2809cc72015-03-30 20:01:41 +00003029 if (RetDbgLoc)
Benjamin Kramer03278662015-02-07 13:15:54 +00003030 Ret->setDebugLoc(std::move(RetDbgLoc));
Daniel Dunbar613855c2008-09-09 23:27:19 +00003031}
3032
Vedant Kumarc34d3432017-06-23 21:32:38 +00003033void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003034 // A current decl may not be available when emitting vtable thunks.
3035 if (!CurCodeDecl)
3036 return;
3037
3038 ReturnsNonNullAttr *RetNNAttr = nullptr;
3039 if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3040 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3041
3042 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3043 return;
3044
3045 // Prefer the returns_nonnull attribute if it's present.
3046 SourceLocation AttrLoc;
3047 SanitizerMask CheckKind;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003048 SanitizerHandler Handler;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003049 if (RetNNAttr) {
3050 assert(!requiresReturnValueNullabilityCheck() &&
3051 "Cannot check nullability and the nonnull attribute");
3052 AttrLoc = RetNNAttr->getLocation();
3053 CheckKind = SanitizerKind::ReturnsNonnullAttribute;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003054 Handler = SanitizerHandler::NonnullReturn;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003055 } else {
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003056 if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3057 if (auto *TSI = DD->getTypeSourceInfo())
3058 if (auto FTL = TSI->getTypeLoc().castAs<FunctionTypeLoc>())
3059 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3060 CheckKind = SanitizerKind::NullabilityReturn;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003061 Handler = SanitizerHandler::NullabilityReturn;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003062 }
3063
3064 SanitizerScope SanScope(this);
3065
Vedant Kumarc34d3432017-06-23 21:32:38 +00003066 // Make sure the "return" source location is valid. If we're checking a
3067 // nullability annotation, make sure the preconditions for the check are met.
3068 llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3069 llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3070 llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3071 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003072 if (requiresReturnValueNullabilityCheck())
Vedant Kumarc34d3432017-06-23 21:32:38 +00003073 CanNullCheck =
3074 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3075 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3076 EmitBlock(Check);
3077
3078 // Now do the null check.
3079 llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3080 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3081 llvm::Value *DynamicData[] = {SLocPtr};
3082 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3083
3084 EmitBlock(NoCheck);
3085
3086#ifndef NDEBUG
3087 // The return location should not be used after the check has been emitted.
3088 ReturnLocation = Address::invalid();
3089#endif
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003090}
3091
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003092static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3093 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3094 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
3095}
3096
John McCall7f416cc2015-09-08 08:05:57 +00003097static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3098 QualType Ty) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003099 // FIXME: Generate IR in one pass, rather than going back and fixing up these
3100 // placeholders.
3101 llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
Peter Collingbourneb367c562016-11-28 22:30:21 +00003102 llvm::Type *IRPtrTy = IRTy->getPointerTo();
3103 llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
John McCall7f416cc2015-09-08 08:05:57 +00003104
3105 // FIXME: When we generate this IR in one pass, we shouldn't need
3106 // this win32-specific alignment hack.
3107 CharUnits Align = CharUnits::fromQuantity(4);
Peter Collingbourneb367c562016-11-28 22:30:21 +00003108 Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
John McCall7f416cc2015-09-08 08:05:57 +00003109
3110 return AggValueSlot::forAddr(Address(Placeholder, Align),
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003111 Ty.getQualifiers(),
3112 AggValueSlot::IsNotDestructed,
3113 AggValueSlot::DoesNotNeedGCBarriers,
Richard Smithe78fac52018-04-05 20:52:58 +00003114 AggValueSlot::IsNotAliased,
3115 AggValueSlot::DoesNotOverlap);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003116}
3117
John McCall32ea9692011-03-11 20:59:21 +00003118void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
Nick Lewycky2d84e842013-10-02 02:29:49 +00003119 const VarDecl *param,
3120 SourceLocation loc) {
John McCall23f66262010-05-26 22:34:26 +00003121 // StartFunction converted the ABI-lowered parameter(s) into a
3122 // local alloca. We need to turn that into an r-value suitable
3123 // for EmitCall.
John McCall7f416cc2015-09-08 08:05:57 +00003124 Address local = GetAddrOfLocalVar(param);
John McCall23f66262010-05-26 22:34:26 +00003125
John McCall32ea9692011-03-11 20:59:21 +00003126 QualType type = param->getType();
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00003127
Reid Kleckner25b56022018-12-12 23:46:06 +00003128 if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3129 CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3130 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003131
John McCall811b2912016-11-18 01:08:24 +00003132 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3133 // but the argument needs to be the original pointer.
3134 if (type->isReferenceType()) {
3135 args.add(RValue::get(Builder.CreateLoad(local)), type);
3136
3137 // In ARC, move out of consumed arguments so that the release cleanup
3138 // entered by StartFunction doesn't cause an over-release. This isn't
3139 // optimal -O0 code generation, but it should get cleaned up when
3140 // optimization is enabled. This also assumes that delegate calls are
3141 // performed exactly once for a set of arguments, but that should be safe.
3142 } else if (getLangOpts().ObjCAutoRefCount &&
3143 param->hasAttr<NSConsumedAttr>() &&
3144 type->isObjCRetainableType()) {
3145 llvm::Value *ptr = Builder.CreateLoad(local);
3146 auto null =
3147 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3148 Builder.CreateStore(null, local);
3149 args.add(RValue::get(ptr), type);
3150
Richard Smithd62d4982016-06-14 01:13:21 +00003151 // For the most part, we just need to load the alloca, except that
3152 // aggregate r-values are actually pointers to temporaries.
John McCall811b2912016-11-18 01:08:24 +00003153 } else {
Richard Smithd62d4982016-06-14 01:13:21 +00003154 args.add(convertTempToRValue(local, type, loc), type);
John McCall811b2912016-11-18 01:08:24 +00003155 }
Akira Hatanakaccda3d22018-04-27 06:57:00 +00003156
3157 // Deactivate the cleanup for the callee-destructed param that was pushed.
3158 if (hasAggregateEvaluationKind(type) && !CurFuncIsThunk &&
Simon Pilgrim1cd399c2019-10-03 11:22:48 +00003159 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
Richard Smith2b4fa532019-09-29 05:08:46 +00003160 param->needsDestruction(getContext())) {
Akira Hatanakaccda3d22018-04-27 06:57:00 +00003161 EHScopeStack::stable_iterator cleanup =
3162 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3163 assert(cleanup.isValid() &&
3164 "cleanup for callee-destructed param not recorded");
3165 // This unreachable is a temporary marker which will be removed later.
3166 llvm::Instruction *isActive = Builder.CreateUnreachable();
3167 args.addArgCleanupDeactivation(cleanup, isActive);
3168 }
John McCall23f66262010-05-26 22:34:26 +00003169}
3170
John McCall31168b02011-06-15 23:02:42 +00003171static bool isProvablyNull(llvm::Value *addr) {
3172 return isa<llvm::ConstantPointerNull>(addr);
3173}
3174
John McCall31168b02011-06-15 23:02:42 +00003175/// Emit the actual writing-back of a writeback.
3176static void emitWriteback(CodeGenFunction &CGF,
3177 const CallArgList::Writeback &writeback) {
John McCalleff18842013-03-23 02:35:54 +00003178 const LValue &srcLV = writeback.Source;
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003179 Address srcAddr = srcLV.getAddress(CGF);
John McCall7f416cc2015-09-08 08:05:57 +00003180 assert(!isProvablyNull(srcAddr.getPointer()) &&
John McCall31168b02011-06-15 23:02:42 +00003181 "shouldn't have writeback for provably null argument");
3182
Craig Topper8a13c412014-05-21 05:09:00 +00003183 llvm::BasicBlock *contBB = nullptr;
John McCall31168b02011-06-15 23:02:42 +00003184
3185 // If the argument wasn't provably non-null, we need to null check
3186 // before doing the store.
Nuno Lopes9211cee2017-09-09 18:25:36 +00003187 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3188 CGF.CGM.getDataLayout());
John McCall31168b02011-06-15 23:02:42 +00003189 if (!provablyNonNull) {
3190 llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3191 contBB = CGF.createBasicBlock("icr.done");
3192
John McCall7f416cc2015-09-08 08:05:57 +00003193 llvm::Value *isNull =
3194 CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
John McCall31168b02011-06-15 23:02:42 +00003195 CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3196 CGF.EmitBlock(writebackBB);
3197 }
3198
3199 // Load the value to writeback.
3200 llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3201
3202 // Cast it back, in case we're writing an id to a Foo* or something.
John McCall7f416cc2015-09-08 08:05:57 +00003203 value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3204 "icr.writeback-cast");
Fangrui Song6907ce22018-07-30 19:24:48 +00003205
John McCall31168b02011-06-15 23:02:42 +00003206 // Perform the writeback.
John McCalleff18842013-03-23 02:35:54 +00003207
3208 // If we have a "to use" value, it's something we need to emit a use
3209 // of. This has to be carefully threaded in: if it's done after the
3210 // release it's potentially undefined behavior (and the optimizer
3211 // will ignore it), and if it happens before the retain then the
3212 // optimizer could move the release there.
3213 if (writeback.ToUse) {
3214 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3215
3216 // Retain the new value. No need to block-copy here: the block's
3217 // being passed up the stack.
3218 value = CGF.EmitARCRetainNonBlock(value);
3219
3220 // Emit the intrinsic use here.
3221 CGF.EmitARCIntrinsicUse(writeback.ToUse);
3222
3223 // Load the old value (primitively).
Nick Lewycky2d84e842013-10-02 02:29:49 +00003224 llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
John McCalleff18842013-03-23 02:35:54 +00003225
3226 // Put the new value in place (primitively).
3227 CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3228
3229 // Release the old value.
3230 CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3231
3232 // Otherwise, we can just do a normal lvalue store.
3233 } else {
3234 CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3235 }
John McCall31168b02011-06-15 23:02:42 +00003236
3237 // Jump to the continuation block.
3238 if (!provablyNonNull)
3239 CGF.EmitBlock(contBB);
3240}
3241
3242static void emitWritebacks(CodeGenFunction &CGF,
3243 const CallArgList &args) {
Aaron Ballman36a7fa82014-03-17 17:22:27 +00003244 for (const auto &I : args.writebacks())
3245 emitWriteback(CGF, I);
John McCall31168b02011-06-15 23:02:42 +00003246}
3247
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003248static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3249 const CallArgList &CallArgs) {
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003250 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3251 CallArgs.getCleanupsToDeactivate();
3252 // Iterate in reverse to increase the likelihood of popping the cleanup.
Pete Cooper57d3f142015-07-30 17:22:52 +00003253 for (const auto &I : llvm::reverse(Cleanups)) {
3254 CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3255 I.IsActiveIP->eraseFromParent();
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003256 }
3257}
3258
John McCalleff18842013-03-23 02:35:54 +00003259static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3260 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3261 if (uop->getOpcode() == UO_AddrOf)
3262 return uop->getSubExpr();
Craig Topper8a13c412014-05-21 05:09:00 +00003263 return nullptr;
John McCalleff18842013-03-23 02:35:54 +00003264}
3265
John McCall31168b02011-06-15 23:02:42 +00003266/// Emit an argument that's being passed call-by-writeback. That is,
John McCall7f416cc2015-09-08 08:05:57 +00003267/// we are passing the address of an __autoreleased temporary; it
3268/// might be copy-initialized with the current value of the given
3269/// address, but it will definitely be copied out of after the call.
John McCall31168b02011-06-15 23:02:42 +00003270static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3271 const ObjCIndirectCopyRestoreExpr *CRE) {
John McCalleff18842013-03-23 02:35:54 +00003272 LValue srcLV;
3273
3274 // Make an optimistic effort to emit the address as an l-value.
Eric Christopher2c4555a2015-06-19 01:52:53 +00003275 // This can fail if the argument expression is more complicated.
John McCalleff18842013-03-23 02:35:54 +00003276 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3277 srcLV = CGF.EmitLValue(lvExpr);
3278
3279 // Otherwise, just emit it as a scalar.
3280 } else {
John McCall7f416cc2015-09-08 08:05:57 +00003281 Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
John McCalleff18842013-03-23 02:35:54 +00003282
3283 QualType srcAddrType =
3284 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
John McCall7f416cc2015-09-08 08:05:57 +00003285 srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
John McCalleff18842013-03-23 02:35:54 +00003286 }
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003287 Address srcAddr = srcLV.getAddress(CGF);
John McCall31168b02011-06-15 23:02:42 +00003288
3289 // The dest and src types don't necessarily match in LLVM terms
3290 // because of the crazy ObjC compatibility rules.
3291
Chris Lattner2192fe52011-07-18 04:24:23 +00003292 llvm::PointerType *destType =
John McCall31168b02011-06-15 23:02:42 +00003293 cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3294
3295 // If the address is a constant null, just pass the appropriate null.
John McCall7f416cc2015-09-08 08:05:57 +00003296 if (isProvablyNull(srcAddr.getPointer())) {
John McCall31168b02011-06-15 23:02:42 +00003297 args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3298 CRE->getType());
3299 return;
3300 }
3301
John McCall31168b02011-06-15 23:02:42 +00003302 // Create the temporary.
John McCall7f416cc2015-09-08 08:05:57 +00003303 Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3304 CGF.getPointerAlign(),
3305 "icr.temp");
Fariborz Jahanianfbd19742012-11-27 23:02:53 +00003306 // Loading an l-value can introduce a cleanup if the l-value is __weak,
3307 // and that cleanup will be conditional if we can't prove that the l-value
3308 // isn't null, so we need to register a dominating point so that the cleanups
3309 // system will make valid IR.
3310 CodeGenFunction::ConditionalEvaluation condEval(CGF);
Fangrui Song6907ce22018-07-30 19:24:48 +00003311
John McCall31168b02011-06-15 23:02:42 +00003312 // Zero-initialize it if we're not doing a copy-initialization.
3313 bool shouldCopy = CRE->shouldCopy();
3314 if (!shouldCopy) {
3315 llvm::Value *null =
3316 llvm::ConstantPointerNull::get(
3317 cast<llvm::PointerType>(destType->getElementType()));
3318 CGF.Builder.CreateStore(null, temp);
3319 }
Craig Topper8a13c412014-05-21 05:09:00 +00003320
3321 llvm::BasicBlock *contBB = nullptr;
3322 llvm::BasicBlock *originBB = nullptr;
John McCall31168b02011-06-15 23:02:42 +00003323
3324 // If the address is *not* known to be non-null, we need to switch.
3325 llvm::Value *finalArgument;
3326
Nuno Lopes9211cee2017-09-09 18:25:36 +00003327 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3328 CGF.CGM.getDataLayout());
John McCall31168b02011-06-15 23:02:42 +00003329 if (provablyNonNull) {
John McCall7f416cc2015-09-08 08:05:57 +00003330 finalArgument = temp.getPointer();
John McCall31168b02011-06-15 23:02:42 +00003331 } else {
John McCall7f416cc2015-09-08 08:05:57 +00003332 llvm::Value *isNull =
3333 CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
John McCall31168b02011-06-15 23:02:42 +00003334
Fangrui Song6907ce22018-07-30 19:24:48 +00003335 finalArgument = CGF.Builder.CreateSelect(isNull,
John McCall31168b02011-06-15 23:02:42 +00003336 llvm::ConstantPointerNull::get(destType),
John McCall7f416cc2015-09-08 08:05:57 +00003337 temp.getPointer(), "icr.argument");
John McCall31168b02011-06-15 23:02:42 +00003338
3339 // If we need to copy, then the load has to be conditional, which
3340 // means we need control flow.
3341 if (shouldCopy) {
John McCalleff18842013-03-23 02:35:54 +00003342 originBB = CGF.Builder.GetInsertBlock();
John McCall31168b02011-06-15 23:02:42 +00003343 contBB = CGF.createBasicBlock("icr.cont");
3344 llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3345 CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3346 CGF.EmitBlock(copyBB);
Fariborz Jahanianfbd19742012-11-27 23:02:53 +00003347 condEval.begin(CGF);
John McCall31168b02011-06-15 23:02:42 +00003348 }
3349 }
3350
Craig Topper8a13c412014-05-21 05:09:00 +00003351 llvm::Value *valueToUse = nullptr;
John McCalleff18842013-03-23 02:35:54 +00003352
John McCall31168b02011-06-15 23:02:42 +00003353 // Perform a copy if necessary.
3354 if (shouldCopy) {
Nick Lewycky2d84e842013-10-02 02:29:49 +00003355 RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
John McCall31168b02011-06-15 23:02:42 +00003356 assert(srcRV.isScalar());
3357
3358 llvm::Value *src = srcRV.getScalarVal();
3359 src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3360 "icr.cast");
3361
3362 // Use an ordinary store, not a store-to-lvalue.
3363 CGF.Builder.CreateStore(src, temp);
John McCalleff18842013-03-23 02:35:54 +00003364
3365 // If optimization is enabled, and the value was held in a
3366 // __strong variable, we need to tell the optimizer that this
3367 // value has to stay alive until we're doing the store back.
3368 // This is because the temporary is effectively unretained,
3369 // and so otherwise we can violate the high-level semantics.
3370 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3371 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
3372 valueToUse = src;
3373 }
John McCall31168b02011-06-15 23:02:42 +00003374 }
Fangrui Song6907ce22018-07-30 19:24:48 +00003375
John McCall31168b02011-06-15 23:02:42 +00003376 // Finish the control flow if we needed it.
Fariborz Jahanianfbd19742012-11-27 23:02:53 +00003377 if (shouldCopy && !provablyNonNull) {
John McCalleff18842013-03-23 02:35:54 +00003378 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
John McCall31168b02011-06-15 23:02:42 +00003379 CGF.EmitBlock(contBB);
John McCalleff18842013-03-23 02:35:54 +00003380
3381 // Make a phi for the value to intrinsically use.
3382 if (valueToUse) {
3383 llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3384 "icr.to-use");
3385 phiToUse->addIncoming(valueToUse, copyBB);
3386 phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3387 originBB);
3388 valueToUse = phiToUse;
3389 }
3390
Fariborz Jahanianfbd19742012-11-27 23:02:53 +00003391 condEval.end(CGF);
3392 }
John McCall31168b02011-06-15 23:02:42 +00003393
John McCalleff18842013-03-23 02:35:54 +00003394 args.addWriteback(srcLV, temp, valueToUse);
John McCall31168b02011-06-15 23:02:42 +00003395 args.add(RValue::get(finalArgument), CRE->getType());
3396}
3397
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003398void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
Richard Smith762672a2016-09-28 19:09:10 +00003399 assert(!StackBase);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003400
3401 // Save the stack.
3402 llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
David Blaikie43f9bb72015-05-18 22:14:03 +00003403 StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003404}
3405
Nico Weber8cdb3f92015-08-25 18:43:32 +00003406void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3407 if (StackBase) {
Reid Kleckner7c2f9e82015-10-08 00:17:45 +00003408 // Restore the stack after the call.
James Y Knight8799cae2019-02-03 21:53:49 +00003409 llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
Nico Weber8cdb3f92015-08-25 18:43:32 +00003410 CGF.Builder.CreateCall(F, StackBase);
3411 }
3412}
3413
Nuno Lopes1ba2d782015-05-30 16:11:40 +00003414void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3415 SourceLocation ArgLoc,
Vedant Kumared00ea02017-03-06 05:28:22 +00003416 AbstractCallee AC,
Nuno Lopes1ba2d782015-05-30 16:11:40 +00003417 unsigned ParmNum) {
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003418 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
3419 SanOpts.has(SanitizerKind::NullabilityArg)))
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003420 return;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003421
3422 // The param decl may be missing in a variadic function.
Vedant Kumared00ea02017-03-06 05:28:22 +00003423 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr;
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003424 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003425
Fangrui Song6907ce22018-07-30 19:24:48 +00003426 // Prefer the nonnull attribute if it's present.
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003427 const NonNullAttr *NNAttr = nullptr;
3428 if (SanOpts.has(SanitizerKind::NonnullAttribute))
3429 NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3430
3431 bool CanCheckNullability = false;
3432 if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
3433 auto Nullability = PVD->getType()->getNullability(getContext());
3434 CanCheckNullability = Nullability &&
3435 *Nullability == NullabilityKind::NonNull &&
3436 PVD->getTypeSourceInfo();
3437 }
3438
3439 if (!NNAttr && !CanCheckNullability)
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003440 return;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003441
3442 SourceLocation AttrLoc;
3443 SanitizerMask CheckKind;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003444 SanitizerHandler Handler;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003445 if (NNAttr) {
3446 AttrLoc = NNAttr->getLocation();
3447 CheckKind = SanitizerKind::NonnullAttribute;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003448 Handler = SanitizerHandler::NonnullArg;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003449 } else {
3450 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
3451 CheckKind = SanitizerKind::NullabilityArg;
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003452 Handler = SanitizerHandler::NullabilityArg;
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003453 }
3454
Nuno Lopes1ba2d782015-05-30 16:11:40 +00003455 SanitizerScope SanScope(this);
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003456 assert(RV.isScalar());
3457 llvm::Value *V = RV.getScalarVal();
3458 llvm::Value *Cond =
Nuno Lopes1ba2d782015-05-30 16:11:40 +00003459 Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003460 llvm::Constant *StaticData[] = {
Vedant Kumar42c17ec2017-03-14 01:56:34 +00003461 EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
Nuno Lopes1ba2d782015-05-30 16:11:40 +00003462 llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003463 };
Vedant Kumar2b9f48a2017-03-14 16:48:29 +00003464 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
Alexey Samsonov8e1162c2014-09-08 17:22:45 +00003465}
3466
David Blaikief05779e2015-07-21 18:37:18 +00003467void CodeGenFunction::EmitCallArgs(
3468 CallArgList &Args, ArrayRef<QualType> ArgTypes,
3469 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
Vedant Kumared00ea02017-03-06 05:28:22 +00003470 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
David Blaikief05779e2015-07-21 18:37:18 +00003471 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
George Burgess IV3e3bb95b2015-12-02 21:58:08 +00003472
Reid Kleckner739756c2013-12-04 19:23:12 +00003473 // We *have* to evaluate arguments from right to left in the MS C++ ABI,
Richard Smitha560ccf2016-09-29 21:30:12 +00003474 // because arguments are destroyed left to right in the callee. As a special
3475 // case, there are certain language constructs that require left-to-right
3476 // evaluation, and in those cases we consider the evaluation order requirement
3477 // to trump the "destruction order is reverse construction order" guarantee.
3478 bool LeftToRight =
3479 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
3480 ? Order == EvaluationOrder::ForceLeftToRight
3481 : Order != EvaluationOrder::ForceRightToLeft;
3482
George Burgess IV0d6592a2017-02-23 05:59:56 +00003483 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
3484 RValue EmittedArg) {
Vedant Kumared00ea02017-03-06 05:28:22 +00003485 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
George Burgess IV0d6592a2017-02-23 05:59:56 +00003486 return;
Vedant Kumared00ea02017-03-06 05:28:22 +00003487 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
George Burgess IV0d6592a2017-02-23 05:59:56 +00003488 if (PS == nullptr)
3489 return;
3490
3491 const auto &Context = getContext();
3492 auto SizeTy = Context.getSizeType();
3493 auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3494 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
3495 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
Erik Pilkington9c3b5882019-01-30 20:34:53 +00003496 EmittedArg.getScalarVal(),
Erik Pilkington02d5fb12019-03-19 20:44:18 +00003497 PS->isDynamic());
George Burgess IV0d6592a2017-02-23 05:59:56 +00003498 Args.add(RValue::get(V), SizeTy);
3499 // If we're emitting args in reverse, be sure to do so with
3500 // pass_object_size, as well.
3501 if (!LeftToRight)
3502 std::swap(Args.back(), *(&Args.back() - 1));
3503 };
3504
Richard Smitha560ccf2016-09-29 21:30:12 +00003505 // Insert a stack save if we're going to need any inalloca args.
3506 bool HasInAllocaArgs = false;
3507 if (CGM.getTarget().getCXXABI().isMicrosoft()) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003508 for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
3509 I != E && !HasInAllocaArgs; ++I)
3510 HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
3511 if (HasInAllocaArgs) {
3512 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3513 Args.allocateArgumentMemory(*this);
3514 }
Richard Smitha560ccf2016-09-29 21:30:12 +00003515 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003516
Richard Smitha560ccf2016-09-29 21:30:12 +00003517 // Evaluate each argument in the appropriate order.
3518 size_t CallArgsStart = Args.size();
3519 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
3520 unsigned Idx = LeftToRight ? I : E - I - 1;
3521 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
George Burgess IV0d6592a2017-02-23 05:59:56 +00003522 unsigned InitialArgSize = Args.size();
Akira Hatanaka46dd7dbc2017-06-28 00:42:48 +00003523 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
3524 // the argument and parameter match or the objc method is parameterized.
3525 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
3526 getContext().hasSameUnqualifiedType((*Arg)->getType(),
3527 ArgTypes[Idx]) ||
3528 (isa<ObjCMethodDecl>(AC.getDecl()) &&
3529 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
3530 "Argument and parameter types don't match");
Richard Smitha560ccf2016-09-29 21:30:12 +00003531 EmitCallArg(Args, *Arg, ArgTypes[Idx]);
George Burgess IV0d6592a2017-02-23 05:59:56 +00003532 // In particular, we depend on it being the last arg in Args, and the
3533 // objectsize bits depend on there only being one arg if !LeftToRight.
3534 assert(InitialArgSize + 1 == Args.size() &&
3535 "The code below depends on only adding one arg per EmitCallArg");
3536 (void)InitialArgSize;
Yaxun Liu5b330e82018-03-15 15:25:19 +00003537 // Since pointer argument are never emitted as LValue, it is safe to emit
3538 // non-null argument check for r-value only.
3539 if (!Args.back().hasLValue()) {
3540 RValue RVArg = Args.back().getKnownRValue();
3541 EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
3542 ParamsToSkip + Idx);
3543 // @llvm.objectsize should never have side-effects and shouldn't need
3544 // destruction/cleanups, so we can safely "emit" it after its arg,
3545 // regardless of right-to-leftness
3546 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
3547 }
Richard Smitha560ccf2016-09-29 21:30:12 +00003548 }
Reid Kleckner739756c2013-12-04 19:23:12 +00003549
Richard Smitha560ccf2016-09-29 21:30:12 +00003550 if (!LeftToRight) {
Reid Kleckner739756c2013-12-04 19:23:12 +00003551 // Un-reverse the arguments we just evaluated so they match up with the LLVM
3552 // IR function.
3553 std::reverse(Args.begin() + CallArgsStart, Args.end());
Reid Kleckner739756c2013-12-04 19:23:12 +00003554 }
3555}
3556
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003557namespace {
3558
David Blaikie7e70d682015-08-18 22:40:54 +00003559struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
John McCall7f416cc2015-09-08 08:05:57 +00003560 DestroyUnpassedArg(Address Addr, QualType Ty)
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003561 : Addr(Addr), Ty(Ty) {}
3562
John McCall7f416cc2015-09-08 08:05:57 +00003563 Address Addr;
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003564 QualType Ty;
3565
Craig Topper4f12f102014-03-12 06:41:41 +00003566 void Emit(CodeGenFunction &CGF, Flags flags) override {
Akira Hatanaka7275da02018-02-28 07:15:55 +00003567 QualType::DestructionKind DtorKind = Ty.isDestructedType();
3568 if (DtorKind == QualType::DK_cxx_destructor) {
3569 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
3570 assert(!Dtor->isTrivial());
3571 CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
Marco Antognini88559632019-07-22 09:39:13 +00003572 /*Delegating=*/false, Addr, Ty);
Akira Hatanaka7275da02018-02-28 07:15:55 +00003573 } else {
3574 CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
3575 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003576 }
3577};
3578
David Blaikie38b25912015-02-09 19:13:51 +00003579struct DisableDebugLocationUpdates {
3580 CodeGenFunction &CGF;
3581 bool disabledDebugInfo;
3582 DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
3583 if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo()))
3584 CGF.disableDebugInfo();
3585 }
3586 ~DisableDebugLocationUpdates() {
3587 if (disabledDebugInfo)
3588 CGF.enableDebugInfo();
3589 }
3590};
3591
Benjamin Kramer5b4296a2015-10-28 17:16:26 +00003592} // end anonymous namespace
3593
Yaxun Liu5b330e82018-03-15 15:25:19 +00003594RValue CallArg::getRValue(CodeGenFunction &CGF) const {
3595 if (!HasLV)
3596 return RV;
3597 LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
Richard Smithe78fac52018-04-05 20:52:58 +00003598 CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
3599 LV.isVolatile());
Yaxun Liu5b330e82018-03-15 15:25:19 +00003600 IsUsed = true;
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003601 return RValue::getAggregate(Copy.getAddress(CGF));
Yaxun Liu5b330e82018-03-15 15:25:19 +00003602}
3603
3604void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
3605 LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
3606 if (!HasLV && RV.isScalar())
Rui Ueyama49a3ad22019-07-16 04:46:31 +00003607 CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
Yaxun Liu5b330e82018-03-15 15:25:19 +00003608 else if (!HasLV && RV.isComplex())
3609 CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
3610 else {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003611 auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress();
Yaxun Liu5b330e82018-03-15 15:25:19 +00003612 LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
Richard Smithe78fac52018-04-05 20:52:58 +00003613 // We assume that call args are never copied into subobjects.
3614 CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
Yaxun Liu5b330e82018-03-15 15:25:19 +00003615 HasLV ? LV.isVolatileQualified()
3616 : RV.isVolatileQualified());
3617 }
3618 IsUsed = true;
3619}
3620
John McCall32ea9692011-03-11 20:59:21 +00003621void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
3622 QualType type) {
David Blaikie38b25912015-02-09 19:13:51 +00003623 DisableDebugLocationUpdates Dis(*this, E);
John McCall31168b02011-06-15 23:02:42 +00003624 if (const ObjCIndirectCopyRestoreExpr *CRE
3625 = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
Richard Smith9c6890a2012-11-01 22:30:59 +00003626 assert(getLangOpts().ObjCAutoRefCount);
John McCall31168b02011-06-15 23:02:42 +00003627 return emitWritebackArg(*this, args, CRE);
3628 }
3629
John McCall0a76c0c2011-08-26 18:42:59 +00003630 assert(type->isReferenceType() == E->isGLValue() &&
3631 "reference binding to unmaterialized r-value!");
3632
John McCall17054bd62011-08-26 21:08:13 +00003633 if (E->isGLValue()) {
3634 assert(E->getObjectKind() == OK_Ordinary);
Richard Smitha1c9d4d2013-06-12 23:38:09 +00003635 return args.add(EmitReferenceBindingToExpr(E), type);
John McCall17054bd62011-08-26 21:08:13 +00003636 }
Mike Stump11289f42009-09-09 15:08:12 +00003637
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003638 bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
3639
3640 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
3641 // However, we still have to push an EH-only cleanup in case we unwind before
3642 // we make it to the call.
Akira Hatanaka85282972018-05-15 21:00:30 +00003643 if (HasAggregateEvalKind &&
Simon Pilgrim1cd399c2019-10-03 11:22:48 +00003644 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
Reid Klecknerac640602014-05-01 03:07:18 +00003645 // If we're using inalloca, use the argument memory. Otherwise, use a
Reid Klecknere39ee212014-05-03 00:33:28 +00003646 // temporary.
Reid Klecknerac640602014-05-01 03:07:18 +00003647 AggValueSlot Slot;
3648 if (args.isUsingInAlloca())
3649 Slot = createPlaceholderSlot(*this, type);
3650 else
3651 Slot = CreateAggTemp(type, "agg.tmp");
Reid Klecknere39ee212014-05-03 00:33:28 +00003652
Akira Hatanaka4ce0e5a2018-04-18 23:33:15 +00003653 bool DestroyedInCallee = true, NeedsEHCleanup = true;
3654 if (const auto *RD = type->getAsCXXRecordDecl())
3655 DestroyedInCallee = RD->hasNonTrivialDestructor();
3656 else
3657 NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
3658
3659 if (DestroyedInCallee)
3660 Slot.setExternallyDestructed();
Reid Klecknere39ee212014-05-03 00:33:28 +00003661
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003662 EmitAggExpr(E, Slot);
3663 RValue RV = Slot.asRValue();
3664 args.add(RV, type);
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003665
Akira Hatanaka4ce0e5a2018-04-18 23:33:15 +00003666 if (DestroyedInCallee && NeedsEHCleanup) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003667 // Create a no-op GEP between the placeholder and the cleanup so we can
3668 // RAUW it successfully. It also serves as a marker of the first
3669 // instruction where the cleanup is active.
John McCall7f416cc2015-09-08 08:05:57 +00003670 pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
3671 type);
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003672 // This unreachable is a temporary marker which will be removed later.
3673 llvm::Instruction *IsActive = Builder.CreateUnreachable();
3674 args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003675 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003676 return;
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00003677 }
3678
3679 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
Eli Friedmandf968192011-05-26 00:10:27 +00003680 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
3681 LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
3682 assert(L.isSimple());
Yaxun Liu5b330e82018-03-15 15:25:19 +00003683 args.addUncopiedAggregate(L, type);
Eli Friedmandf968192011-05-26 00:10:27 +00003684 return;
3685 }
3686
John McCall32ea9692011-03-11 20:59:21 +00003687 args.add(EmitAnyExprToTemp(E), type);
Anders Carlsson60ce3fe2009-04-08 20:47:54 +00003688}
3689
Reid Kleckner79b0fd72014-10-10 00:05:45 +00003690QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
3691 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
3692 // implicitly widens null pointer constants that are arguments to varargs
3693 // functions to pointer-sized ints.
3694 if (!getTarget().getTriple().isOSWindows())
3695 return Arg->getType();
3696
3697 if (Arg->getType()->isIntegerType() &&
3698 getContext().getTypeSize(Arg->getType()) <
3699 getContext().getTargetInfo().getPointerWidth(0) &&
3700 Arg->isNullPointerConstant(getContext(),
3701 Expr::NPC_ValueDependentIsNotNull)) {
3702 return getContext().getIntPtrType();
3703 }
3704
3705 return Arg->getType();
3706}
3707
Dan Gohman515a60d2012-02-16 00:57:37 +00003708// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3709// optimizer it can aggressively ignore unwind edges.
3710void
3711CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
3712 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3713 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
3714 Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
3715 CGM.getNoObjCARCExceptionsMetadata());
3716}
3717
John McCall882987f2013-02-28 19:01:20 +00003718/// Emits a call to the given no-arguments nounwind runtime function.
3719llvm::CallInst *
James Y Knight9871db02019-02-05 16:42:33 +00003720CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
John McCall882987f2013-02-28 19:01:20 +00003721 const llvm::Twine &name) {
Craig Topper5fc8fc22014-08-27 06:28:36 +00003722 return EmitNounwindRuntimeCall(callee, None, name);
John McCall882987f2013-02-28 19:01:20 +00003723}
3724
3725/// Emits a call to the given nounwind runtime function.
3726llvm::CallInst *
James Y Knight9871db02019-02-05 16:42:33 +00003727CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
3728 ArrayRef<llvm::Value *> args,
John McCall882987f2013-02-28 19:01:20 +00003729 const llvm::Twine &name) {
3730 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
3731 call->setDoesNotThrow();
3732 return call;
3733}
3734
3735/// Emits a simple call (never an invoke) to the given no-arguments
3736/// runtime function.
James Y Knight9871db02019-02-05 16:42:33 +00003737llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3738 const llvm::Twine &name) {
Craig Topper5fc8fc22014-08-27 06:28:36 +00003739 return EmitRuntimeCall(callee, None, name);
John McCall882987f2013-02-28 19:01:20 +00003740}
3741
David Majnemer0b17d442015-12-15 21:27:59 +00003742// Calls which may throw must have operand bundles indicating which funclet
3743// they are nested within.
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003744SmallVector<llvm::OperandBundleDef, 1>
3745CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
3746 SmallVector<llvm::OperandBundleDef, 1> BundleList;
Sanjay Patel846b63b2016-01-18 22:15:33 +00003747 // There is no need for a funclet operand bundle if we aren't inside a
3748 // funclet.
David Majnemer0b17d442015-12-15 21:27:59 +00003749 if (!CurrentFuncletPad)
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003750 return BundleList;
David Majnemer0b17d442015-12-15 21:27:59 +00003751
3752 // Skip intrinsics which cannot throw.
3753 auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
3754 if (CalleeFn && CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow())
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003755 return BundleList;
David Majnemer0b17d442015-12-15 21:27:59 +00003756
3757 BundleList.emplace_back("funclet", CurrentFuncletPad);
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003758 return BundleList;
David Majnemer0b17d442015-12-15 21:27:59 +00003759}
3760
David Majnemer971d31b2016-02-24 17:02:45 +00003761/// Emits a simple call (never an invoke) to the given runtime function.
James Y Knight9871db02019-02-05 16:42:33 +00003762llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3763 ArrayRef<llvm::Value *> args,
3764 const llvm::Twine &name) {
3765 llvm::CallInst *call = Builder.CreateCall(
3766 callee, args, getBundlesForFunclet(callee.getCallee()), name);
David Majnemer971d31b2016-02-24 17:02:45 +00003767 call->setCallingConv(getRuntimeCC());
3768 return call;
3769}
3770
John McCall882987f2013-02-28 19:01:20 +00003771/// Emits a call or invoke to the given noreturn runtime function.
James Y Knight9871db02019-02-05 16:42:33 +00003772void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
3773 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003774 SmallVector<llvm::OperandBundleDef, 1> BundleList =
James Y Knight9871db02019-02-05 16:42:33 +00003775 getBundlesForFunclet(callee.getCallee());
David Majnemer0b17d442015-12-15 21:27:59 +00003776
John McCall882987f2013-02-28 19:01:20 +00003777 if (getInvokeDest()) {
Fangrui Song6907ce22018-07-30 19:24:48 +00003778 llvm::InvokeInst *invoke =
John McCall882987f2013-02-28 19:01:20 +00003779 Builder.CreateInvoke(callee,
3780 getUnreachableBlock(),
3781 getInvokeDest(),
David Majnemer0b17d442015-12-15 21:27:59 +00003782 args,
3783 BundleList);
John McCall882987f2013-02-28 19:01:20 +00003784 invoke->setDoesNotReturn();
3785 invoke->setCallingConv(getRuntimeCC());
3786 } else {
David Majnemer0b17d442015-12-15 21:27:59 +00003787 llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
John McCall882987f2013-02-28 19:01:20 +00003788 call->setDoesNotReturn();
3789 call->setCallingConv(getRuntimeCC());
3790 Builder.CreateUnreachable();
3791 }
3792}
3793
Sanjay Patel846b63b2016-01-18 22:15:33 +00003794/// Emits a call or invoke instruction to the given nullary runtime function.
James Y Knight9871db02019-02-05 16:42:33 +00003795llvm::CallBase *
3796CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3797 const Twine &name) {
Craig Topper5fc8fc22014-08-27 06:28:36 +00003798 return EmitRuntimeCallOrInvoke(callee, None, name);
John McCall882987f2013-02-28 19:01:20 +00003799}
3800
3801/// Emits a call or invoke instruction to the given runtime function.
James Y Knight9871db02019-02-05 16:42:33 +00003802llvm::CallBase *
3803CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3804 ArrayRef<llvm::Value *> args,
3805 const Twine &name) {
James Y Knight3933add2019-01-30 02:54:28 +00003806 llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
3807 call->setCallingConv(getRuntimeCC());
3808 return call;
John McCall882987f2013-02-28 19:01:20 +00003809}
3810
John McCallbd309292010-07-06 01:34:17 +00003811/// Emits a call or invoke instruction to the given function, depending
3812/// on the current state of the EH stack.
James Y Knight9871db02019-02-05 16:42:33 +00003813llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
James Y Knight3933add2019-01-30 02:54:28 +00003814 ArrayRef<llvm::Value *> Args,
3815 const Twine &Name) {
John McCallbd309292010-07-06 01:34:17 +00003816 llvm::BasicBlock *InvokeDest = getInvokeDest();
Reid Klecknerb75a3f02018-02-09 00:16:41 +00003817 SmallVector<llvm::OperandBundleDef, 1> BundleList =
James Y Knight9871db02019-02-05 16:42:33 +00003818 getBundlesForFunclet(Callee.getCallee());
John McCallbd309292010-07-06 01:34:17 +00003819
James Y Knight3933add2019-01-30 02:54:28 +00003820 llvm::CallBase *Inst;
Dan Gohman515a60d2012-02-16 00:57:37 +00003821 if (!InvokeDest)
David Majnemer3df77bc2016-01-26 23:14:47 +00003822 Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
Dan Gohman515a60d2012-02-16 00:57:37 +00003823 else {
3824 llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
David Majnemer3df77bc2016-01-26 23:14:47 +00003825 Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
3826 Name);
Dan Gohman515a60d2012-02-16 00:57:37 +00003827 EmitBlock(ContBB);
3828 }
3829
3830 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3831 // optimizer it can aggressively ignore unwind edges.
David Blaikiebbafb8a2012-03-11 07:00:24 +00003832 if (CGM.getLangOpts().ObjCAutoRefCount)
Dan Gohman515a60d2012-02-16 00:57:37 +00003833 AddObjCARCExceptionMetadata(Inst);
3834
James Y Knight3933add2019-01-30 02:54:28 +00003835 return Inst;
John McCallbd309292010-07-06 01:34:17 +00003836}
3837
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003838void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
3839 llvm::Value *New) {
3840 DeferredReplacements.push_back(std::make_pair(Old, New));
3841}
Chris Lattnerd59d8672011-07-12 06:29:11 +00003842
Roman Lebedeve819f7c2020-01-23 22:50:15 +03003843namespace {
3844
3845/// Specify given \p NewAlign as the alignment of return value attribute. If
3846/// such attribute already exists, re-set it to the maximal one of two options.
3847LLVM_NODISCARD llvm::AttributeList
3848maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
3849 const llvm::AttributeList &Attrs,
3850 llvm::Align NewAlign) {
3851 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
3852 if (CurAlign >= NewAlign)
3853 return Attrs;
3854 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
3855 return Attrs
3856 .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex,
3857 llvm::Attribute::AttrKind::Alignment)
3858 .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr);
3859}
3860
3861template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
3862protected:
3863 CodeGenFunction &CGF;
3864
3865 /// We do nothing if this is, or becomes, nullptr.
3866 const AlignedAttrTy *AA = nullptr;
3867
3868 llvm::Value *Alignment = nullptr; // May or may not be a constant.
3869 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
3870
3871 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
3872 : CGF(CGF_) {
3873 if (!FuncDecl)
3874 return;
3875 AA = FuncDecl->getAttr<AlignedAttrTy>();
3876 }
3877
3878public:
3879 /// If we can, materialize the alignment as an attribute on return value.
3880 LLVM_NODISCARD llvm::AttributeList
3881 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
3882 if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))
3883 return Attrs;
3884 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
3885 if (!AlignmentCI)
3886 return Attrs;
3887 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
3888 CGF.getLLVMContext(), Attrs,
3889 llvm::Align(
3890 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
3891 AA = nullptr; // We're done. Disallow doing anything else.
3892 return NewAttrs;
3893 }
3894
3895 /// Emit alignment assumption.
3896 /// This is a general fallback that we take if either there is an offset,
3897 /// or the alignment is variable or we are sanitizing for alignment.
3898 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
3899 if (!AA)
3900 return;
3901 CGF.EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
3902 AA->getLocation(), Alignment, OffsetCI);
3903 AA = nullptr; // We're done. Disallow doing anything else.
3904 }
3905};
3906
3907/// Helper data structure to emit `AssumeAlignedAttr`.
3908class AssumeAlignedAttrEmitter final
3909 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
3910public:
3911 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
3912 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
3913 if (!AA)
3914 return;
3915 // It is guaranteed that the alignment/offset are constants.
3916 Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
3917 if (Expr *Offset = AA->getOffset()) {
3918 OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
3919 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
3920 OffsetCI = nullptr;
3921 }
3922 }
3923};
3924
Roman Lebedev5ffe6402020-01-23 22:50:24 +03003925/// Helper data structure to emit `AllocAlignAttr`.
3926class AllocAlignAttrEmitter final
3927 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
3928public:
3929 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
3930 const CallArgList &CallArgs)
3931 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
3932 if (!AA)
3933 return;
3934 // Alignment may or may not be a constant, and that is okay.
3935 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
3936 .getRValue(CGF)
3937 .getScalarVal();
3938 }
3939};
3940
Roman Lebedeve819f7c2020-01-23 22:50:15 +03003941} // namespace
3942
Daniel Dunbard931a872009-02-02 22:03:45 +00003943RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
John McCallb92ab1a2016-10-26 23:46:34 +00003944 const CGCallee &Callee,
Anders Carlsson61a401c2009-12-24 19:25:24 +00003945 ReturnValueSlot ReturnValue,
Daniel Dunbarcdbb5e32009-02-20 18:06:48 +00003946 const CallArgList &CallArgs,
James Y Knight3933add2019-01-30 02:54:28 +00003947 llvm::CallBase **callOrInvoke,
Vedant Kumar09b5bfd2017-12-21 00:10:25 +00003948 SourceLocation Loc) {
Mike Stump18bb9282009-05-16 07:57:57 +00003949 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
Daniel Dunbar613855c2008-09-09 23:27:19 +00003950
Peter Collingbourneea211002018-02-05 23:09:13 +00003951 assert(Callee.isOrdinary() || Callee.isVirtual());
John McCallb92ab1a2016-10-26 23:46:34 +00003952
Daniel Dunbar613855c2008-09-09 23:27:19 +00003953 // Handle struct-return functions by passing a pointer to the
3954 // location that we would like to return into.
Daniel Dunbar7633cbf2009-02-02 21:43:58 +00003955 QualType RetTy = CallInfo.getReturnType();
Daniel Dunbarb52d0772009-02-03 05:59:18 +00003956 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
Mike Stump11289f42009-09-09 15:08:12 +00003957
James Y Knightcfe8cd72019-02-07 01:15:41 +00003958 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
John McCallb92ab1a2016-10-26 23:46:34 +00003959
Amy Huang0d0334f2019-04-12 20:25:30 +00003960 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
Erich Keane36176242019-06-21 22:29:32 +00003961 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
3962 // We can only guarantee that a function is called from the correct
3963 // context/function based on the appropriate target attributes,
3964 // so only check in the case where we have both always_inline and target
3965 // since otherwise we could be making a conditional call after a check for
3966 // the proper cpu features (and it won't cause code generation issues due to
3967 // function based code generation).
3968 if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
3969 TargetDecl->hasAttr<TargetAttr>())
3970 checkTargetFeatures(Loc, FD);
Amy Huang0d0334f2019-04-12 20:25:30 +00003971
James Y Knightb92d2902019-02-05 16:05:50 +00003972#ifndef NDEBUG
3973 if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) {
3974 // For an inalloca varargs function, we don't expect CallInfo to match the
3975 // function pointer's type, because the inalloca struct a will have extra
3976 // fields in it for the varargs parameters. Code later in this function
3977 // bitcasts the function pointer to the type derived from CallInfo.
3978 //
3979 // In other cases, we assert that the types match up (until pointers stop
3980 // having pointee types).
James Y Knightcfe8cd72019-02-07 01:15:41 +00003981 llvm::Type *TypeFromVal;
3982 if (Callee.isVirtual())
3983 TypeFromVal = Callee.getVirtualFunctionType();
3984 else
3985 TypeFromVal =
3986 Callee.getFunctionPointer()->getType()->getPointerElementType();
3987 assert(IRFuncTy == TypeFromVal);
James Y Knightb92d2902019-02-05 16:05:50 +00003988 }
3989#endif
3990
John McCallb92ab1a2016-10-26 23:46:34 +00003991 // 1. Set up the arguments.
Mike Stump11289f42009-09-09 15:08:12 +00003992
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003993 // If we're using inalloca, insert the allocation after the stack save.
3994 // FIXME: Do this earlier rather than hacking it in here!
John McCall7f416cc2015-09-08 08:05:57 +00003995 Address ArgMemory = Address::invalid();
Reid Kleckner314ef7b2014-02-01 00:04:45 +00003996 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
Matt Arsenault502ad602017-04-10 22:28:02 +00003997 const llvm::DataLayout &DL = CGM.getDataLayout();
Reid Kleckner9df1d972014-04-10 01:40:15 +00003998 llvm::Instruction *IP = CallArgs.getStackBase();
3999 llvm::AllocaInst *AI;
4000 if (IP) {
4001 IP = IP->getNextNode();
Matt Arsenault502ad602017-04-10 22:28:02 +00004002 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4003 "argmem", IP);
Reid Kleckner9df1d972014-04-10 01:40:15 +00004004 } else {
Reid Kleckner966abe72014-05-15 23:01:46 +00004005 AI = CreateTempAlloca(ArgStruct, "argmem");
Reid Kleckner9df1d972014-04-10 01:40:15 +00004006 }
John McCall7f416cc2015-09-08 08:05:57 +00004007 auto Align = CallInfo.getArgStructAlignment();
Guillaume Chateletc79099e2019-10-03 13:00:29 +00004008 AI->setAlignment(Align.getAsAlign());
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004009 AI->setUsedWithInAlloca(true);
4010 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
John McCall7f416cc2015-09-08 08:05:57 +00004011 ArgMemory = Address(AI, Align);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004012 }
4013
Alexey Samsonov153004f2014-09-29 22:08:00 +00004014 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004015 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4016
Chris Lattner4ca97c32009-06-13 00:26:38 +00004017 // If the call returns a temporary with struct return, create a temporary
Anders Carlsson17490832009-12-24 20:40:36 +00004018 // alloca to hold the result, unless one is given to us.
John McCall7f416cc2015-09-08 08:05:57 +00004019 Address SRetPtr = Address::invalid();
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004020 Address SRetAlloca = Address::invalid();
George Burgess IV003be7c2018-03-08 05:32:30 +00004021 llvm::Value *UnusedReturnSizePtr = nullptr;
John McCallf26e73d2016-03-11 04:30:43 +00004022 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
John McCall7f416cc2015-09-08 08:05:57 +00004023 if (!ReturnValue.isNull()) {
4024 SRetPtr = ReturnValue.getValue();
4025 } else {
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004026 SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
Leny Kholodov6aab1112015-06-08 10:23:49 +00004027 if (HaveInsertPoint() && ReturnValue.isUnused()) {
4028 uint64_t size =
4029 CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004030 UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
Leny Kholodov6aab1112015-06-08 10:23:49 +00004031 }
4032 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004033 if (IRFunctionArgs.hasSRetArg()) {
John McCall7f416cc2015-09-08 08:05:57 +00004034 IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
John McCallf26e73d2016-03-11 04:30:43 +00004035 } else if (RetAI.isInAlloca()) {
James Y Knight751fe282019-02-09 22:22:28 +00004036 Address Addr =
4037 Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
John McCall7f416cc2015-09-08 08:05:57 +00004038 Builder.CreateStore(SRetPtr.getPointer(), Addr);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004039 }
Anders Carlsson17490832009-12-24 20:40:36 +00004040 }
Mike Stump11289f42009-09-09 15:08:12 +00004041
John McCall12f23522016-04-04 18:33:08 +00004042 Address swiftErrorTemp = Address::invalid();
4043 Address swiftErrorArg = Address::invalid();
4044
Francis Visoiu Mistrih143f6b82019-10-08 22:10:38 +00004045 // When passing arguments using temporary allocas, we need to add the
4046 // appropriate lifetime markers. This vector keeps track of all the lifetime
4047 // markers that need to be ended right after the call.
4048 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4049
John McCallb92ab1a2016-10-26 23:46:34 +00004050 // Translate all of the arguments as necessary to match the IR lowering.
Daniel Dunbara45bdbb2009-02-04 21:17:21 +00004051 assert(CallInfo.arg_size() == CallArgs.size() &&
4052 "Mismatch between function signature & arguments.");
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004053 unsigned ArgNo = 0;
Daniel Dunbarb52d0772009-02-03 05:59:18 +00004054 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
Mike Stump11289f42009-09-09 15:08:12 +00004055 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004056 I != E; ++I, ++info_it, ++ArgNo) {
Daniel Dunbarb52d0772009-02-03 05:59:18 +00004057 const ABIArgInfo &ArgInfo = info_it->info;
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00004058
Rafael Espindolafad28de2012-10-24 01:59:00 +00004059 // Insert a padding argument to ensure proper alignment.
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004060 if (IRFunctionArgs.hasPaddingArg(ArgNo))
4061 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4062 llvm::UndefValue::get(ArgInfo.getPaddingType());
4063
4064 unsigned FirstIRArg, NumIRArgs;
4065 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
Rafael Espindolafad28de2012-10-24 01:59:00 +00004066
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00004067 switch (ArgInfo.getKind()) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004068 case ABIArgInfo::InAlloca: {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004069 assert(NumIRArgs == 0);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004070 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
Yaxun Liu5b330e82018-03-15 15:25:19 +00004071 if (I->isAggregate()) {
Hans Wennborgeaabaf72020-01-28 22:17:31 +01004072 // Replace the placeholder with the appropriate argument slot GEP.
Yaxun Liu5b330e82018-03-15 15:25:19 +00004073 Address Addr = I->hasLValue()
Akira Hatanakaf139ae32019-12-03 15:17:01 -08004074 ? I->getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00004075 : I->getKnownRValue().getAggregateAddress();
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004076 llvm::Instruction *Placeholder =
Yaxun Liu5b330e82018-03-15 15:25:19 +00004077 cast<llvm::Instruction>(Addr.getPointer());
Hans Wennborgeaabaf72020-01-28 22:17:31 +01004078 CGBuilderTy::InsertPoint IP = Builder.saveIP();
4079 Builder.SetInsertPoint(Placeholder);
4080 Addr =
Reid Kleckner2af74e22019-12-30 16:15:54 -08004081 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
Hans Wennborgeaabaf72020-01-28 22:17:31 +01004082 Builder.restoreIP(IP);
4083 deferPlaceholderReplacement(Placeholder, Addr.getPointer());
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004084 } else {
4085 // Store the RValue into the argument struct.
James Y Knight751fe282019-02-09 22:22:28 +00004086 Address Addr =
4087 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
John McCall7f416cc2015-09-08 08:05:57 +00004088 unsigned AS = Addr.getType()->getPointerAddressSpace();
David Majnemer32b57b02014-03-31 16:12:47 +00004089 llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
4090 // There are some cases where a trivial bitcast is not avoidable. The
4091 // definition of a type later in a translation unit may change it's type
4092 // from {}* to (%struct.foo*)*.
John McCall7f416cc2015-09-08 08:05:57 +00004093 if (Addr.getType() != MemType)
David Majnemer32b57b02014-03-31 16:12:47 +00004094 Addr = Builder.CreateBitCast(Addr, MemType);
Yaxun Liu5b330e82018-03-15 15:25:19 +00004095 I->copyInto(*this, Addr);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004096 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004097 break;
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004098 }
4099
Daniel Dunbar03816342010-08-21 02:24:36 +00004100 case ABIArgInfo::Indirect: {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004101 assert(NumIRArgs == 1);
Yaxun Liu5b330e82018-03-15 15:25:19 +00004102 if (!I->isAggregate()) {
Daniel Dunbar747865a2009-02-05 09:16:39 +00004103 // Make a temporary alloca to pass the argument.
Yaxun Liuaefdb8e2018-06-15 15:33:22 +00004104 Address Addr = CreateMemTempWithoutCast(
4105 I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
John McCall7f416cc2015-09-08 08:05:57 +00004106 IRCallArgs[FirstIRArg] = Addr.getPointer();
John McCall47fb9502013-03-07 21:37:08 +00004107
Yaxun Liu5b330e82018-03-15 15:25:19 +00004108 I->copyInto(*this, Addr);
Daniel Dunbar747865a2009-02-05 09:16:39 +00004109 } else {
Eli Friedmaneb7fab62011-06-14 01:37:52 +00004110 // We want to avoid creating an unnecessary temporary+copy here;
Guy Benyei3832bfd2013-03-10 12:59:00 +00004111 // however, we need one in three cases:
Eli Friedmaneb7fab62011-06-14 01:37:52 +00004112 // 1. If the argument is not byval, and we are required to copy the
4113 // source. (This case doesn't occur on any common architecture.)
4114 // 2. If the argument is byval, RV is not sufficiently aligned, and
4115 // we cannot force it to be sufficiently aligned.
Yaxun Liu5b330e82018-03-15 15:25:19 +00004116 // 3. If the argument is byval, but RV is not located in default
4117 // or alloca address space.
4118 Address Addr = I->hasLValue()
Akira Hatanakaf139ae32019-12-03 15:17:01 -08004119 ? I->getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00004120 : I->getKnownRValue().getAggregateAddress();
4121 llvm::Value *V = Addr.getPointer();
John McCall7f416cc2015-09-08 08:05:57 +00004122 CharUnits Align = ArgInfo.getIndirectAlign();
Micah Villmowdd31ca12012-10-08 16:25:52 +00004123 const llvm::DataLayout *TD = &CGM.getDataLayout();
Yaxun Liu5b330e82018-03-15 15:25:19 +00004124
4125 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4126 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4127 TD->getAllocaAddrSpace()) &&
4128 "indirect argument must be in alloca address space");
4129
4130 bool NeedCopy = false;
4131
4132 if (Addr.getAlignment() < Align &&
4133 llvm::getOrEnforceKnownAlignment(V, Align.getQuantity(), *TD) <
4134 Align.getQuantity()) {
4135 NeedCopy = true;
4136 } else if (I->hasLValue()) {
4137 auto LV = I->getKnownLValue();
4138 auto AS = LV.getAddressSpace();
Andrew Savonichev1bf1a152018-12-10 12:03:00 +00004139
Yaxun (Sam) Liu21b43882019-12-10 11:55:33 -05004140 if (!ArgInfo.getIndirectByVal() ||
4141 (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
Andrew Savonichev1bf1a152018-12-10 12:03:00 +00004142 NeedCopy = true;
4143 }
4144 if (!getLangOpts().OpenCL) {
4145 if ((ArgInfo.getIndirectByVal() &&
4146 (AS != LangAS::Default &&
4147 AS != CGM.getASTAllocaAddressSpace()))) {
4148 NeedCopy = true;
4149 }
4150 }
4151 // For OpenCL even if RV is located in default or alloca address space
4152 // we don't want to perform address space cast for it.
4153 else if ((ArgInfo.getIndirectByVal() &&
4154 Addr.getType()->getAddressSpace() != IRFuncTy->
4155 getParamType(FirstIRArg)->getPointerAddressSpace())) {
Yaxun Liu5b330e82018-03-15 15:25:19 +00004156 NeedCopy = true;
4157 }
4158 }
Andrew Savonichev1bf1a152018-12-10 12:03:00 +00004159
Yaxun Liu5b330e82018-03-15 15:25:19 +00004160 if (NeedCopy) {
Eli Friedmaneb7fab62011-06-14 01:37:52 +00004161 // Create an aligned temporary, and copy to it.
Yaxun Liuaefdb8e2018-06-15 15:33:22 +00004162 Address AI = CreateMemTempWithoutCast(
4163 I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
John McCall7f416cc2015-09-08 08:05:57 +00004164 IRCallArgs[FirstIRArg] = AI.getPointer();
Francis Visoiu Mistrih143f6b82019-10-08 22:10:38 +00004165
4166 // Emit lifetime markers for the temporary alloca.
4167 uint64_t ByvalTempElementSize =
4168 CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4169 llvm::Value *LifetimeSize =
4170 EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4171
4172 // Add cleanup code to emit the end lifetime marker after the call.
4173 if (LifetimeSize) // In case we disabled lifetime markers.
4174 CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4175
4176 // Generate the copy.
Yaxun Liu5b330e82018-03-15 15:25:19 +00004177 I->copyInto(*this, AI);
Eli Friedmaneb7fab62011-06-14 01:37:52 +00004178 } else {
4179 // Skip the extra memcpy call.
Yaxun Liu5b330e82018-03-15 15:25:19 +00004180 auto *T = V->getType()->getPointerElementType()->getPointerTo(
4181 CGM.getDataLayout().getAllocaAddrSpace());
4182 IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4183 *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4184 true);
Eli Friedmaneb7fab62011-06-14 01:37:52 +00004185 }
Daniel Dunbar747865a2009-02-05 09:16:39 +00004186 }
4187 break;
Daniel Dunbar03816342010-08-21 02:24:36 +00004188 }
Daniel Dunbar747865a2009-02-05 09:16:39 +00004189
Daniel Dunbar94a6f252009-01-26 21:26:08 +00004190 case ABIArgInfo::Ignore:
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004191 assert(NumIRArgs == 0);
Daniel Dunbar94a6f252009-01-26 21:26:08 +00004192 break;
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004193
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004194 case ABIArgInfo::Extend:
4195 case ABIArgInfo::Direct: {
4196 if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
Chris Lattner8a2f3c72010-07-30 04:02:24 +00004197 ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
4198 ArgInfo.getDirectOffset() == 0) {
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004199 assert(NumIRArgs == 1);
Chris Lattnerbb1952c2011-07-12 04:46:18 +00004200 llvm::Value *V;
Yaxun Liu5b330e82018-03-15 15:25:19 +00004201 if (!I->isAggregate())
4202 V = I->getKnownRValue().getScalarVal();
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004203 else
Yaxun Liu5b330e82018-03-15 15:25:19 +00004204 V = Builder.CreateLoad(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08004205 I->hasLValue() ? I->getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00004206 : I->getKnownRValue().getAggregateAddress());
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004207
John McCall12f23522016-04-04 18:33:08 +00004208 // Implement swifterror by copying into a new swifterror argument.
4209 // We'll write back in the normal path out of the call.
4210 if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4211 == ParameterABI::SwiftErrorResult) {
4212 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4213
4214 QualType pointeeTy = I->Ty->getPointeeType();
4215 swiftErrorArg =
4216 Address(V, getContext().getTypeAlignInChars(pointeeTy));
4217
4218 swiftErrorTemp =
4219 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4220 V = swiftErrorTemp.getPointer();
4221 cast<llvm::AllocaInst>(V)->setSwiftError(true);
4222
4223 llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4224 Builder.CreateStore(errorValue, swiftErrorTemp);
4225 }
4226
Reid Kleckner79b0fd72014-10-10 00:05:45 +00004227 // We might have to widen integers, but we should never truncate.
4228 if (ArgInfo.getCoerceToType() != V->getType() &&
4229 V->getType()->isIntegerTy())
4230 V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4231
Chris Lattner3ce86682011-07-12 04:53:39 +00004232 // If the argument doesn't match, perform a bitcast to coerce it. This
4233 // can happen due to trivial type mismatches.
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004234 if (FirstIRArg < IRFuncTy->getNumParams() &&
4235 V->getType() != IRFuncTy->getParamType(FirstIRArg))
4236 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
John McCall12f23522016-04-04 18:33:08 +00004237
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004238 IRCallArgs[FirstIRArg] = V;
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004239 break;
4240 }
Daniel Dunbar94a6f252009-01-26 21:26:08 +00004241
Daniel Dunbar2f219b02009-02-03 19:12:28 +00004242 // FIXME: Avoid the conversion through memory if possible.
John McCall7f416cc2015-09-08 08:05:57 +00004243 Address Src = Address::invalid();
Yaxun Liu5b330e82018-03-15 15:25:19 +00004244 if (!I->isAggregate()) {
John McCall7f416cc2015-09-08 08:05:57 +00004245 Src = CreateMemTemp(I->Ty, "coerce");
Yaxun Liu5b330e82018-03-15 15:25:19 +00004246 I->copyInto(*this, Src);
Ulrich Weigand6e2cea62015-07-10 11:31:43 +00004247 } else {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08004248 Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00004249 : I->getKnownRValue().getAggregateAddress();
Ulrich Weigand6e2cea62015-07-10 11:31:43 +00004250 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004251
Chris Lattner8a2f3c72010-07-30 04:02:24 +00004252 // If the value is offset in memory, apply the offset now.
John McCall7f416cc2015-09-08 08:05:57 +00004253 Src = emitAddressAtOffset(*this, Src, ArgInfo);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004254
Oliver Stannard2bfdc5b2014-08-27 10:43:15 +00004255 // Fast-isel and the optimizer generally like scalar values better than
4256 // FCAs, so we flatten them if this is safe to do for this argument.
James Molloy6f244b62014-05-09 16:21:39 +00004257 llvm::StructType *STy =
4258 dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
Oliver Stannard2bfdc5b2014-08-27 10:43:15 +00004259 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
John McCall7f416cc2015-09-08 08:05:57 +00004260 llvm::Type *SrcTy = Src.getType()->getElementType();
Chandler Carrutha6399a52012-10-10 11:29:08 +00004261 uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4262 uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4263
4264 // If the source type is smaller than the destination type of the
4265 // coerce-to logic, copy the source value into a temp alloca the size
4266 // of the destination type to allow loading all of it. The bits past
4267 // the source value are left undef.
4268 if (SrcSize < DstSize) {
John McCall7f416cc2015-09-08 08:05:57 +00004269 Address TempAlloca
4270 = CreateTempAlloca(STy, Src.getAlignment(),
4271 Src.getName() + ".coerce");
4272 Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4273 Src = TempAlloca;
Chandler Carrutha6399a52012-10-10 11:29:08 +00004274 } else {
Matt Arsenault7a124f32017-08-01 20:36:57 +00004275 Src = Builder.CreateBitCast(Src,
4276 STy->getPointerTo(Src.getAddressSpace()));
Chandler Carrutha6399a52012-10-10 11:29:08 +00004277 }
4278
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004279 assert(NumIRArgs == STy->getNumElements());
Chris Lattnerceddafb2010-07-05 20:41:41 +00004280 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
James Y Knight751fe282019-02-09 22:22:28 +00004281 Address EltPtr = Builder.CreateStructGEP(Src, i);
John McCall7f416cc2015-09-08 08:05:57 +00004282 llvm::Value *LI = Builder.CreateLoad(EltPtr);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004283 IRCallArgs[FirstIRArg + i] = LI;
Chris Lattner15ec3612010-06-29 00:06:42 +00004284 }
Chris Lattner3dd716c2010-06-28 23:44:11 +00004285 } else {
Chris Lattner15ec3612010-06-29 00:06:42 +00004286 // In the simple case, just pass the coerced loaded value.
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004287 assert(NumIRArgs == 1);
4288 IRCallArgs[FirstIRArg] =
John McCall7f416cc2015-09-08 08:05:57 +00004289 CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
Chris Lattner3dd716c2010-06-28 23:44:11 +00004290 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004291
Daniel Dunbar2f219b02009-02-03 19:12:28 +00004292 break;
4293 }
4294
John McCallf26e73d2016-03-11 04:30:43 +00004295 case ABIArgInfo::CoerceAndExpand: {
John McCallf26e73d2016-03-11 04:30:43 +00004296 auto coercionType = ArgInfo.getCoerceAndExpandType();
4297 auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4298
John McCall12f23522016-04-04 18:33:08 +00004299 llvm::Value *tempSize = nullptr;
4300 Address addr = Address::invalid();
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004301 Address AllocaAddr = Address::invalid();
Yaxun Liu5b330e82018-03-15 15:25:19 +00004302 if (I->isAggregate()) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08004303 addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
Yaxun Liu5b330e82018-03-15 15:25:19 +00004304 : I->getKnownRValue().getAggregateAddress();
4305
John McCall12f23522016-04-04 18:33:08 +00004306 } else {
Yaxun Liu5b330e82018-03-15 15:25:19 +00004307 RValue RV = I->getKnownRValue();
John McCall12f23522016-04-04 18:33:08 +00004308 assert(RV.isScalar()); // complex should always just be direct
4309
4310 llvm::Type *scalarType = RV.getScalarVal()->getType();
4311 auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
4312 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
4313
John McCall12f23522016-04-04 18:33:08 +00004314 // Materialize to a temporary.
Guillaume Chateletf7caf3c2019-09-23 09:04:12 +00004315 addr = CreateTempAlloca(
4316 RV.getScalarVal()->getType(),
4317 CharUnits::fromQuantity(std::max(
4318 (unsigned)layout->getAlignment().value(), scalarAlign)),
4319 "tmp",
4320 /*ArraySize=*/nullptr, &AllocaAddr);
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004321 tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
John McCall12f23522016-04-04 18:33:08 +00004322
4323 Builder.CreateStore(RV.getScalarVal(), addr);
4324 }
4325
John McCallf26e73d2016-03-11 04:30:43 +00004326 addr = Builder.CreateElementBitCast(addr, coercionType);
4327
4328 unsigned IRArgPos = FirstIRArg;
4329 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4330 llvm::Type *eltType = coercionType->getElementType(i);
4331 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
James Y Knight751fe282019-02-09 22:22:28 +00004332 Address eltAddr = Builder.CreateStructGEP(addr, i);
John McCallf26e73d2016-03-11 04:30:43 +00004333 llvm::Value *elt = Builder.CreateLoad(eltAddr);
4334 IRCallArgs[IRArgPos++] = elt;
4335 }
4336 assert(IRArgPos == FirstIRArg + NumIRArgs);
4337
John McCall12f23522016-04-04 18:33:08 +00004338 if (tempSize) {
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004339 EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
John McCall12f23522016-04-04 18:33:08 +00004340 }
4341
John McCallf26e73d2016-03-11 04:30:43 +00004342 break;
4343 }
4344
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00004345 case ABIArgInfo::Expand:
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004346 unsigned IRArgPos = FirstIRArg;
Yaxun Liu5b330e82018-03-15 15:25:19 +00004347 ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004348 assert(IRArgPos == FirstIRArg + NumIRArgs);
Daniel Dunbar8fc81b02008-09-17 00:51:38 +00004349 break;
Daniel Dunbar613855c2008-09-09 23:27:19 +00004350 }
4351 }
Mike Stump11289f42009-09-09 15:08:12 +00004352
John McCall9831b842018-02-06 18:52:44 +00004353 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
4354 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
John McCallb92ab1a2016-10-26 23:46:34 +00004355
4356 // If we're using inalloca, set up that argument.
John McCall7f416cc2015-09-08 08:05:57 +00004357 if (ArgMemory.isValid()) {
4358 llvm::Value *Arg = ArgMemory.getPointer();
Reid Klecknerafba553e2014-07-08 02:24:27 +00004359 if (CallInfo.isVariadic()) {
4360 // When passing non-POD arguments by value to variadic functions, we will
4361 // end up with a variadic prototype and an inalloca call site. In such
4362 // cases, we can't do any parameter mismatch checks. Give up and bitcast
4363 // the callee.
John McCallb92ab1a2016-10-26 23:46:34 +00004364 unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
James Y Knightcfe8cd72019-02-07 01:15:41 +00004365 CalleePtr =
4366 Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
Reid Klecknerafba553e2014-07-08 02:24:27 +00004367 } else {
4368 llvm::Type *LastParamTy =
4369 IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
4370 if (Arg->getType() != LastParamTy) {
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004371#ifndef NDEBUG
Reid Klecknerafba553e2014-07-08 02:24:27 +00004372 // Assert that these structs have equivalent element types.
4373 llvm::StructType *FullTy = CallInfo.getArgStruct();
4374 llvm::StructType *DeclaredTy = cast<llvm::StructType>(
4375 cast<llvm::PointerType>(LastParamTy)->getElementType());
4376 assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
4377 for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
4378 DE = DeclaredTy->element_end(),
4379 FI = FullTy->element_begin();
4380 DI != DE; ++DI, ++FI)
4381 assert(*DI == *FI);
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004382#endif
Reid Klecknerafba553e2014-07-08 02:24:27 +00004383 Arg = Builder.CreateBitCast(Arg, LastParamTy);
4384 }
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004385 }
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004386 assert(IRFunctionArgs.hasInallocaArg());
4387 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
Reid Kleckner314ef7b2014-02-01 00:04:45 +00004388 }
4389
John McCallb92ab1a2016-10-26 23:46:34 +00004390 // 2. Prepare the function pointer.
4391
4392 // If the callee is a bitcast of a non-variadic function to have a
4393 // variadic function pointer type, check to see if we can remove the
4394 // bitcast. This comes up with unprototyped functions.
4395 //
4396 // This makes the IR nicer, but more importantly it ensures that we
4397 // can inline the function at -O0 if it is marked always_inline.
James Y Knightcfe8cd72019-02-07 01:15:41 +00004398 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
4399 llvm::Value *Ptr) -> llvm::Function * {
John McCallb92ab1a2016-10-26 23:46:34 +00004400 if (!CalleeFT->isVarArg())
James Y Knightcfe8cd72019-02-07 01:15:41 +00004401 return nullptr;
John McCallb92ab1a2016-10-26 23:46:34 +00004402
James Y Knightcfe8cd72019-02-07 01:15:41 +00004403 // Get underlying value if it's a bitcast
4404 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
4405 if (CE->getOpcode() == llvm::Instruction::BitCast)
4406 Ptr = CE->getOperand(0);
4407 }
John McCallb92ab1a2016-10-26 23:46:34 +00004408
James Y Knightcfe8cd72019-02-07 01:15:41 +00004409 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
John McCallb92ab1a2016-10-26 23:46:34 +00004410 if (!OrigFn)
James Y Knightcfe8cd72019-02-07 01:15:41 +00004411 return nullptr;
John McCallb92ab1a2016-10-26 23:46:34 +00004412
4413 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
4414
4415 // If the original type is variadic, or if any of the component types
4416 // disagree, we cannot remove the cast.
4417 if (OrigFT->isVarArg() ||
4418 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
4419 OrigFT->getReturnType() != CalleeFT->getReturnType())
James Y Knightcfe8cd72019-02-07 01:15:41 +00004420 return nullptr;
John McCallb92ab1a2016-10-26 23:46:34 +00004421
4422 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
4423 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
James Y Knightcfe8cd72019-02-07 01:15:41 +00004424 return nullptr;
John McCallb92ab1a2016-10-26 23:46:34 +00004425
4426 return OrigFn;
4427 };
James Y Knightcfe8cd72019-02-07 01:15:41 +00004428
4429 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
4430 CalleePtr = OrigFn;
4431 IRFuncTy = OrigFn->getFunctionType();
4432 }
John McCallb92ab1a2016-10-26 23:46:34 +00004433
4434 // 3. Perform the actual call.
4435
4436 // Deactivate any cleanups that we're supposed to do immediately before
4437 // the call.
Reid Kleckner23f4c4b2013-06-21 12:45:15 +00004438 if (!CallArgs.getCleanupsToDeactivate().empty())
4439 deactivateArgCleanupsBeforeCall(*this, CallArgs);
4440
John McCallb92ab1a2016-10-26 23:46:34 +00004441 // Assert that the arguments we computed match up. The IR verifier
4442 // will catch this, but this is a common enough source of problems
4443 // during IRGen changes that it's way better for debugging to catch
4444 // it ourselves here.
4445#ifndef NDEBUG
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004446 assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
4447 for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
4448 // Inalloca argument can have different type.
4449 if (IRFunctionArgs.hasInallocaArg() &&
4450 i == IRFunctionArgs.getInallocaArgNo())
4451 continue;
4452 if (i < IRFuncTy->getNumParams())
4453 assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
4454 }
John McCallb92ab1a2016-10-26 23:46:34 +00004455#endif
Alexey Samsonov91cf4552014-08-22 01:06:06 +00004456
Craig Topper3113ec32018-10-24 17:42:17 +00004457 // Update the largest vector width if any arguments have vector types.
4458 for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
4459 if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
Graham Hunterb3025612019-10-08 12:53:54 +00004460 LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
4461 VT->getPrimitiveSizeInBits().getFixedSize());
Craig Topper3113ec32018-10-24 17:42:17 +00004462 }
4463
John McCallb92ab1a2016-10-26 23:46:34 +00004464 // Compute the calling convention and attributes.
Daniel Dunbar0ef34792009-09-12 00:59:20 +00004465 unsigned CallingConv;
Reid Klecknercdd26792017-04-18 23:50:03 +00004466 llvm::AttributeList Attrs;
John McCallb92ab1a2016-10-26 23:46:34 +00004467 CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
Reid Klecknercdd26792017-04-18 23:50:03 +00004468 Callee.getAbstractInfo(), Attrs, CallingConv,
Chad Rosier7dbc9cf2016-01-06 14:35:46 +00004469 /*AttrOnCallSite=*/true);
Mike Stump11289f42009-09-09 15:08:12 +00004470
Melanie Blower7f9b5132019-12-04 12:23:46 -08004471 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4472 if (FD->usesFPIntrin())
4473 // All calls within a strictfp function are marked strictfp
4474 Attrs =
4475 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4476 llvm::Attribute::StrictFP);
4477
John McCallb92ab1a2016-10-26 23:46:34 +00004478 // Apply some call-site-specific attributes.
4479 // TODO: work this into building the attribute set.
4480
4481 // Apply always_inline to all calls within flatten functions.
4482 // FIXME: should this really take priority over __try, below?
4483 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
Amy Huang0d0334f2019-04-12 20:25:30 +00004484 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
John McCallb92ab1a2016-10-26 23:46:34 +00004485 Attrs =
Reid Klecknerde864822017-03-21 16:57:30 +00004486 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
John McCallb92ab1a2016-10-26 23:46:34 +00004487 llvm::Attribute::AlwaysInline);
4488 }
4489
4490 // Disable inlining inside SEH __try blocks.
4491 if (isSEHTryScope()) {
4492 Attrs =
Reid Klecknerde864822017-03-21 16:57:30 +00004493 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
John McCallb92ab1a2016-10-26 23:46:34 +00004494 llvm::Attribute::NoInline);
4495 }
4496
4497 // Decide whether to use a call or an invoke.
David Majnemer4e52d6f2015-12-12 05:39:21 +00004498 bool CannotThrow;
4499 if (currentFunctionUsesSEHTry()) {
John McCallb92ab1a2016-10-26 23:46:34 +00004500 // SEH cares about asynchronous exceptions, so everything can "throw."
David Majnemer4e52d6f2015-12-12 05:39:21 +00004501 CannotThrow = false;
4502 } else if (isCleanupPadScope() &&
4503 EHPersonality::get(*this).isMSVCXXPersonality()) {
4504 // The MSVC++ personality will implicitly terminate the program if an
John McCallb92ab1a2016-10-26 23:46:34 +00004505 // exception is thrown during a cleanup outside of a try/catch.
4506 // We don't need to model anything in IR to get this behavior.
David Majnemer4e52d6f2015-12-12 05:39:21 +00004507 CannotThrow = true;
4508 } else {
John McCallb92ab1a2016-10-26 23:46:34 +00004509 // Otherwise, nounwind call sites will never throw.
Reid Klecknerde864822017-03-21 16:57:30 +00004510 CannotThrow = Attrs.hasAttribute(llvm::AttributeList::FunctionIndex,
David Majnemer4e52d6f2015-12-12 05:39:21 +00004511 llvm::Attribute::NoUnwind);
4512 }
George Burgess IV003be7c2018-03-08 05:32:30 +00004513
4514 // If we made a temporary, be sure to clean up after ourselves. Note that we
4515 // can't depend on being inside of an ExprWithCleanups, so we need to manually
4516 // pop this cleanup later on. Being eager about this is OK, since this
4517 // temporary is 'invisible' outside of the callee.
4518 if (UnusedReturnSizePtr)
Yaxun Liua2a9cfa2018-05-17 11:16:35 +00004519 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
George Burgess IV003be7c2018-03-08 05:32:30 +00004520 UnusedReturnSizePtr);
4521
David Majnemer4e52d6f2015-12-12 05:39:21 +00004522 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
John McCallbd309292010-07-06 01:34:17 +00004523
Reid Klecknerb75a3f02018-02-09 00:16:41 +00004524 SmallVector<llvm::OperandBundleDef, 1> BundleList =
4525 getBundlesForFunclet(CalleePtr);
David Majnemer0b17d442015-12-15 21:27:59 +00004526
Melanie Blower7f9b5132019-12-04 12:23:46 -08004527 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4528 if (FD->usesFPIntrin())
4529 // All calls within a strictfp function are marked strictfp
4530 Attrs =
4531 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4532 llvm::Attribute::StrictFP);
4533
Roman Lebedeve819f7c2020-01-23 22:50:15 +03004534 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
4535 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4536
Roman Lebedev5ffe6402020-01-23 22:50:24 +03004537 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
4538 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4539
John McCallb92ab1a2016-10-26 23:46:34 +00004540 // Emit the actual call/invoke instruction.
James Y Knight3933add2019-01-30 02:54:28 +00004541 llvm::CallBase *CI;
John McCallbd309292010-07-06 01:34:17 +00004542 if (!InvokeDest) {
James Y Knightcfe8cd72019-02-07 01:15:41 +00004543 CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
Daniel Dunbar12347492009-02-23 17:26:39 +00004544 } else {
4545 llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
James Y Knightcfe8cd72019-02-07 01:15:41 +00004546 CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
David Majnemer0b17d442015-12-15 21:27:59 +00004547 BundleList);
Daniel Dunbar12347492009-02-23 17:26:39 +00004548 EmitBlock(Cont);
Daniel Dunbar5006f4a2009-02-20 18:54:31 +00004549 }
Chris Lattnere70a0072010-06-29 16:40:28 +00004550 if (callOrInvoke)
John McCallb92ab1a2016-10-26 23:46:34 +00004551 *callOrInvoke = CI;
Daniel Dunbar5006f4a2009-02-20 18:54:31 +00004552
Andrew Paverdbdd88b72020-01-10 11:08:18 +00004553 // If this is within a function that has the guard(nocf) attribute and is an
4554 // indirect call, add the "guard_nocf" attribute to this call to indicate that
4555 // Control Flow Guard checks should not be added, even if the call is inlined.
4556 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
4557 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
4558 if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
4559 Attrs = Attrs.addAttribute(
4560 getLLVMContext(), llvm::AttributeList::FunctionIndex, "guard_nocf");
4561 }
4562 }
4563
John McCallb92ab1a2016-10-26 23:46:34 +00004564 // Apply the attributes and calling convention.
James Y Knight3933add2019-01-30 02:54:28 +00004565 CI->setAttributes(Attrs);
4566 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004567
John McCallb92ab1a2016-10-26 23:46:34 +00004568 // Apply various metadata.
4569
4570 if (!CI->getType()->isVoidTy())
4571 CI->setName("call");
4572
Craig Topper3113ec32018-10-24 17:42:17 +00004573 // Update largest vector width from the return type.
4574 if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
Graham Hunterb3025612019-10-08 12:53:54 +00004575 LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
4576 VT->getPrimitiveSizeInBits().getFixedSize());
Craig Topper3113ec32018-10-24 17:42:17 +00004577
Adam Nemet1e217bc2016-03-28 22:18:53 +00004578 // Insert instrumentation or attach profile metadata at indirect call sites.
4579 // For more details, see the comment before the definition of
4580 // IPVK_IndirectCallTarget in InstrProfData.inc.
James Y Knight3933add2019-01-30 02:54:28 +00004581 if (!CI->getCalledFunction())
Betul Buyukkurt518276a2016-01-23 22:50:44 +00004582 PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
John McCallb92ab1a2016-10-26 23:46:34 +00004583 CI, CalleePtr);
Betul Buyukkurt518276a2016-01-23 22:50:44 +00004584
Dan Gohman515a60d2012-02-16 00:57:37 +00004585 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4586 // optimizer it can aggressively ignore unwind edges.
David Blaikiebbafb8a2012-03-11 07:00:24 +00004587 if (CGM.getLangOpts().ObjCAutoRefCount)
John McCallb92ab1a2016-10-26 23:46:34 +00004588 AddObjCARCExceptionMetadata(CI);
4589
4590 // Suppress tail calls if requested.
4591 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
John McCallb92ab1a2016-10-26 23:46:34 +00004592 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
4593 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
4594 }
4595
Amy Huang0d0334f2019-04-12 20:25:30 +00004596 // Add metadata for calls to MSAllocator functions
Amy Huang0d0334f2019-04-12 20:25:30 +00004597 if (getDebugInfo() && TargetDecl &&
4598 TargetDecl->hasAttr<MSAllocatorAttr>())
4599 getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy, Loc);
4600
John McCallb92ab1a2016-10-26 23:46:34 +00004601 // 4. Finish the call.
Dan Gohman515a60d2012-02-16 00:57:37 +00004602
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004603 // If the call doesn't return, finish the basic block and clear the
John McCallb92ab1a2016-10-26 23:46:34 +00004604 // insertion point; this allows the rest of IRGen to discard
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004605 // unreachable code.
James Y Knight3933add2019-01-30 02:54:28 +00004606 if (CI->doesNotReturn()) {
George Burgess IV003be7c2018-03-08 05:32:30 +00004607 if (UnusedReturnSizePtr)
4608 PopCleanupBlock();
Leny Kholodov6aab1112015-06-08 10:23:49 +00004609
Julian Lettnerb62e9dc2019-01-24 18:04:21 +00004610 // Strip away the noreturn attribute to better diagnose unreachable UB.
Vedant Kumar09b5bfd2017-12-21 00:10:25 +00004611 if (SanOpts.has(SanitizerKind::Unreachable)) {
Julian Lettnerf82d8922019-02-02 02:05:16 +00004612 // Also remove from function since CallBase::hasFnAttr additionally checks
4613 // attributes of the called function.
James Y Knight3933add2019-01-30 02:54:28 +00004614 if (auto *F = CI->getCalledFunction())
Vedant Kumar09b5bfd2017-12-21 00:10:25 +00004615 F->removeFnAttr(llvm::Attribute::NoReturn);
James Y Knight3933add2019-01-30 02:54:28 +00004616 CI->removeAttribute(llvm::AttributeList::FunctionIndex,
4617 llvm::Attribute::NoReturn);
Julian Lettnerb6c06dc2019-02-01 02:51:00 +00004618
4619 // Avoid incompatibility with ASan which relies on the `noreturn`
4620 // attribute to insert handler calls.
Julian Lettner98b9f5b2019-02-04 23:37:50 +00004621 if (SanOpts.hasOneOf(SanitizerKind::Address |
4622 SanitizerKind::KernelAddress)) {
Julian Lettnerb6c06dc2019-02-01 02:51:00 +00004623 SanitizerScope SanScope(this);
4624 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
4625 Builder.SetInsertPoint(CI);
4626 auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
James Y Knight9871db02019-02-05 16:42:33 +00004627 llvm::FunctionCallee Fn =
4628 CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
Julian Lettnerb6c06dc2019-02-01 02:51:00 +00004629 EmitNounwindRuntimeCall(Fn);
4630 }
Vedant Kumar09b5bfd2017-12-21 00:10:25 +00004631 }
4632
4633 EmitUnreachable(Loc);
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004634 Builder.ClearInsertionPoint();
Mike Stump11289f42009-09-09 15:08:12 +00004635
Mike Stump18bb9282009-05-16 07:57:57 +00004636 // FIXME: For now, emit a dummy basic block because expr emitters in
4637 // generally are not ready to handle emitting expressions at unreachable
4638 // points.
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004639 EnsureInsertPoint();
Mike Stump11289f42009-09-09 15:08:12 +00004640
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004641 // Return a reasonable RValue.
4642 return GetUndefRValue(RetTy);
Mike Stump11289f42009-09-09 15:08:12 +00004643 }
Daniel Dunbarb960b7b2009-03-02 04:32:35 +00004644
John McCall12f23522016-04-04 18:33:08 +00004645 // Perform the swifterror writeback.
4646 if (swiftErrorTemp.isValid()) {
4647 llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
4648 Builder.CreateStore(errorResult, swiftErrorArg);
4649 }
4650
John McCallb92ab1a2016-10-26 23:46:34 +00004651 // Emit any call-associated writebacks immediately. Arguably this
4652 // should happen after any return-value munging.
John McCall31168b02011-06-15 23:02:42 +00004653 if (CallArgs.hasWritebacks())
4654 emitWritebacks(*this, CallArgs);
4655
Nico Weber8cdb3f92015-08-25 18:43:32 +00004656 // The stack cleanup for inalloca arguments has to run out of the normal
4657 // lexical order, so deactivate it and run it manually here.
4658 CallArgs.freeArgumentMemory(*this);
4659
John McCallb92ab1a2016-10-26 23:46:34 +00004660 // Extract the return value.
Hal Finkelee90a222014-09-26 05:04:30 +00004661 RValue Ret = [&] {
4662 switch (RetAI.getKind()) {
John McCallf26e73d2016-03-11 04:30:43 +00004663 case ABIArgInfo::CoerceAndExpand: {
4664 auto coercionType = RetAI.getCoerceAndExpandType();
John McCallf26e73d2016-03-11 04:30:43 +00004665
4666 Address addr = SRetPtr;
4667 addr = Builder.CreateElementBitCast(addr, coercionType);
4668
John McCall12f23522016-04-04 18:33:08 +00004669 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
4670 bool requiresExtract = isa<llvm::StructType>(CI->getType());
4671
John McCallf26e73d2016-03-11 04:30:43 +00004672 unsigned unpaddedIndex = 0;
4673 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4674 llvm::Type *eltType = coercionType->getElementType(i);
4675 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
James Y Knight751fe282019-02-09 22:22:28 +00004676 Address eltAddr = Builder.CreateStructGEP(addr, i);
John McCall12f23522016-04-04 18:33:08 +00004677 llvm::Value *elt = CI;
4678 if (requiresExtract)
4679 elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
4680 else
4681 assert(unpaddedIndex == 0);
John McCallf26e73d2016-03-11 04:30:43 +00004682 Builder.CreateStore(elt, eltAddr);
4683 }
John McCall12f23522016-04-04 18:33:08 +00004684 // FALLTHROUGH
Galina Kistanova0872d6c2017-06-03 06:30:46 +00004685 LLVM_FALLTHROUGH;
John McCall12f23522016-04-04 18:33:08 +00004686 }
4687
4688 case ABIArgInfo::InAlloca:
4689 case ABIArgInfo::Indirect: {
4690 RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
George Burgess IV003be7c2018-03-08 05:32:30 +00004691 if (UnusedReturnSizePtr)
4692 PopCleanupBlock();
John McCall12f23522016-04-04 18:33:08 +00004693 return ret;
John McCallf26e73d2016-03-11 04:30:43 +00004694 }
4695
Hal Finkelee90a222014-09-26 05:04:30 +00004696 case ABIArgInfo::Ignore:
4697 // If we are ignoring an argument that had a result, make sure to
4698 // construct the appropriate return value for our caller.
4699 return GetUndefRValue(RetTy);
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004700
Hal Finkelee90a222014-09-26 05:04:30 +00004701 case ABIArgInfo::Extend:
4702 case ABIArgInfo::Direct: {
4703 llvm::Type *RetIRTy = ConvertType(RetTy);
4704 if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
4705 switch (getEvaluationKind(RetTy)) {
4706 case TEK_Complex: {
4707 llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
4708 llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
4709 return RValue::getComplex(std::make_pair(Real, Imag));
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004710 }
Hal Finkelee90a222014-09-26 05:04:30 +00004711 case TEK_Aggregate: {
John McCall7f416cc2015-09-08 08:05:57 +00004712 Address DestPtr = ReturnValue.getValue();
Hal Finkelee90a222014-09-26 05:04:30 +00004713 bool DestIsVolatile = ReturnValue.isVolatile();
4714
John McCall7f416cc2015-09-08 08:05:57 +00004715 if (!DestPtr.isValid()) {
Hal Finkelee90a222014-09-26 05:04:30 +00004716 DestPtr = CreateMemTemp(RetTy, "agg.tmp");
4717 DestIsVolatile = false;
4718 }
John McCall7f416cc2015-09-08 08:05:57 +00004719 BuildAggStore(*this, CI, DestPtr, DestIsVolatile);
Hal Finkelee90a222014-09-26 05:04:30 +00004720 return RValue::getAggregate(DestPtr);
4721 }
4722 case TEK_Scalar: {
4723 // If the argument doesn't match, perform a bitcast to coerce it. This
4724 // can happen due to trivial type mismatches.
4725 llvm::Value *V = CI;
4726 if (V->getType() != RetIRTy)
4727 V = Builder.CreateBitCast(V, RetIRTy);
4728 return RValue::get(V);
4729 }
4730 }
4731 llvm_unreachable("bad evaluation kind");
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004732 }
Hal Finkelee90a222014-09-26 05:04:30 +00004733
John McCall7f416cc2015-09-08 08:05:57 +00004734 Address DestPtr = ReturnValue.getValue();
Hal Finkelee90a222014-09-26 05:04:30 +00004735 bool DestIsVolatile = ReturnValue.isVolatile();
4736
John McCall7f416cc2015-09-08 08:05:57 +00004737 if (!DestPtr.isValid()) {
Hal Finkelee90a222014-09-26 05:04:30 +00004738 DestPtr = CreateMemTemp(RetTy, "coerce");
4739 DestIsVolatile = false;
John McCall47fb9502013-03-07 21:37:08 +00004740 }
Hal Finkelee90a222014-09-26 05:04:30 +00004741
4742 // If the value is offset in memory, apply the offset now.
John McCall7f416cc2015-09-08 08:05:57 +00004743 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
4744 CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
Hal Finkelee90a222014-09-26 05:04:30 +00004745
4746 return convertTempToRValue(DestPtr, RetTy, SourceLocation());
Chris Lattnerfe34c1d2010-07-29 06:26:06 +00004747 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004748
Hal Finkelee90a222014-09-26 05:04:30 +00004749 case ABIArgInfo::Expand:
4750 llvm_unreachable("Invalid ABI kind for return argument");
Anders Carlsson17490832009-12-24 20:40:36 +00004751 }
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004752
Hal Finkelee90a222014-09-26 05:04:30 +00004753 llvm_unreachable("Unhandled ABIArgInfo::Kind");
4754 } ();
Michael J. Spencerf5a1fbc2010-10-19 06:39:39 +00004755
John McCallb92ab1a2016-10-26 23:46:34 +00004756 // Emit the assume_aligned check on the return value.
Hal Finkelee90a222014-09-26 05:04:30 +00004757 if (Ret.isScalar() && TargetDecl) {
Roman Lebedeve819f7c2020-01-23 22:50:15 +03004758 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
Roman Lebedev5ffe6402020-01-23 22:50:24 +03004759 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
Daniel Dunbar573884e2008-09-10 07:04:09 +00004760 }
Daniel Dunbard3674e62008-09-11 01:48:57 +00004761
Francis Visoiu Mistrih143f6b82019-10-08 22:10:38 +00004762 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
4763 // we can't use the full cleanup mechanism.
4764 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
4765 LifetimeEnd.Emit(*this, /*Flags=*/{});
4766
Hal Finkelee90a222014-09-26 05:04:30 +00004767 return Ret;
Daniel Dunbar613855c2008-09-09 23:27:19 +00004768}
Daniel Dunbar2d0746f2009-02-10 20:44:09 +00004769
John McCall9831b842018-02-06 18:52:44 +00004770CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
4771 if (isVirtual()) {
4772 const CallExpr *CE = getVirtualCallExpr();
4773 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
James Y Knightcfe8cd72019-02-07 01:15:41 +00004774 CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004775 CE ? CE->getBeginLoc() : SourceLocation());
John McCall9831b842018-02-06 18:52:44 +00004776 }
4777
4778 return *this;
4779}
4780
Daniel Dunbar2d0746f2009-02-10 20:44:09 +00004781/* VarArg handling */
4782
Charles Davisc7d5c942015-09-17 20:55:33 +00004783Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
4784 VAListAddr = VE->isMicrosoftABI()
4785 ? EmitMSVAListRef(VE->getSubExpr())
4786 : EmitVAListRef(VE->getSubExpr());
4787 QualType Ty = VE->getType();
4788 if (VE->isMicrosoftABI())
4789 return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty);
John McCall7f416cc2015-09-08 08:05:57 +00004790 return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty);
Daniel Dunbar2d0746f2009-02-10 20:44:09 +00004791}