blob: b0295ce69bfff19476b51ea1e0c14b292d64c781 [file] [log] [blame]
Hans Wennborgdcfba332015-10-06 23:40:43 +00001//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
Anders Carlsson2437cbf2009-02-12 00:39:25 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Anders Carlsson2437cbf2009-02-12 00:39:25 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit blocks.
10//
11//===----------------------------------------------------------------------===//
12
John McCallad7c5c12011-02-08 08:22:06 +000013#include "CGBlocks.h"
Akira Hatanaka9978da32018-08-10 15:09:24 +000014#include "CGCXXABI.h"
Chandler Carruth3a022472012-12-04 09:13:33 +000015#include "CGDebugInfo.h"
16#include "CGObjCRuntime.h"
Yaxun Liu10712d92017-10-04 20:32:17 +000017#include "CGOpenCLRuntime.h"
Chandler Carruth3a022472012-12-04 09:13:33 +000018#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
John McCallde0fe072017-08-15 21:42:52 +000020#include "ConstantEmitter.h"
Yaxun Liu10712d92017-10-04 20:32:17 +000021#include "TargetInfo.h"
Reid Kleckner98031782019-12-09 16:11:56 -080022#include "clang/AST/Attr.h"
Mike Stump692c6e32009-03-20 21:53:12 +000023#include "clang/AST/DeclObjC.h"
Yaxun Liu10712d92017-10-04 20:32:17 +000024#include "clang/CodeGen/ConstantInitBuilder.h"
Benjamin Kramer9e2e1c92010-03-31 15:04:05 +000025#include "llvm/ADT/SmallSet.h"
Chandler Carruthffd55512013-01-02 11:45:17 +000026#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Module.h"
Akira Hatanaka9978da32018-08-10 15:09:24 +000028#include "llvm/Support/ScopedPrinter.h"
Anders Carlsson2437cbf2009-02-12 00:39:25 +000029#include <algorithm>
Fariborz Jahanian983ae492012-11-14 17:43:08 +000030#include <cstdio>
Torok Edwindb714922009-08-24 13:25:12 +000031
Anders Carlsson2437cbf2009-02-12 00:39:25 +000032using namespace clang;
33using namespace CodeGen;
34
John McCall08ef4662011-11-10 08:15:53 +000035CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36 : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
Fariborz Jahanian23290b02012-11-01 18:32:55 +000037 HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false),
Akira Hatanaka9978da32018-08-10 15:09:24 +000038 CapturesNonExternalType(false), LocalAddress(Address::invalid()),
39 StructureType(nullptr), Block(block), DominatingIP(nullptr) {
Craig Topper8a13c412014-05-21 05:09:00 +000040
John McCall08ef4662011-11-10 08:15:53 +000041 // Skip asm prefix, if any. 'name' is usually taken directly from
42 // the mangled name of the enclosing function.
43 if (!name.empty() && name[0] == '\01')
44 name = name.substr(1);
John McCall9d42f0f2010-05-21 04:11:14 +000045}
46
John McCallf9b056b2011-03-31 08:03:29 +000047// Anchor the vtable to this translation unit.
Angel Garcia Gomez637d1e62015-10-20 13:23:58 +000048BlockByrefHelpers::~BlockByrefHelpers() {}
John McCallf9b056b2011-03-31 08:03:29 +000049
John McCall351762c2011-02-07 10:33:21 +000050/// Build the given block as a global block.
51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52 const CGBlockInfo &blockInfo,
53 llvm::Constant *blockFn);
John McCall9d42f0f2010-05-21 04:11:14 +000054
John McCall351762c2011-02-07 10:33:21 +000055/// Build the helper function to copy a block.
56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57 const CGBlockInfo &blockInfo) {
58 return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59}
60
Alp Tokerf6a24ce2013-12-05 16:25:25 +000061/// Build the helper function to dispose of a block.
John McCall351762c2011-02-07 10:33:21 +000062static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63 const CGBlockInfo &blockInfo) {
64 return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65}
66
Akira Hatanaka2ec36f02018-08-17 15:46:07 +000067namespace {
68
69/// Represents a type of copy/destroy operation that should be performed for an
70/// entity that's captured by a block.
71enum class BlockCaptureEntityKind {
72 CXXRecord, // Copy or destroy
73 ARCWeak,
74 ARCStrong,
75 NonTrivialCStruct,
76 BlockObject, // Assign or release
77 None
78};
79
80/// Represents a captured entity that requires extra operations in order for
81/// this entity to be copied or destroyed correctly.
82struct BlockCaptureManagedEntity {
83 BlockCaptureEntityKind CopyKind, DisposeKind;
84 BlockFieldFlags CopyFlags, DisposeFlags;
85 const BlockDecl::Capture *CI;
86 const CGBlockInfo::Capture *Capture;
87
88 BlockCaptureManagedEntity(BlockCaptureEntityKind CopyType,
89 BlockCaptureEntityKind DisposeType,
90 BlockFieldFlags CopyFlags,
91 BlockFieldFlags DisposeFlags,
92 const BlockDecl::Capture &CI,
93 const CGBlockInfo::Capture &Capture)
94 : CopyKind(CopyType), DisposeKind(DisposeType), CopyFlags(CopyFlags),
95 DisposeFlags(DisposeFlags), CI(&CI), Capture(&Capture) {}
96
97 bool operator<(const BlockCaptureManagedEntity &Other) const {
98 return Capture->getOffset() < Other.Capture->getOffset();
99 }
100};
101
102enum class CaptureStrKind {
103 // String for the copy helper.
104 CopyHelper,
105 // String for the dispose helper.
106 DisposeHelper,
107 // Merge the strings for the copy helper and dispose helper.
108 Merged
109};
110
111} // end anonymous namespace
112
113static void findBlockCapturedManagedEntities(
114 const CGBlockInfo &BlockInfo, const LangOptions &LangOpts,
115 SmallVectorImpl<BlockCaptureManagedEntity> &ManagedCaptures);
116
117static std::string getBlockCaptureStr(const BlockCaptureManagedEntity &E,
118 CaptureStrKind StrKind,
119 CharUnits BlockAlignment,
120 CodeGenModule &CGM);
121
122static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
123 CodeGenModule &CGM) {
124 std::string Name = "__block_descriptor_";
125 Name += llvm::to_string(BlockInfo.BlockSize.getQuantity()) + "_";
126
127 if (BlockInfo.needsCopyDisposeHelpers()) {
128 if (CGM.getLangOpts().Exceptions)
129 Name += "e";
130 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
131 Name += "a";
132 Name += llvm::to_string(BlockInfo.BlockAlign.getQuantity()) + "_";
133
134 SmallVector<BlockCaptureManagedEntity, 4> ManagedCaptures;
135 findBlockCapturedManagedEntities(BlockInfo, CGM.getContext().getLangOpts(),
136 ManagedCaptures);
137
138 for (const BlockCaptureManagedEntity &E : ManagedCaptures) {
139 Name += llvm::to_string(E.Capture->getOffset().getQuantity());
140
141 if (E.CopyKind == E.DisposeKind) {
142 // If CopyKind and DisposeKind are the same, merge the capture
143 // information.
144 assert(E.CopyKind != BlockCaptureEntityKind::None &&
145 "shouldn't see BlockCaptureManagedEntity that is None");
146 Name += getBlockCaptureStr(E, CaptureStrKind::Merged,
147 BlockInfo.BlockAlign, CGM);
148 } else {
149 // If CopyKind and DisposeKind are not the same, which can happen when
150 // either Kind is None or the captured object is a __strong block,
151 // concatenate the copy and dispose strings.
152 Name += getBlockCaptureStr(E, CaptureStrKind::CopyHelper,
153 BlockInfo.BlockAlign, CGM);
154 Name += getBlockCaptureStr(E, CaptureStrKind::DisposeHelper,
155 BlockInfo.BlockAlign, CGM);
156 }
157 }
158 Name += "_";
159 }
160
161 std::string TypeAtEncoding =
162 CGM.getContext().getObjCEncodingForBlock(BlockInfo.getBlockExpr());
Akira Hatanakac7c75742018-12-29 17:28:30 +0000163 /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms as
164 /// a separator between symbol name and symbol version.
165 std::replace(TypeAtEncoding.begin(), TypeAtEncoding.end(), '@', '\1');
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000166 Name += "e" + llvm::to_string(TypeAtEncoding.size()) + "_" + TypeAtEncoding;
167 Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, BlockInfo);
168 return Name;
169}
170
Fariborz Jahanianbf7bf292012-10-25 18:06:53 +0000171/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
172/// buildBlockDescriptor is accessed from 5th field of the Block_literal
173/// meta-data and contains stationary information about the block literal.
Alexander Kornienko2a8c18d2018-04-06 15:14:32 +0000174/// Its definition will have 4 (or optionally 6) words.
Dmitri Gribenko6c96ba22013-05-08 23:09:44 +0000175/// \code
Fariborz Jahanianbf7bf292012-10-25 18:06:53 +0000176/// struct Block_descriptor {
177/// unsigned long reserved;
178/// unsigned long size; // size of Block_literal metadata in bytes.
179/// void *copy_func_helper_decl; // optional copy helper.
Raphael Isemannb23ccec2018-12-10 12:37:46 +0000180/// void *destroy_func_decl; // optional destructor helper.
Dmitri Gribenko6c96ba22013-05-08 23:09:44 +0000181/// void *block_method_encoding_address; // @encode for block literal signature.
Fariborz Jahanianbf7bf292012-10-25 18:06:53 +0000182/// void *block_layout_info; // encoding of captured block variables.
183/// };
Dmitri Gribenko6c96ba22013-05-08 23:09:44 +0000184/// \endcode
John McCall351762c2011-02-07 10:33:21 +0000185static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
186 const CGBlockInfo &blockInfo) {
187 ASTContext &C = CGM.getContext();
188
John McCall6c9f1fdb2016-11-19 08:17:24 +0000189 llvm::IntegerType *ulong =
190 cast<llvm::IntegerType>(CGM.getTypes().ConvertType(C.UnsignedLongTy));
191 llvm::PointerType *i8p = nullptr;
Pekka Jaaskelainenab751a82014-08-14 09:37:50 +0000192 if (CGM.getLangOpts().OpenCL)
Fangrui Song6907ce22018-07-30 19:24:48 +0000193 i8p =
Pekka Jaaskelainenab751a82014-08-14 09:37:50 +0000194 llvm::Type::getInt8PtrTy(
195 CGM.getLLVMContext(), C.getTargetAddressSpace(LangAS::opencl_constant));
196 else
John McCall6c9f1fdb2016-11-19 08:17:24 +0000197 i8p = CGM.VoidPtrTy;
John McCall351762c2011-02-07 10:33:21 +0000198
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000199 std::string descName;
200
201 // If an equivalent block descriptor global variable exists, return it.
Erik Pilkingtonfa983902018-10-30 20:31:30 +0000202 if (C.getLangOpts().ObjC &&
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000203 CGM.getLangOpts().getGC() == LangOptions::NonGC) {
204 descName = getBlockDescriptorName(blockInfo, CGM);
205 if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(descName))
206 return llvm::ConstantExpr::getBitCast(desc,
207 CGM.getBlockDescriptorType());
208 }
209
210 // If there isn't an equivalent block descriptor global variable, create a new
211 // one.
John McCall23c9dc62016-11-28 22:18:27 +0000212 ConstantInitBuilder builder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +0000213 auto elements = builder.beginStruct();
Mike Stump85284ba2009-02-13 16:19:19 +0000214
215 // reserved
John McCall6c9f1fdb2016-11-19 08:17:24 +0000216 elements.addInt(ulong, 0);
Mike Stump85284ba2009-02-13 16:19:19 +0000217
218 // Size
Mike Stump2ac40a92009-02-21 20:07:44 +0000219 // FIXME: What is the right way to say this doesn't fit? We should give
220 // a user diagnostic in that case. Better fix would be to change the
221 // API to size_t.
John McCall6c9f1fdb2016-11-19 08:17:24 +0000222 elements.addInt(ulong, blockInfo.BlockSize.getQuantity());
Mike Stump85284ba2009-02-13 16:19:19 +0000223
John McCall351762c2011-02-07 10:33:21 +0000224 // Optional copy/dispose helpers.
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000225 bool hasInternalHelper = false;
Akira Hatanakadbfa4532018-07-20 17:10:32 +0000226 if (blockInfo.needsCopyDisposeHelpers()) {
Mike Stump85284ba2009-02-13 16:19:19 +0000227 // copy_func_helper_decl
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000228 llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
229 elements.add(copyHelper);
Mike Stump85284ba2009-02-13 16:19:19 +0000230
231 // destroy_func_decl
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000232 llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
233 elements.add(disposeHelper);
234
235 if (cast<llvm::Function>(copyHelper->getOperand(0))->hasInternalLinkage() ||
236 cast<llvm::Function>(disposeHelper->getOperand(0))
237 ->hasInternalLinkage())
238 hasInternalHelper = true;
Mike Stump85284ba2009-02-13 16:19:19 +0000239 }
240
John McCall351762c2011-02-07 10:33:21 +0000241 // Signature. Mandatory ObjC-style method descriptor @encode sequence.
242 std::string typeAtEncoding =
243 CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr());
John McCall6c9f1fdb2016-11-19 08:17:24 +0000244 elements.add(llvm::ConstantExpr::getBitCast(
John McCall7f416cc2015-09-08 08:05:57 +0000245 CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer(), i8p));
Fangrui Song6907ce22018-07-30 19:24:48 +0000246
John McCall351762c2011-02-07 10:33:21 +0000247 // GC layout.
Erik Pilkingtonfa983902018-10-30 20:31:30 +0000248 if (C.getLangOpts().ObjC) {
Fariborz Jahanian0c58ce92012-10-27 21:10:38 +0000249 if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
John McCall6c9f1fdb2016-11-19 08:17:24 +0000250 elements.add(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
Fariborz Jahanian0c58ce92012-10-27 21:10:38 +0000251 else
John McCall6c9f1fdb2016-11-19 08:17:24 +0000252 elements.add(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
Fariborz Jahanian0c58ce92012-10-27 21:10:38 +0000253 }
John McCall351762c2011-02-07 10:33:21 +0000254 else
John McCall6c9f1fdb2016-11-19 08:17:24 +0000255 elements.addNullPointer(i8p);
Mike Stump85284ba2009-02-13 16:19:19 +0000256
Joey Goulyddbda402016-08-10 15:57:02 +0000257 unsigned AddrSpace = 0;
258 if (C.getLangOpts().OpenCL)
259 AddrSpace = C.getTargetAddressSpace(LangAS::opencl_constant);
John McCall6c9f1fdb2016-11-19 08:17:24 +0000260
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000261 llvm::GlobalValue::LinkageTypes linkage;
262 if (descName.empty()) {
263 linkage = llvm::GlobalValue::InternalLinkage;
264 descName = "__block_descriptor_tmp";
265 } else if (hasInternalHelper) {
266 // If either the copy helper or the dispose helper has internal linkage,
267 // the block descriptor must have internal linkage too.
268 linkage = llvm::GlobalValue::InternalLinkage;
269 } else {
270 linkage = llvm::GlobalValue::LinkOnceODRLinkage;
271 }
272
John McCall351762c2011-02-07 10:33:21 +0000273 llvm::GlobalVariable *global =
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000274 elements.finishAndCreateGlobal(descName, CGM.getPointerAlign(),
275 /*constant*/ true, linkage, AddrSpace);
276
277 if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
David Chisnall17d42952019-03-31 11:22:26 +0000278 if (CGM.supportsCOMDAT())
279 global->setComdat(CGM.getModule().getOrInsertComdat(descName));
Akira Hatanaka2ec36f02018-08-17 15:46:07 +0000280 global->setVisibility(llvm::GlobalValue::HiddenVisibility);
281 global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
282 }
Mike Stump85284ba2009-02-13 16:19:19 +0000283
John McCall351762c2011-02-07 10:33:21 +0000284 return llvm::ConstantExpr::getBitCast(global, CGM.getBlockDescriptorType());
Anders Carlssoned5e69f2009-03-01 01:09:12 +0000285}
286
John McCall351762c2011-02-07 10:33:21 +0000287/*
288 Purely notional variadic template describing the layout of a block.
Anders Carlssoned5e69f2009-03-01 01:09:12 +0000289
John McCall351762c2011-02-07 10:33:21 +0000290 template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
291 struct Block_literal {
292 /// Initialized to one of:
293 /// extern void *_NSConcreteStackBlock[];
294 /// extern void *_NSConcreteGlobalBlock[];
295 ///
296 /// In theory, we could start one off malloc'ed by setting
297 /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
298 /// this isa:
299 /// extern void *_NSConcreteMallocBlock[];
300 struct objc_class *isa;
Mike Stump4446dcf2009-03-05 08:32:30 +0000301
John McCall351762c2011-02-07 10:33:21 +0000302 /// These are the flags (with corresponding bit number) that the
303 /// compiler is actually supposed to know about.
Akira Hatanakadbfa4532018-07-20 17:10:32 +0000304 /// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
John McCall351762c2011-02-07 10:33:21 +0000305 /// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
306 /// descriptor provides copy and dispose helper functions
307 /// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
308 /// object with a nontrivial destructor or copy constructor
309 /// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
310 /// as global memory
311 /// 29. BLOCK_USE_STRET - indicates that the block function
312 /// uses stret, which objc_msgSend needs to know about
313 /// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
314 /// @encoded signature string
315 /// And we're not supposed to manipulate these:
316 /// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
317 /// to malloc'ed memory
318 /// 27. BLOCK_IS_GC - indicates that the block has been moved to
319 /// to GC-allocated memory
320 /// Additionally, the bottom 16 bits are a reference count which
321 /// should be zero on the stack.
322 int flags;
David Chisnall950a9512009-11-17 19:33:30 +0000323
John McCall351762c2011-02-07 10:33:21 +0000324 /// Reserved; should be zero-initialized.
325 int reserved;
David Chisnall950a9512009-11-17 19:33:30 +0000326
John McCall351762c2011-02-07 10:33:21 +0000327 /// Function pointer generated from block literal.
328 _ResultType (*invoke)(Block_literal *, _ParamTypes...);
Mike Stump85284ba2009-02-13 16:19:19 +0000329
John McCall351762c2011-02-07 10:33:21 +0000330 /// Block description metadata generated from block literal.
331 struct Block_descriptor *block_descriptor;
John McCall3882ace2011-01-05 12:14:39 +0000332
John McCall351762c2011-02-07 10:33:21 +0000333 /// Captured values follow.
334 _CapturesTypes captures...;
335 };
336 */
David Chisnall950a9512009-11-17 19:33:30 +0000337
John McCall351762c2011-02-07 10:33:21 +0000338namespace {
339 /// A chunk of data that we actually have to capture in the block.
340 struct BlockLayoutChunk {
341 CharUnits Alignment;
342 CharUnits Size;
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000343 Qualifiers::ObjCLifetime Lifetime;
John McCall351762c2011-02-07 10:33:21 +0000344 const BlockDecl::Capture *Capture; // null for 'this'
Jay Foad7c57be32011-07-11 09:56:20 +0000345 llvm::Type *Type;
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000346 QualType FieldType;
Mike Stump85284ba2009-02-13 16:19:19 +0000347
John McCall351762c2011-02-07 10:33:21 +0000348 BlockLayoutChunk(CharUnits align, CharUnits size,
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000349 Qualifiers::ObjCLifetime lifetime,
John McCall351762c2011-02-07 10:33:21 +0000350 const BlockDecl::Capture *capture,
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000351 llvm::Type *type, QualType fieldType)
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000352 : Alignment(align), Size(size), Lifetime(lifetime),
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000353 Capture(capture), Type(type), FieldType(fieldType) {}
Mike Stump85284ba2009-02-13 16:19:19 +0000354
John McCall351762c2011-02-07 10:33:21 +0000355 /// Tell the block info that this chunk has the given field index.
John McCall7f416cc2015-09-08 08:05:57 +0000356 void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
357 if (!Capture) {
John McCall351762c2011-02-07 10:33:21 +0000358 info.CXXThisIndex = index;
John McCall7f416cc2015-09-08 08:05:57 +0000359 info.CXXThisOffset = offset;
360 } else {
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000361 auto C = CGBlockInfo::Capture::makeIndex(index, offset, FieldType);
362 info.Captures.insert({Capture->getVariable(), C});
John McCall7f416cc2015-09-08 08:05:57 +0000363 }
John McCall87fe5d52010-05-20 01:18:31 +0000364 }
John McCall351762c2011-02-07 10:33:21 +0000365 };
Mike Stumpd6ef62f2009-03-06 18:42:23 +0000366
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000367 /// Order by 1) all __strong together 2) next, all byfref together 3) next,
368 /// all __weak together. Preserve descending alignment in all situations.
John McCall351762c2011-02-07 10:33:21 +0000369 bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
John McCall7f416cc2015-09-08 08:05:57 +0000370 if (left.Alignment != right.Alignment)
371 return left.Alignment > right.Alignment;
372
373 auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
John McCall9c52b282015-09-11 22:00:51 +0000374 if (chunk.Capture && chunk.Capture->isByRef())
John McCall7f416cc2015-09-08 08:05:57 +0000375 return 1;
376 if (chunk.Lifetime == Qualifiers::OCL_Strong)
377 return 0;
378 if (chunk.Lifetime == Qualifiers::OCL_Weak)
379 return 2;
380 return 3;
381 };
382
383 return getPrefOrder(left) < getPrefOrder(right);
John McCall351762c2011-02-07 10:33:21 +0000384 }
Hans Wennborgdcfba332015-10-06 23:40:43 +0000385} // end anonymous namespace
John McCall351762c2011-02-07 10:33:21 +0000386
John McCallb0a3ecb2011-02-08 03:07:00 +0000387/// Determines if the given type is safe for constant capture in C++.
388static bool isSafeForCXXConstantCapture(QualType type) {
389 const RecordType *recordType =
390 type->getBaseElementTypeUnsafe()->getAs<RecordType>();
391
392 // Only records can be unsafe.
393 if (!recordType) return true;
394
Rafael Espindola2ae250c2014-05-09 00:08:36 +0000395 const auto *record = cast<CXXRecordDecl>(recordType->getDecl());
John McCallb0a3ecb2011-02-08 03:07:00 +0000396
397 // Maintain semantics for classes with non-trivial dtors or copy ctors.
398 if (!record->hasTrivialDestructor()) return false;
Richard Smith16488472012-11-16 00:53:38 +0000399 if (record->hasNonTrivialCopyConstructor()) return false;
John McCallb0a3ecb2011-02-08 03:07:00 +0000400
401 // Otherwise, we just have to make sure there aren't any mutable
402 // fields that might have changed since initialization.
Douglas Gregor61226d32011-05-13 01:05:07 +0000403 return !record->hasMutableFields();
John McCallb0a3ecb2011-02-08 03:07:00 +0000404}
405
John McCall351762c2011-02-07 10:33:21 +0000406/// It is illegal to modify a const object after initialization.
407/// Therefore, if a const object has a constant initializer, we don't
408/// actually need to keep storage for it in the block; we'll just
409/// rematerialize it at the start of the block function. This is
410/// acceptable because we make no promises about address stability of
411/// captured variables.
412static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
Richard Smithdafff942012-01-14 04:30:29 +0000413 CodeGenFunction *CGF,
John McCall351762c2011-02-07 10:33:21 +0000414 const VarDecl *var) {
Simon Pilgrim2c518802017-03-30 14:13:19 +0000415 // Return if this is a function parameter. We shouldn't try to
Akira Hatanaka1cfa2732016-05-02 22:29:40 +0000416 // rematerialize default arguments of function parameters.
417 if (isa<ParmVarDecl>(var))
418 return nullptr;
Akira Hatanaka3ba65352016-05-02 21:52:57 +0000419
John McCall351762c2011-02-07 10:33:21 +0000420 QualType type = var->getType();
421
422 // We can only do this if the variable is const.
Craig Topper8a13c412014-05-21 05:09:00 +0000423 if (!type.isConstQualified()) return nullptr;
John McCall351762c2011-02-07 10:33:21 +0000424
John McCallb0a3ecb2011-02-08 03:07:00 +0000425 // Furthermore, in C++ we have to worry about mutable fields:
426 // C++ [dcl.type.cv]p4:
427 // Except that any class member declared mutable can be
428 // modified, any attempt to modify a const object during its
429 // lifetime results in undefined behavior.
David Blaikiebbafb8a2012-03-11 07:00:24 +0000430 if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
Craig Topper8a13c412014-05-21 05:09:00 +0000431 return nullptr;
John McCall351762c2011-02-07 10:33:21 +0000432
433 // If the variable doesn't have any initializer (shouldn't this be
434 // invalid?), it's not clear what we should do. Maybe capture as
435 // zero?
436 const Expr *init = var->getInit();
Craig Topper8a13c412014-05-21 05:09:00 +0000437 if (!init) return nullptr;
John McCall351762c2011-02-07 10:33:21 +0000438
John McCallde0fe072017-08-15 21:42:52 +0000439 return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(*var);
John McCall351762c2011-02-07 10:33:21 +0000440}
441
442/// Get the low bit of a nonzero character count. This is the
443/// alignment of the nth byte if the 0th byte is universally aligned.
444static CharUnits getLowBit(CharUnits v) {
445 return CharUnits::fromQuantity(v.getQuantity() & (~v.getQuantity() + 1));
446}
447
448static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000449 SmallVectorImpl<llvm::Type*> &elementTypes) {
John McCall351762c2011-02-07 10:33:21 +0000450
451 assert(elementTypes.empty());
Yaxun Liu10712d92017-10-04 20:32:17 +0000452 if (CGM.getLangOpts().OpenCL) {
Sven van Haastregtda3b6322018-10-02 13:02:24 +0000453 // The header is basically 'struct { int; int; generic void *;
Yaxun Liu10712d92017-10-04 20:32:17 +0000454 // custom_fields; }'. Assert that struct is packed.
Sven van Haastregtda3b6322018-10-02 13:02:24 +0000455 auto GenericAS =
456 CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic);
457 auto GenPtrAlign =
458 CharUnits::fromQuantity(CGM.getTarget().getPointerAlign(GenericAS) / 8);
459 auto GenPtrSize =
460 CharUnits::fromQuantity(CGM.getTarget().getPointerWidth(GenericAS) / 8);
461 assert(CGM.getIntSize() <= GenPtrSize);
462 assert(CGM.getIntAlign() <= GenPtrAlign);
463 assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
Yaxun Liu10712d92017-10-04 20:32:17 +0000464 elementTypes.push_back(CGM.IntTy); /* total size */
465 elementTypes.push_back(CGM.IntTy); /* align */
Sven van Haastregtda3b6322018-10-02 13:02:24 +0000466 elementTypes.push_back(
467 CGM.getOpenCLRuntime()
468 .getGenericVoidPointerType()); /* invoke function */
469 unsigned Offset =
470 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
471 unsigned BlockAlign = GenPtrAlign.getQuantity();
Yaxun Liu10712d92017-10-04 20:32:17 +0000472 if (auto *Helper =
473 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
474 for (auto I : Helper->getCustomFieldTypes()) /* custom fields */ {
475 // TargetOpenCLBlockHelp needs to make sure the struct is packed.
476 // If necessary, add padding fields to the custom fields.
477 unsigned Align = CGM.getDataLayout().getABITypeAlignment(I);
478 if (BlockAlign < Align)
479 BlockAlign = Align;
480 assert(Offset % Align == 0);
481 Offset += CGM.getDataLayout().getTypeAllocSize(I);
482 elementTypes.push_back(I);
483 }
484 }
485 info.BlockAlign = CharUnits::fromQuantity(BlockAlign);
486 info.BlockSize = CharUnits::fromQuantity(Offset);
487 } else {
488 // The header is basically 'struct { void *; int; int; void *; void *; }'.
Alexander Kornienko2a8c18d2018-04-06 15:14:32 +0000489 // Assert that the struct is packed.
Yaxun Liu10712d92017-10-04 20:32:17 +0000490 assert(CGM.getIntSize() <= CGM.getPointerSize());
491 assert(CGM.getIntAlign() <= CGM.getPointerAlign());
492 assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
493 info.BlockAlign = CGM.getPointerAlign();
494 info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
495 elementTypes.push_back(CGM.VoidPtrTy);
496 elementTypes.push_back(CGM.IntTy);
497 elementTypes.push_back(CGM.IntTy);
498 elementTypes.push_back(CGM.VoidPtrTy);
499 elementTypes.push_back(CGM.getBlockDescriptorType());
500 }
John McCall351762c2011-02-07 10:33:21 +0000501}
502
Akira Hatanakaf1b3fc72017-02-14 06:46:55 +0000503static QualType getCaptureFieldType(const CodeGenFunction &CGF,
504 const BlockDecl::Capture &CI) {
505 const VarDecl *VD = CI.getVariable();
506
507 // If the variable is captured by an enclosing block or lambda expression,
508 // use the type of the capture field.
509 if (CGF.BlockInfo && CI.isNested())
510 return CGF.BlockInfo->getCapture(VD).fieldType();
511 if (auto *FD = CGF.LambdaCaptureFields.lookup(VD))
512 return FD->getType();
Akira Hatanaka8e57b072018-10-01 21:51:28 +0000513 // If the captured variable is a non-escaping __block variable, the field
514 // type is the reference type. If the variable is a __block variable that
515 // already has a reference type, the field type is the variable's type.
516 return VD->isNonEscapingByref() ?
517 CGF.getContext().getLValueReferenceType(VD->getType()) : VD->getType();
Akira Hatanakaf1b3fc72017-02-14 06:46:55 +0000518}
519
John McCall351762c2011-02-07 10:33:21 +0000520/// Compute the layout of the given block. Attempts to lay the block
521/// out with minimal space requirements.
Richard Smithdafff942012-01-14 04:30:29 +0000522static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
523 CGBlockInfo &info) {
John McCall351762c2011-02-07 10:33:21 +0000524 ASTContext &C = CGM.getContext();
525 const BlockDecl *block = info.getBlockDecl();
526
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000527 SmallVector<llvm::Type*, 8> elementTypes;
John McCall351762c2011-02-07 10:33:21 +0000528 initializeForBlockHeader(CGM, info, elementTypes);
Yaxun Liu10712d92017-10-04 20:32:17 +0000529 bool hasNonConstantCustomFields = false;
530 if (auto *OpenCLHelper =
531 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
532 hasNonConstantCustomFields =
533 !OpenCLHelper->areAllCustomFieldValuesConstant(info);
534 if (!block->hasCaptures() && !hasNonConstantCustomFields) {
John McCall351762c2011-02-07 10:33:21 +0000535 info.StructureType =
536 llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
537 info.CanBeGlobal = true;
538 return;
Mike Stump85284ba2009-02-13 16:19:19 +0000539 }
Erik Pilkingtonfa983902018-10-30 20:31:30 +0000540 else if (C.getLangOpts().ObjC &&
Fariborz Jahanian23290b02012-11-01 18:32:55 +0000541 CGM.getLangOpts().getGC() == LangOptions::NonGC)
542 info.HasCapturedVariableLayout = true;
Fangrui Song6907ce22018-07-30 19:24:48 +0000543
John McCall351762c2011-02-07 10:33:21 +0000544 // Collect the layout chunks.
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000545 SmallVector<BlockLayoutChunk, 16> layout;
John McCall351762c2011-02-07 10:33:21 +0000546 layout.reserve(block->capturesCXXThis() +
547 (block->capture_end() - block->capture_begin()));
548
549 CharUnits maxFieldAlign;
550
551 // First, 'this'.
552 if (block->capturesCXXThis()) {
Eli Friedmanc6036aa2013-07-12 22:05:26 +0000553 assert(CGF && CGF->CurFuncDecl && isa<CXXMethodDecl>(CGF->CurFuncDecl) &&
554 "Can't capture 'this' outside a method");
Brian Gesiak5488ab42019-01-11 01:54:53 +0000555 QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType();
John McCall351762c2011-02-07 10:33:21 +0000556
John McCall7f416cc2015-09-08 08:05:57 +0000557 // Theoretically, this could be in a different address space, so
558 // don't assume standard pointer size/align.
Jay Foad7c57be32011-07-11 09:56:20 +0000559 llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType);
John McCall351762c2011-02-07 10:33:21 +0000560 std::pair<CharUnits,CharUnits> tinfo
561 = CGM.getContext().getTypeInfoInChars(thisType);
562 maxFieldAlign = std::max(maxFieldAlign, tinfo.second);
563
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000564 layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first,
565 Qualifiers::OCL_None,
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000566 nullptr, llvmType, thisType));
John McCall351762c2011-02-07 10:33:21 +0000567 }
568
569 // Next, all the block captures.
Aaron Ballman9371dd22014-03-14 18:34:04 +0000570 for (const auto &CI : block->captures()) {
571 const VarDecl *variable = CI.getVariable();
John McCall351762c2011-02-07 10:33:21 +0000572
Akira Hatanaka8e57b072018-10-01 21:51:28 +0000573 if (CI.isEscapingByref()) {
John McCall351762c2011-02-07 10:33:21 +0000574 // We have to copy/dispose of the __block reference.
575 info.NeedsCopyDispose = true;
576
John McCall351762c2011-02-07 10:33:21 +0000577 // Just use void* instead of a pointer to the byref type.
John McCall7f416cc2015-09-08 08:05:57 +0000578 CharUnits align = CGM.getPointerAlign();
579 maxFieldAlign = std::max(maxFieldAlign, align);
John McCall351762c2011-02-07 10:33:21 +0000580
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000581 // Since a __block variable cannot be captured by lambdas, its type and
582 // the capture field type should always match.
583 assert(getCaptureFieldType(*CGF, CI) == variable->getType() &&
584 "capture type differs from the variable type");
John McCall7f416cc2015-09-08 08:05:57 +0000585 layout.push_back(BlockLayoutChunk(align, CGM.getPointerSize(),
586 Qualifiers::OCL_None, &CI,
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000587 CGM.VoidPtrTy, variable->getType()));
John McCall351762c2011-02-07 10:33:21 +0000588 continue;
589 }
590
591 // Otherwise, build a layout chunk with the size and alignment of
592 // the declaration.
Richard Smithdafff942012-01-14 04:30:29 +0000593 if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, variable)) {
John McCall351762c2011-02-07 10:33:21 +0000594 info.Captures[variable] = CGBlockInfo::Capture::makeConstant(constant);
595 continue;
596 }
597
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000598 QualType VT = getCaptureFieldType(*CGF, CI);
599
John McCall31168b02011-06-15 23:02:42 +0000600 // If we have a lifetime qualifier, honor it for capture purposes.
601 // That includes *not* copying it if it's __unsafe_unretained.
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000602 Qualifiers::ObjCLifetime lifetime = VT.getObjCLifetime();
Fariborz Jahanianc8892052013-01-17 00:25:06 +0000603 if (lifetime) {
John McCall31168b02011-06-15 23:02:42 +0000604 switch (lifetime) {
605 case Qualifiers::OCL_None: llvm_unreachable("impossible");
606 case Qualifiers::OCL_ExplicitNone:
607 case Qualifiers::OCL_Autoreleasing:
608 break;
John McCall351762c2011-02-07 10:33:21 +0000609
John McCall31168b02011-06-15 23:02:42 +0000610 case Qualifiers::OCL_Strong:
611 case Qualifiers::OCL_Weak:
612 info.NeedsCopyDispose = true;
613 }
614
615 // Block pointers require copy/dispose. So do Objective-C pointers.
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000616 } else if (VT->isObjCRetainableType()) {
John McCall00b2bbb2015-11-19 02:28:03 +0000617 // But honor the inert __unsafe_unretained qualifier, which doesn't
618 // actually make it into the type system.
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000619 if (VT->isObjCInertUnsafeUnretainedType()) {
John McCall00b2bbb2015-11-19 02:28:03 +0000620 lifetime = Qualifiers::OCL_ExplicitNone;
621 } else {
622 info.NeedsCopyDispose = true;
623 // used for mrr below.
624 lifetime = Qualifiers::OCL_Strong;
625 }
John McCall351762c2011-02-07 10:33:21 +0000626
627 // So do types that require non-trivial copy construction.
Aaron Ballman9371dd22014-03-14 18:34:04 +0000628 } else if (CI.hasCopyExpr()) {
John McCall351762c2011-02-07 10:33:21 +0000629 info.NeedsCopyDispose = true;
630 info.HasCXXObject = true;
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000631 if (!VT->getAsCXXRecordDecl()->isExternallyVisible())
Akira Hatanaka9978da32018-08-10 15:09:24 +0000632 info.CapturesNonExternalType = true;
John McCall351762c2011-02-07 10:33:21 +0000633
Akira Hatanaka7275da02018-02-28 07:15:55 +0000634 // So do C structs that require non-trivial copy construction or
635 // destruction.
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000636 } else if (VT.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct ||
637 VT.isDestructedType() == QualType::DK_nontrivial_c_struct) {
Akira Hatanaka7275da02018-02-28 07:15:55 +0000638 info.NeedsCopyDispose = true;
639
John McCall351762c2011-02-07 10:33:21 +0000640 // And so do types with destructors.
David Blaikiebbafb8a2012-03-11 07:00:24 +0000641 } else if (CGM.getLangOpts().CPlusPlus) {
Akira Hatanaka2a5e4632018-08-22 13:41:19 +0000642 if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl()) {
John McCall351762c2011-02-07 10:33:21 +0000643 if (!record->hasTrivialDestructor()) {
644 info.HasCXXObject = true;
645 info.NeedsCopyDispose = true;
Akira Hatanaka9978da32018-08-10 15:09:24 +0000646 if (!record->isExternallyVisible())
647 info.CapturesNonExternalType = true;
John McCall351762c2011-02-07 10:33:21 +0000648 }
649 }
650 }
651
Fariborz Jahanianf0cda632011-10-31 23:44:33 +0000652 CharUnits size = C.getTypeSizeInChars(VT);
Fariborz Jahanian10317ea2011-11-02 22:53:43 +0000653 CharUnits align = C.getDeclAlign(variable);
Fangrui Song6907ce22018-07-30 19:24:48 +0000654
John McCall351762c2011-02-07 10:33:21 +0000655 maxFieldAlign = std::max(maxFieldAlign, align);
656
Jay Foad7c57be32011-07-11 09:56:20 +0000657 llvm::Type *llvmType =
Fariborz Jahanianf0cda632011-10-31 23:44:33 +0000658 CGM.getTypes().ConvertTypeForMem(VT);
Fangrui Song6907ce22018-07-30 19:24:48 +0000659
Akira Hatanakad542ccf2016-09-16 00:02:06 +0000660 layout.push_back(
661 BlockLayoutChunk(align, size, lifetime, &CI, llvmType, VT));
John McCall351762c2011-02-07 10:33:21 +0000662 }
663
664 // If that was everything, we're done here.
665 if (layout.empty()) {
666 info.StructureType =
667 llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
668 info.CanBeGlobal = true;
669 return;
670 }
671
672 // Sort the layout by alignment. We have to use a stable sort here
673 // to get reproducible results. There should probably be an
674 // llvm::array_pod_stable_sort.
Fangrui Song899d1392019-04-24 14:43:05 +0000675 llvm::stable_sort(layout);
Fangrui Song6907ce22018-07-30 19:24:48 +0000676
Fariborz Jahanian4cf177e2012-12-04 17:20:57 +0000677 // Needed for blocks layout info.
678 info.BlockHeaderForcedGapOffset = info.BlockSize;
679 info.BlockHeaderForcedGapSize = CharUnits::Zero();
Fangrui Song6907ce22018-07-30 19:24:48 +0000680
John McCall351762c2011-02-07 10:33:21 +0000681 CharUnits &blockSize = info.BlockSize;
682 info.BlockAlign = std::max(maxFieldAlign, info.BlockAlign);
683
684 // Assuming that the first byte in the header is maximally aligned,
685 // get the alignment of the first byte following the header.
686 CharUnits endAlign = getLowBit(blockSize);
687
688 // If the end of the header isn't satisfactorily aligned for the
689 // maximum thing, look for things that are okay with the header-end
690 // alignment, and keep appending them until we get something that's
691 // aligned right. This algorithm is only guaranteed optimal if
692 // that condition is satisfied at some point; otherwise we can get
693 // things like:
694 // header // next byte has alignment 4
695 // something_with_size_5; // next byte has alignment 1
696 // something_with_alignment_8;
697 // which has 7 bytes of padding, as opposed to the naive solution
698 // which might have less (?).
699 if (endAlign < maxFieldAlign) {
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000700 SmallVectorImpl<BlockLayoutChunk>::iterator
John McCall351762c2011-02-07 10:33:21 +0000701 li = layout.begin() + 1, le = layout.end();
702
703 // Look for something that the header end is already
704 // satisfactorily aligned for.
705 for (; li != le && endAlign < li->Alignment; ++li)
706 ;
707
708 // If we found something that's naturally aligned for the end of
709 // the header, keep adding things...
710 if (li != le) {
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000711 SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
John McCall351762c2011-02-07 10:33:21 +0000712 for (; li != le; ++li) {
713 assert(endAlign >= li->Alignment);
714
John McCall7f416cc2015-09-08 08:05:57 +0000715 li->setIndex(info, elementTypes.size(), blockSize);
John McCall351762c2011-02-07 10:33:21 +0000716 elementTypes.push_back(li->Type);
717 blockSize += li->Size;
718 endAlign = getLowBit(blockSize);
719
720 // ...until we get to the alignment of the maximum field.
Fariborz Jahanian4cf177e2012-12-04 17:20:57 +0000721 if (endAlign >= maxFieldAlign) {
John McCall351762c2011-02-07 10:33:21 +0000722 break;
Fariborz Jahanian4cf177e2012-12-04 17:20:57 +0000723 }
John McCall351762c2011-02-07 10:33:21 +0000724 }
John McCall351762c2011-02-07 10:33:21 +0000725 // Don't re-append everything we just appended.
726 layout.erase(first, li);
727 }
728 }
729
John McCallac0350a2012-04-26 21:14:42 +0000730 assert(endAlign == getLowBit(blockSize));
Fangrui Song6907ce22018-07-30 19:24:48 +0000731
John McCall351762c2011-02-07 10:33:21 +0000732 // At this point, we just have to add padding if the end align still
733 // isn't aligned right.
734 if (endAlign < maxFieldAlign) {
Rui Ueyama83aa9792016-01-14 21:00:27 +0000735 CharUnits newBlockSize = blockSize.alignTo(maxFieldAlign);
John McCallac0350a2012-04-26 21:14:42 +0000736 CharUnits padding = newBlockSize - blockSize;
John McCall351762c2011-02-07 10:33:21 +0000737
John McCall7f416cc2015-09-08 08:05:57 +0000738 // If we haven't yet added any fields, remember that there was an
739 // initial gap; this need to go into the block layout bit map.
740 if (blockSize == info.BlockHeaderForcedGapOffset) {
741 info.BlockHeaderForcedGapSize = padding;
742 }
743
John McCalle3dc1702011-02-15 09:22:45 +0000744 elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,
745 padding.getQuantity()));
John McCallac0350a2012-04-26 21:14:42 +0000746 blockSize = newBlockSize;
John McCall1db0a2f2012-05-01 20:28:00 +0000747 endAlign = getLowBit(blockSize); // might be > maxFieldAlign
John McCall351762c2011-02-07 10:33:21 +0000748 }
749
John McCall1db0a2f2012-05-01 20:28:00 +0000750 assert(endAlign >= maxFieldAlign);
John McCallac0350a2012-04-26 21:14:42 +0000751 assert(endAlign == getLowBit(blockSize));
John McCall351762c2011-02-07 10:33:21 +0000752 // Slam everything else on now. This works because they have
753 // strictly decreasing alignment and we expect that size is always a
754 // multiple of alignment.
Chris Lattner0e62c1c2011-07-23 10:55:15 +0000755 for (SmallVectorImpl<BlockLayoutChunk>::iterator
John McCall351762c2011-02-07 10:33:21 +0000756 li = layout.begin(), le = layout.end(); li != le; ++li) {
Fariborz Jahanian9c56fc92014-08-12 15:51:49 +0000757 if (endAlign < li->Alignment) {
758 // size may not be multiple of alignment. This can only happen with
759 // an over-aligned variable. We will be adding a padding field to
760 // make the size be multiple of alignment.
761 CharUnits padding = li->Alignment - endAlign;
762 elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,
763 padding.getQuantity()));
764 blockSize += padding;
765 endAlign = getLowBit(blockSize);
766 }
John McCall351762c2011-02-07 10:33:21 +0000767 assert(endAlign >= li->Alignment);
John McCall7f416cc2015-09-08 08:05:57 +0000768 li->setIndex(info, elementTypes.size(), blockSize);
John McCall351762c2011-02-07 10:33:21 +0000769 elementTypes.push_back(li->Type);
770 blockSize += li->Size;
771 endAlign = getLowBit(blockSize);
772 }
773
774 info.StructureType =
775 llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
776}
777
John McCall08ef4662011-11-10 08:15:53 +0000778/// Enter the scope of a block. This should be run at the entrance to
779/// a full-expression so that the block's cleanups are pushed at the
780/// right place in the stack.
781static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) {
John McCall8c38d352012-04-13 18:44:05 +0000782 assert(CGF.HaveInsertPoint());
783
John McCall08ef4662011-11-10 08:15:53 +0000784 // Allocate the block info and place it at the head of the list.
785 CGBlockInfo &blockInfo =
786 *new CGBlockInfo(block, CGF.CurFn->getName());
787 blockInfo.NextBlockInfo = CGF.FirstBlockInfo;
788 CGF.FirstBlockInfo = &blockInfo;
789
790 // Compute information about the layout, etc., of this block,
791 // pushing cleanups as necessary.
Richard Smithdafff942012-01-14 04:30:29 +0000792 computeBlockInfo(CGF.CGM, &CGF, blockInfo);
John McCall08ef4662011-11-10 08:15:53 +0000793
794 // Nothing else to do if it can be global.
795 if (blockInfo.CanBeGlobal) return;
796
797 // Make the allocation for the block.
John McCall7f416cc2015-09-08 08:05:57 +0000798 blockInfo.LocalAddress = CGF.CreateTempAlloca(blockInfo.StructureType,
799 blockInfo.BlockAlign, "block");
John McCall08ef4662011-11-10 08:15:53 +0000800
801 // If there are cleanups to emit, enter them (but inactive).
802 if (!blockInfo.NeedsCopyDispose) return;
803
804 // Walk through the captures (in order) and find the ones not
805 // captured by constant.
Aaron Ballman9371dd22014-03-14 18:34:04 +0000806 for (const auto &CI : block->captures()) {
John McCall08ef4662011-11-10 08:15:53 +0000807 // Ignore __block captures; there's nothing special in the
808 // on-stack block that we need to do for them.
Aaron Ballman9371dd22014-03-14 18:34:04 +0000809 if (CI.isByRef()) continue;
John McCall08ef4662011-11-10 08:15:53 +0000810
811 // Ignore variables that are constant-captured.
Aaron Ballman9371dd22014-03-14 18:34:04 +0000812 const VarDecl *variable = CI.getVariable();
John McCall08ef4662011-11-10 08:15:53 +0000813 CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
814 if (capture.isConstant()) continue;
815
816 // Ignore objects that aren't destructed.
Akira Hatanakaf1b3fc72017-02-14 06:46:55 +0000817 QualType VT = getCaptureFieldType(CGF, CI);
818 QualType::DestructionKind dtorKind = VT.isDestructedType();
John McCall08ef4662011-11-10 08:15:53 +0000819 if (dtorKind == QualType::DK_none) continue;
820
821 CodeGenFunction::Destroyer *destroyer;
822
823 // Block captures count as local values and have imprecise semantics.
824 // They also can't be arrays, so need to worry about that.
Akira Hatanakaa6b6dcc2017-04-28 18:50:57 +0000825 //
826 // For const-qualified captures, emit clang.arc.use to ensure the captured
827 // object doesn't get released while we are still depending on its validity
828 // within the block.
Saleem Abdulrasoold95f6252017-05-05 18:39:06 +0000829 if (VT.isConstQualified() &&
830 VT.getObjCLifetime() == Qualifiers::OCL_Strong &&
831 CGF.CGM.getCodeGenOpts().OptimizationLevel != 0) {
832 assert(CGF.CGM.getLangOpts().ObjCAutoRefCount &&
833 "expected ObjC ARC to be enabled");
Akira Hatanakaa6b6dcc2017-04-28 18:50:57 +0000834 destroyer = CodeGenFunction::emitARCIntrinsicUse;
Saleem Abdulrasoold95f6252017-05-05 18:39:06 +0000835 } else if (dtorKind == QualType::DK_objc_strong_lifetime) {
Peter Collingbourne1425b452012-01-26 03:33:36 +0000836 destroyer = CodeGenFunction::destroyARCStrongImprecise;
John McCall08ef4662011-11-10 08:15:53 +0000837 } else {
Peter Collingbourne1425b452012-01-26 03:33:36 +0000838 destroyer = CGF.getDestroyer(dtorKind);
John McCall08ef4662011-11-10 08:15:53 +0000839 }
840
841 // GEP down to the address.
James Y Knight751fe282019-02-09 22:22:28 +0000842 Address addr =
843 CGF.Builder.CreateStructGEP(blockInfo.LocalAddress, capture.getIndex());
John McCall08ef4662011-11-10 08:15:53 +0000844
John McCallf4beacd2011-11-10 10:43:54 +0000845 // We can use that GEP as the dominating IP.
846 if (!blockInfo.DominatingIP)
John McCall7f416cc2015-09-08 08:05:57 +0000847 blockInfo.DominatingIP = cast<llvm::Instruction>(addr.getPointer());
John McCallf4beacd2011-11-10 10:43:54 +0000848
John McCall08ef4662011-11-10 08:15:53 +0000849 CleanupKind cleanupKind = InactiveNormalCleanup;
850 bool useArrayEHCleanup = CGF.needsEHCleanup(dtorKind);
Fangrui Song6907ce22018-07-30 19:24:48 +0000851 if (useArrayEHCleanup)
John McCall08ef4662011-11-10 08:15:53 +0000852 cleanupKind = InactiveNormalAndEHCleanup;
853
Akira Hatanakaf1b3fc72017-02-14 06:46:55 +0000854 CGF.pushDestroy(cleanupKind, addr, VT,
Peter Collingbourne1425b452012-01-26 03:33:36 +0000855 destroyer, useArrayEHCleanup);
John McCall08ef4662011-11-10 08:15:53 +0000856
857 // Remember where that cleanup was.
858 capture.setCleanup(CGF.EHStack.stable_begin());
859 }
860}
861
862/// Enter a full-expression with a non-trivial number of objects to
863/// clean up. This is in this file because, at the moment, the only
864/// kind of cleanup object is a BlockDecl*.
Bill Wendling7c44da22018-10-31 03:48:47 +0000865void CodeGenFunction::enterNonTrivialFullExpression(const FullExpr *E) {
866 if (const auto EWC = dyn_cast<ExprWithCleanups>(E)) {
867 assert(EWC->getNumObjects() != 0);
868 for (const ExprWithCleanups::CleanupObject &C : EWC->getObjects())
869 enterBlockScope(*this, C);
870 }
John McCall08ef4662011-11-10 08:15:53 +0000871}
872
873/// Find the layout for the given block in a linked list and remove it.
874static CGBlockInfo *findAndRemoveBlockInfo(CGBlockInfo **head,
875 const BlockDecl *block) {
876 while (true) {
877 assert(head && *head);
878 CGBlockInfo *cur = *head;
879
880 // If this is the block we're looking for, splice it out of the list.
881 if (cur->getBlockDecl() == block) {
882 *head = cur->NextBlockInfo;
883 return cur;
884 }
885
886 head = &cur->NextBlockInfo;
887 }
888}
889
890/// Destroy a chain of block layouts.
891void CodeGenFunction::destroyBlockInfos(CGBlockInfo *head) {
892 assert(head && "destroying an empty chain");
893 do {
894 CGBlockInfo *cur = head;
895 head = cur->NextBlockInfo;
896 delete cur;
Craig Topper8a13c412014-05-21 05:09:00 +0000897 } while (head != nullptr);
John McCall08ef4662011-11-10 08:15:53 +0000898}
899
John McCall351762c2011-02-07 10:33:21 +0000900/// Emit a block literal expression in the current function.
Yaxun Liufa13d012018-02-15 16:39:19 +0000901llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
John McCall08ef4662011-11-10 08:15:53 +0000902 // If the block has no captures, we won't have a pre-computed
903 // layout for it.
904 if (!blockExpr->getBlockDecl()->hasCaptures()) {
Yaxun Liuc2a87a02017-10-14 12:23:50 +0000905 // The block literal is emitted as a global variable, and the block invoke
906 // function has to be extracted from its initializer.
907 if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(blockExpr)) {
George Burgess IVe3763372016-12-22 02:50:20 +0000908 return Block;
Yaxun Liuc2a87a02017-10-14 12:23:50 +0000909 }
John McCall08ef4662011-11-10 08:15:53 +0000910 CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
Richard Smithdafff942012-01-14 04:30:29 +0000911 computeBlockInfo(CGM, this, blockInfo);
John McCall08ef4662011-11-10 08:15:53 +0000912 blockInfo.BlockExpression = blockExpr;
Yaxun Liufa13d012018-02-15 16:39:19 +0000913 return EmitBlockLiteral(blockInfo);
John McCall08ef4662011-11-10 08:15:53 +0000914 }
John McCall351762c2011-02-07 10:33:21 +0000915
John McCall08ef4662011-11-10 08:15:53 +0000916 // Find the block info for this block and take ownership of it.
Ahmed Charlesb8984322014-03-07 20:03:18 +0000917 std::unique_ptr<CGBlockInfo> blockInfo;
John McCall08ef4662011-11-10 08:15:53 +0000918 blockInfo.reset(findAndRemoveBlockInfo(&FirstBlockInfo,
919 blockExpr->getBlockDecl()));
John McCall351762c2011-02-07 10:33:21 +0000920
John McCall08ef4662011-11-10 08:15:53 +0000921 blockInfo->BlockExpression = blockExpr;
Yaxun Liufa13d012018-02-15 16:39:19 +0000922 return EmitBlockLiteral(*blockInfo);
John McCall08ef4662011-11-10 08:15:53 +0000923}
924
Yaxun Liufa13d012018-02-15 16:39:19 +0000925llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
Yaxun Liu10712d92017-10-04 20:32:17 +0000926 bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
Sven van Haastregtda3b6322018-10-02 13:02:24 +0000927 auto GenVoidPtrTy =
928 IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
929 LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
930 auto GenVoidPtrSize = CharUnits::fromQuantity(
931 CGM.getTarget().getPointerWidth(
932 CGM.getContext().getTargetAddressSpace(GenVoidPtrAddr)) /
933 8);
John McCall08ef4662011-11-10 08:15:53 +0000934 // Using the computed layout, generate the actual block function.
Eli Friedman98b01ed2012-03-01 04:01:32 +0000935 bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
Vedant Kumar29477dc2017-12-08 02:47:58 +0000936 CodeGenFunction BlockCGF{CGM, true};
937 BlockCGF.SanOpts = SanOpts;
938 auto *InvokeFn = BlockCGF.GenerateBlockFunction(
Yaxun Liu10712d92017-10-04 20:32:17 +0000939 CurGD, blockInfo, LocalDeclMap, isLambdaConv, blockInfo.CanBeGlobal);
Sven van Haastregtda3b6322018-10-02 13:02:24 +0000940 auto *blockFn = llvm::ConstantExpr::getPointerCast(InvokeFn, GenVoidPtrTy);
John McCall351762c2011-02-07 10:33:21 +0000941
942 // If there is nothing to capture, we can emit this as a global block.
943 if (blockInfo.CanBeGlobal)
Akira Hatanakaba0367a2017-09-22 21:32:06 +0000944 return CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression);
John McCall351762c2011-02-07 10:33:21 +0000945
946 // Otherwise, we have to emit this as a local block.
947
John McCall7f416cc2015-09-08 08:05:57 +0000948 Address blockAddr = blockInfo.LocalAddress;
949 assert(blockAddr.isValid() && "block has no address!");
John McCall351762c2011-02-07 10:33:21 +0000950
Yaxun Liu10712d92017-10-04 20:32:17 +0000951 llvm::Constant *isa;
952 llvm::Constant *descriptor;
953 BlockFlags flags;
954 if (!IsOpenCL) {
Akira Hatanakadbfa4532018-07-20 17:10:32 +0000955 // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
956 // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
957 // block just returns the original block and releasing it is a no-op.
958 llvm::Constant *blockISA = blockInfo.getBlockDecl()->doesNotEscape()
959 ? CGM.getNSConcreteGlobalBlock()
960 : CGM.getNSConcreteStackBlock();
961 isa = llvm::ConstantExpr::getBitCast(blockISA, VoidPtrTy);
Yaxun Liu10712d92017-10-04 20:32:17 +0000962
963 // Build the block descriptor.
964 descriptor = buildBlockDescriptor(CGM, blockInfo);
965
966 // Compute the initial on-stack block flags.
967 flags = BLOCK_HAS_SIGNATURE;
968 if (blockInfo.HasCapturedVariableLayout)
969 flags |= BLOCK_HAS_EXTENDED_LAYOUT;
Akira Hatanakadbfa4532018-07-20 17:10:32 +0000970 if (blockInfo.needsCopyDisposeHelpers())
Yaxun Liu10712d92017-10-04 20:32:17 +0000971 flags |= BLOCK_HAS_COPY_DISPOSE;
972 if (blockInfo.HasCXXObject)
973 flags |= BLOCK_HAS_CXX_OBJ;
974 if (blockInfo.UsesStret)
975 flags |= BLOCK_USE_STRET;
Akira Hatanakadbfa4532018-07-20 17:10:32 +0000976 if (blockInfo.getBlockDecl()->doesNotEscape())
977 flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
Yaxun Liu10712d92017-10-04 20:32:17 +0000978 }
John McCall351762c2011-02-07 10:33:21 +0000979
James Y Knight751fe282019-02-09 22:22:28 +0000980 auto projectField = [&](unsigned index, const Twine &name) -> Address {
981 return Builder.CreateStructGEP(blockAddr, index, name);
982 };
983 auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
984 Builder.CreateStore(value, projectField(index, name));
985 };
John McCall7f416cc2015-09-08 08:05:57 +0000986
987 // Initialize the block header.
988 {
989 // We assume all the header fields are densely packed.
990 unsigned index = 0;
991 CharUnits offset;
James Y Knight751fe282019-02-09 22:22:28 +0000992 auto addHeaderField = [&](llvm::Value *value, CharUnits size,
993 const Twine &name) {
994 storeField(value, index, name);
995 offset += size;
996 index++;
997 };
John McCall7f416cc2015-09-08 08:05:57 +0000998
Yaxun Liu10712d92017-10-04 20:32:17 +0000999 if (!IsOpenCL) {
1000 addHeaderField(isa, getPointerSize(), "block.isa");
1001 addHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
1002 getIntSize(), "block.flags");
1003 addHeaderField(llvm::ConstantInt::get(IntTy, 0), getIntSize(),
1004 "block.reserved");
1005 } else {
1006 addHeaderField(
1007 llvm::ConstantInt::get(IntTy, blockInfo.BlockSize.getQuantity()),
1008 getIntSize(), "block.size");
1009 addHeaderField(
1010 llvm::ConstantInt::get(IntTy, blockInfo.BlockAlign.getQuantity()),
1011 getIntSize(), "block.align");
1012 }
Sven van Haastregtda3b6322018-10-02 13:02:24 +00001013 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
1014 if (!IsOpenCL)
Yaxun Liu10712d92017-10-04 20:32:17 +00001015 addHeaderField(descriptor, getPointerSize(), "block.descriptor");
Sven van Haastregtda3b6322018-10-02 13:02:24 +00001016 else if (auto *Helper =
1017 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
Yaxun Liu10712d92017-10-04 20:32:17 +00001018 for (auto I : Helper->getCustomFieldValues(*this, blockInfo)) {
1019 addHeaderField(
1020 I.first,
1021 CharUnits::fromQuantity(
1022 CGM.getDataLayout().getTypeAllocSize(I.first->getType())),
1023 I.second);
1024 }
1025 }
John McCall7f416cc2015-09-08 08:05:57 +00001026 }
John McCall351762c2011-02-07 10:33:21 +00001027
1028 // Finally, capture all the values into the block.
1029 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1030
1031 // First, 'this'.
1032 if (blockDecl->capturesCXXThis()) {
James Y Knight751fe282019-02-09 22:22:28 +00001033 Address addr =
1034 projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
John McCall351762c2011-02-07 10:33:21 +00001035 Builder.CreateStore(LoadCXXThis(), addr);
1036 }
1037
1038 // Next, captured variables.
Aaron Ballman9371dd22014-03-14 18:34:04 +00001039 for (const auto &CI : blockDecl->captures()) {
1040 const VarDecl *variable = CI.getVariable();
John McCall351762c2011-02-07 10:33:21 +00001041 const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
1042
1043 // Ignore constant captures.
1044 if (capture.isConstant()) continue;
1045
Akira Hatanakad542ccf2016-09-16 00:02:06 +00001046 QualType type = capture.fieldType();
John McCall351762c2011-02-07 10:33:21 +00001047
1048 // This will be a [[type]]*, except that a byref entry will just be
1049 // an i8**.
James Y Knight751fe282019-02-09 22:22:28 +00001050 Address blockField = projectField(capture.getIndex(), "block.captured");
John McCall351762c2011-02-07 10:33:21 +00001051
1052 // Compute the address of the thing we're going to move into the
1053 // block literal.
John McCall7f416cc2015-09-08 08:05:57 +00001054 Address src = Address::invalid();
John McCall351762c2011-02-07 10:33:21 +00001055
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001056 if (blockDecl->isConversionFromLambda()) {
Eli Friedman2495ab02012-02-25 02:48:22 +00001057 // The lambda capture in a lambda's conversion-to-block-pointer is
Eli Friedman98b01ed2012-03-01 04:01:32 +00001058 // special; we'll simply emit it directly.
John McCall7f416cc2015-09-08 08:05:57 +00001059 src = Address::invalid();
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001060 } else if (CI.isEscapingByref()) {
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001061 if (BlockInfo && CI.isNested()) {
1062 // We need to use the capture from the enclosing block.
1063 const CGBlockInfo::Capture &enclosingCapture =
1064 BlockInfo->getCapture(variable);
1065
Alexander Kornienko2a8c18d2018-04-06 15:14:32 +00001066 // This is a [[type]]*, except that a byref entry will just be an i8**.
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001067 src = Builder.CreateStructGEP(LoadBlockStruct(),
1068 enclosingCapture.getIndex(),
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001069 "block.capture.addr");
John McCall7f416cc2015-09-08 08:05:57 +00001070 } else {
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001071 auto I = LocalDeclMap.find(variable);
1072 assert(I != LocalDeclMap.end());
1073 src = I->second;
John McCalla37c2fa2013-03-04 06:32:36 +00001074 }
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001075 } else {
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001076 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001077 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1078 type.getNonReferenceType(), VK_LValue,
1079 SourceLocation());
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001080 src = EmitDeclRefLValue(&declRef).getAddress(*this);
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001081 };
John McCall351762c2011-02-07 10:33:21 +00001082
1083 // For byrefs, we just write the pointer to the byref struct into
1084 // the block field. There's no need to chase the forwarding
1085 // pointer at this point, since we're building something that will
1086 // live a shorter life than the stack byref anyway.
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001087 if (CI.isEscapingByref()) {
John McCalle3dc1702011-02-15 09:22:45 +00001088 // Get a void* that points to the byref struct.
John McCall7f416cc2015-09-08 08:05:57 +00001089 llvm::Value *byrefPointer;
Aaron Ballman9371dd22014-03-14 18:34:04 +00001090 if (CI.isNested())
John McCall7f416cc2015-09-08 08:05:57 +00001091 byrefPointer = Builder.CreateLoad(src, "byref.capture");
John McCall351762c2011-02-07 10:33:21 +00001092 else
John McCall7f416cc2015-09-08 08:05:57 +00001093 byrefPointer = Builder.CreateBitCast(src.getPointer(), VoidPtrTy);
John McCall351762c2011-02-07 10:33:21 +00001094
John McCalle3dc1702011-02-15 09:22:45 +00001095 // Write that void* into the capture field.
John McCall7f416cc2015-09-08 08:05:57 +00001096 Builder.CreateStore(byrefPointer, blockField);
John McCall351762c2011-02-07 10:33:21 +00001097
1098 // If we have a copy constructor, evaluate that into the block field.
Aaron Ballman9371dd22014-03-14 18:34:04 +00001099 } else if (const Expr *copyExpr = CI.getCopyExpr()) {
Eli Friedman98b01ed2012-03-01 04:01:32 +00001100 if (blockDecl->isConversionFromLambda()) {
1101 // If we have a lambda conversion, emit the expression
1102 // directly into the block instead.
Eli Friedman98b01ed2012-03-01 04:01:32 +00001103 AggValueSlot Slot =
John McCall7f416cc2015-09-08 08:05:57 +00001104 AggValueSlot::forAddr(blockField, Qualifiers(),
Eli Friedman98b01ed2012-03-01 04:01:32 +00001105 AggValueSlot::IsDestructed,
1106 AggValueSlot::DoesNotNeedGCBarriers,
Richard Smithe78fac52018-04-05 20:52:58 +00001107 AggValueSlot::IsNotAliased,
1108 AggValueSlot::DoesNotOverlap);
Eli Friedman98b01ed2012-03-01 04:01:32 +00001109 EmitAggExpr(copyExpr, Slot);
1110 } else {
1111 EmitSynthesizedCXXCopyCtor(blockField, src, copyExpr);
1112 }
John McCall351762c2011-02-07 10:33:21 +00001113
1114 // If it's a reference variable, copy the reference into the block field.
Fariborz Jahanian10317ea2011-11-02 22:53:43 +00001115 } else if (type->isReferenceType()) {
Akira Hatanaka1cce6e12016-05-04 18:40:33 +00001116 Builder.CreateStore(src.getPointer(), blockField);
John McCall4d14a902013-04-08 23:27:49 +00001117
Akira Hatanakaa6b6dcc2017-04-28 18:50:57 +00001118 // If type is const-qualified, copy the value into the block field.
1119 } else if (type.isConstQualified() &&
Akira Hatanaka855d70c2017-05-09 01:20:05 +00001120 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1121 CGM.getCodeGenOpts().OptimizationLevel != 0) {
Akira Hatanakaa6b6dcc2017-04-28 18:50:57 +00001122 llvm::Value *value = Builder.CreateLoad(src, "captured");
1123 Builder.CreateStore(value, blockField);
1124
John McCall4d14a902013-04-08 23:27:49 +00001125 // If this is an ARC __strong block-pointer variable, don't do a
1126 // block copy.
1127 //
1128 // TODO: this can be generalized into the normal initialization logic:
1129 // we should never need to do a block-copy when initializing a local
1130 // variable, because the local variable's lifetime should be strictly
1131 // contained within the stack block's.
1132 } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1133 type->isBlockPointerType()) {
1134 // Load the block and do a simple retain.
John McCall7f416cc2015-09-08 08:05:57 +00001135 llvm::Value *value = Builder.CreateLoad(src, "block.captured_block");
John McCall4d14a902013-04-08 23:27:49 +00001136 value = EmitARCRetainNonBlock(value);
1137
1138 // Do a primitive store to the block field.
John McCall7f416cc2015-09-08 08:05:57 +00001139 Builder.CreateStore(value, blockField);
John McCall351762c2011-02-07 10:33:21 +00001140
1141 // Otherwise, fake up a POD copy into the block field.
1142 } else {
John McCall31168b02011-06-15 23:02:42 +00001143 // Fake up a new variable so that EmitScalarInit doesn't think
1144 // we're referring to the variable in its own initializer.
Alexey Bataev56223232017-06-09 13:40:18 +00001145 ImplicitParamDecl BlockFieldPseudoVar(getContext(), type,
1146 ImplicitParamDecl::Other);
John McCall31168b02011-06-15 23:02:42 +00001147
John McCall93be3f72011-02-07 18:37:40 +00001148 // We use one of these or the other depending on whether the
1149 // reference is nested.
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001150 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
Alexey Bataev19acc3d2015-01-12 10:17:46 +00001151 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1152 type, VK_LValue, SourceLocation());
John McCall93be3f72011-02-07 18:37:40 +00001153
Fariborz Jahanian10317ea2011-11-02 22:53:43 +00001154 ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
John McCall113bee02012-03-10 09:33:50 +00001155 &declRef, VK_RValue);
David Blaikie7f138812014-12-09 22:04:13 +00001156 // FIXME: Pass a specific location for the expr init so that the store is
1157 // attributed to a reasonable location - otherwise it may be attributed to
1158 // locations of subexpressions in the initialization.
Alexey Bataev56223232017-06-09 13:40:18 +00001159 EmitExprAsInit(&l2r, &BlockFieldPseudoVar,
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +00001160 MakeAddrLValue(blockField, type, AlignmentSource::Decl),
David Blaikie66e41972015-01-14 07:38:27 +00001161 /*captured by init*/ false);
John McCall351762c2011-02-07 10:33:21 +00001162 }
1163
John McCall08ef4662011-11-10 08:15:53 +00001164 // Activate the cleanup if layout pushed one.
Aaron Ballman9371dd22014-03-14 18:34:04 +00001165 if (!CI.isByRef()) {
John McCall08ef4662011-11-10 08:15:53 +00001166 EHScopeStack::stable_iterator cleanup = capture.getCleanup();
1167 if (cleanup.isValid())
John McCallf4beacd2011-11-10 10:43:54 +00001168 ActivateCleanupBlock(cleanup, blockInfo.DominatingIP);
John McCall31168b02011-06-15 23:02:42 +00001169 }
John McCall351762c2011-02-07 10:33:21 +00001170 }
1171
1172 // Cast to the converted block-pointer type, which happens (somewhat
1173 // unfortunately) to be a pointer to function type.
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001174 llvm::Value *result = Builder.CreatePointerCast(
1175 blockAddr.getPointer(), ConvertType(blockInfo.getBlockExpr()->getType()));
John McCall3882ace2011-01-05 12:14:39 +00001176
Yaxun Liufa13d012018-02-15 16:39:19 +00001177 if (IsOpenCL) {
1178 CGM.getOpenCLRuntime().recordBlockInfo(blockInfo.BlockExpression, InvokeFn,
1179 result);
1180 }
1181
John McCall351762c2011-02-07 10:33:21 +00001182 return result;
Mike Stump85284ba2009-02-13 16:19:19 +00001183}
1184
1185
Chris Lattnera5f58b02011-07-09 17:41:47 +00001186llvm::Type *CodeGenModule::getBlockDescriptorType() {
Mike Stump650c9322009-02-13 15:16:56 +00001187 if (BlockDescriptorType)
1188 return BlockDescriptorType;
1189
Chris Lattnera5f58b02011-07-09 17:41:47 +00001190 llvm::Type *UnsignedLongTy =
Mike Stump650c9322009-02-13 15:16:56 +00001191 getTypes().ConvertType(getContext().UnsignedLongTy);
Mike Stumpb7074c02009-02-13 15:32:32 +00001192
Mike Stump650c9322009-02-13 15:16:56 +00001193 // struct __block_descriptor {
1194 // unsigned long reserved;
1195 // unsigned long block_size;
Blaine Garstfc83aa02010-02-23 21:51:17 +00001196 //
1197 // // later, the following will be added
1198 //
1199 // struct {
1200 // void (*copyHelper)();
1201 // void (*copyHelper)();
1202 // } helpers; // !!! optional
1203 //
1204 // const char *signature; // the block signature
1205 // const char *layout; // reserved
Mike Stump650c9322009-02-13 15:16:56 +00001206 // };
Serge Guelton1d993272017-05-09 19:31:30 +00001207 BlockDescriptorType = llvm::StructType::create(
1208 "struct.__block_descriptor", UnsignedLongTy, UnsignedLongTy);
Mike Stump650c9322009-02-13 15:16:56 +00001209
John McCall351762c2011-02-07 10:33:21 +00001210 // Now form a pointer to that.
Joey Goulyddbda402016-08-10 15:57:02 +00001211 unsigned AddrSpace = 0;
1212 if (getLangOpts().OpenCL)
1213 AddrSpace = getContext().getTargetAddressSpace(LangAS::opencl_constant);
1214 BlockDescriptorType = llvm::PointerType::get(BlockDescriptorType, AddrSpace);
Mike Stump650c9322009-02-13 15:16:56 +00001215 return BlockDescriptorType;
Anders Carlsson2437cbf2009-02-12 00:39:25 +00001216}
1217
Chris Lattnera5f58b02011-07-09 17:41:47 +00001218llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
Mike Stump005c9a62009-02-13 15:25:34 +00001219 if (GenericBlockLiteralType)
1220 return GenericBlockLiteralType;
1221
Chris Lattnera5f58b02011-07-09 17:41:47 +00001222 llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
Mike Stumpb7074c02009-02-13 15:32:32 +00001223
Sven van Haastregtda3b6322018-10-02 13:02:24 +00001224 if (getLangOpts().OpenCL) {
1225 // struct __opencl_block_literal_generic {
1226 // int __size;
1227 // int __align;
1228 // __generic void *__invoke;
1229 // /* custom fields */
1230 // };
1231 SmallVector<llvm::Type *, 8> StructFields(
1232 {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1233 if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1234 for (auto I : Helper->getCustomFieldTypes())
1235 StructFields.push_back(I);
1236 }
1237 GenericBlockLiteralType = llvm::StructType::create(
1238 StructFields, "struct.__opencl_block_literal_generic");
1239 } else {
1240 // struct __block_literal_generic {
1241 // void *__isa;
1242 // int __flags;
1243 // int __reserved;
1244 // void (*__invoke)(void *);
1245 // struct __block_descriptor *__descriptor;
1246 // };
1247 GenericBlockLiteralType =
1248 llvm::StructType::create("struct.__block_literal_generic", VoidPtrTy,
1249 IntTy, IntTy, VoidPtrTy, BlockDescPtrTy);
1250 }
Mike Stumpb7074c02009-02-13 15:32:32 +00001251
Mike Stump005c9a62009-02-13 15:25:34 +00001252 return GenericBlockLiteralType;
Anders Carlsson2437cbf2009-02-12 00:39:25 +00001253}
1254
Yaxun Liu10712d92017-10-04 20:32:17 +00001255RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
Anders Carlssonbfb36712009-12-24 21:13:40 +00001256 ReturnValueSlot ReturnValue) {
Simon Pilgrim0abbb152019-10-04 15:01:54 +00001257 const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
John McCallb92ab1a2016-10-26 23:46:34 +00001258 llvm::Value *BlockPtr = EmitScalarExpr(E->getCallee());
Andrew Savonichev43fceb22019-02-21 11:02:10 +00001259 llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1260 llvm::Value *Func = nullptr;
1261 QualType FnType = BPT->getPointeeType();
1262 ASTContext &Ctx = getContext();
Anders Carlsson2437cbf2009-02-12 00:39:25 +00001263 CallArgList Args;
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001264
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001265 if (getLangOpts().OpenCL) {
Andrew Savonichev43fceb22019-02-21 11:02:10 +00001266 // For OpenCL, BlockPtr is already casted to generic block literal.
1267
1268 // First argument of a block call is a generic block literal casted to
1269 // generic void pointer, i.e. i8 addrspace(4)*
1270 llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1271 BlockPtr, CGM.getOpenCLRuntime().getGenericVoidPointerType());
1272 QualType VoidPtrQualTy = Ctx.getPointerType(
1273 Ctx.getAddrSpaceQualType(Ctx.VoidTy, LangAS::opencl_generic));
1274 Args.add(RValue::get(BlockDescriptor), VoidPtrQualTy);
1275 // And the rest of the arguments.
1276 EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments());
1277
1278 // We *can* call the block directly unless it is a function argument.
1279 if (!isa<ParmVarDecl>(E->getCalleeDecl()))
1280 Func = CGM.getOpenCLRuntime().getInvokeFunction(E->getCallee());
1281 else {
1282 llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2);
1283 Func = Builder.CreateAlignedLoad(FuncPtr, getPointerAlign());
1284 }
1285 } else {
1286 // Bitcast the block literal to a generic block literal.
1287 BlockPtr = Builder.CreatePointerCast(
1288 BlockPtr, llvm::PointerType::get(GenBlockTy, 0), "block.literal");
1289 // Get pointer to the block invoke function
1290 llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3);
1291
1292 // First argument is a block literal casted to a void pointer
1293 BlockPtr = Builder.CreatePointerCast(BlockPtr, VoidPtrTy);
1294 Args.add(RValue::get(BlockPtr), Ctx.VoidPtrTy);
1295 // And the rest of the arguments.
1296 EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments());
1297
1298 // Load the function.
1299 Func = Builder.CreateAlignedLoad(FuncPtr, getPointerAlign());
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001300 }
1301
John McCall85915252011-03-09 08:39:33 +00001302 const FunctionType *FuncTy = FnType->castAs<FunctionType>();
John McCalla729c622012-02-17 03:33:10 +00001303 const CGFunctionInfo &FnInfo =
John McCallc818bbb2012-12-07 07:03:17 +00001304 CGM.getTypes().arrangeBlockFunctionCall(Args, FuncTy);
Mike Stump11289f42009-09-09 15:08:12 +00001305
Anders Carlsson5f50c652009-04-07 22:10:22 +00001306 // Cast the function pointer to the right type.
John McCalla729c622012-02-17 03:33:10 +00001307 llvm::Type *BlockFTy = CGM.getTypes().GetFunctionType(FnInfo);
Mike Stump11289f42009-09-09 15:08:12 +00001308
Chris Lattner2192fe52011-07-18 04:24:23 +00001309 llvm::Type *BlockFTyPtr = llvm::PointerType::getUnqual(BlockFTy);
Yaxun Liu10712d92017-10-04 20:32:17 +00001310 Func = Builder.CreatePointerCast(Func, BlockFTyPtr);
Mike Stump11289f42009-09-09 15:08:12 +00001311
John McCallb92ab1a2016-10-26 23:46:34 +00001312 // Prepare the callee.
1313 CGCallee Callee(CGCalleeInfo(), Func);
1314
Anders Carlsson2437cbf2009-02-12 00:39:25 +00001315 // And call the block.
John McCallb92ab1a2016-10-26 23:46:34 +00001316 return EmitCall(FnInfo, Callee, ReturnValue, Args);
Anders Carlsson2437cbf2009-02-12 00:39:25 +00001317}
Anders Carlsson6a60fa22009-02-12 17:55:02 +00001318
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001319Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
John McCall351762c2011-02-07 10:33:21 +00001320 assert(BlockInfo && "evaluating block ref without block information?");
1321 const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable);
John McCall87fe5d52010-05-20 01:18:31 +00001322
John McCall351762c2011-02-07 10:33:21 +00001323 // Handle constant captures.
John McCall7f416cc2015-09-08 08:05:57 +00001324 if (capture.isConstant()) return LocalDeclMap.find(variable)->second;
John McCall87fe5d52010-05-20 01:18:31 +00001325
James Y Knight751fe282019-02-09 22:22:28 +00001326 Address addr = Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(),
1327 "block.capture.addr");
John McCall87fe5d52010-05-20 01:18:31 +00001328
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001329 if (variable->isEscapingByref()) {
John McCall351762c2011-02-07 10:33:21 +00001330 // addr should be a void** right now. Load, then cast the result
1331 // to byref*.
Mike Stump97d01d52009-03-04 03:23:46 +00001332
John McCall7f416cc2015-09-08 08:05:57 +00001333 auto &byrefInfo = getBlockByrefInfo(variable);
1334 addr = Address(Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
Mike Stump7fe9cc12009-10-21 03:49:08 +00001335
John McCall7f416cc2015-09-08 08:05:57 +00001336 auto byrefPointerType = llvm::PointerType::get(byrefInfo.Type, 0);
1337 addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr");
Mike Stump7fe9cc12009-10-21 03:49:08 +00001338
John McCall7f416cc2015-09-08 08:05:57 +00001339 addr = emitBlockByrefAddress(addr, byrefInfo, /*follow*/ true,
1340 variable->getName());
John McCall87fe5d52010-05-20 01:18:31 +00001341 }
1342
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001343 assert((!variable->isNonEscapingByref() ||
1344 capture.fieldType()->isReferenceType()) &&
1345 "the capture field of a non-escaping variable should have a "
1346 "reference type");
Ivan A. Kosarev9f9d1572017-10-30 11:49:31 +00001347 if (capture.fieldType()->isReferenceType())
1348 addr = EmitLoadOfReference(MakeAddrLValue(addr, capture.fieldType()));
Mike Stump7fe9cc12009-10-21 03:49:08 +00001349
John McCall351762c2011-02-07 10:33:21 +00001350 return addr;
Mike Stump97d01d52009-03-04 03:23:46 +00001351}
1352
George Burgess IVe3763372016-12-22 02:50:20 +00001353void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1354 llvm::Constant *Addr) {
1355 bool Ok = EmittedGlobalBlocks.insert(std::make_pair(BE, Addr)).second;
1356 (void)Ok;
1357 assert(Ok && "Trying to replace an already-existing global block!");
1358}
1359
Mike Stump2d5a2872009-02-14 22:16:35 +00001360llvm::Constant *
George Burgess IV70d15b32016-11-03 02:21:43 +00001361CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1362 StringRef Name) {
George Burgess IVe3763372016-12-22 02:50:20 +00001363 if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1364 return Block;
1365
George Burgess IV70d15b32016-11-03 02:21:43 +00001366 CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1367 blockInfo.BlockExpression = BE;
Mike Stumpb7074c02009-02-13 15:32:32 +00001368
John McCall351762c2011-02-07 10:33:21 +00001369 // Compute information about the layout, etc., of this block.
Craig Topper8a13c412014-05-21 05:09:00 +00001370 computeBlockInfo(*this, nullptr, blockInfo);
Mike Stumpb7074c02009-02-13 15:32:32 +00001371
John McCall351762c2011-02-07 10:33:21 +00001372 // Using that metadata, generate the actual block function.
John McCall351762c2011-02-07 10:33:21 +00001373 {
John McCall7f416cc2015-09-08 08:05:57 +00001374 CodeGenFunction::DeclMapTy LocalDeclMap;
Akira Hatanakaba0367a2017-09-22 21:32:06 +00001375 CodeGenFunction(*this).GenerateBlockFunction(
1376 GlobalDecl(), blockInfo, LocalDeclMap,
1377 /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
John McCall351762c2011-02-07 10:33:21 +00001378 }
Mike Stumpb7074c02009-02-13 15:32:32 +00001379
Akira Hatanakaba0367a2017-09-22 21:32:06 +00001380 return getAddrOfGlobalBlockIfEmitted(BE);
Anders Carlsson6a60fa22009-02-12 17:55:02 +00001381}
1382
John McCall351762c2011-02-07 10:33:21 +00001383static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1384 const CGBlockInfo &blockInfo,
1385 llvm::Constant *blockFn) {
1386 assert(blockInfo.CanBeGlobal);
George Burgess IVe3763372016-12-22 02:50:20 +00001387 // Callers should detect this case on their own: calling this function
1388 // generally requires computing layout information, which is a waste of time
1389 // if we've already emitted this block.
1390 assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1391 "Refusing to re-emit a global block.");
John McCall351762c2011-02-07 10:33:21 +00001392
1393 // Generate the constants for the block literal initializer.
John McCall23c9dc62016-11-28 22:18:27 +00001394 ConstantInitBuilder builder(CGM);
John McCall6c9f1fdb2016-11-19 08:17:24 +00001395 auto fields = builder.beginStruct();
John McCall351762c2011-02-07 10:33:21 +00001396
Yaxun Liu10712d92017-10-04 20:32:17 +00001397 bool IsOpenCL = CGM.getLangOpts().OpenCL;
David Chisnallc5a458c2018-08-09 08:02:42 +00001398 bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
Yaxun Liu10712d92017-10-04 20:32:17 +00001399 if (!IsOpenCL) {
1400 // isa
David Chisnallc5a458c2018-08-09 08:02:42 +00001401 if (IsWindows)
1402 fields.addNullPointer(CGM.Int8PtrPtrTy);
1403 else
1404 fields.add(CGM.getNSConcreteGlobalBlock());
John McCall351762c2011-02-07 10:33:21 +00001405
Yaxun Liu10712d92017-10-04 20:32:17 +00001406 // __flags
1407 BlockFlags flags = BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE;
1408 if (blockInfo.UsesStret)
1409 flags |= BLOCK_USE_STRET;
John McCall351762c2011-02-07 10:33:21 +00001410
Yaxun Liu10712d92017-10-04 20:32:17 +00001411 fields.addInt(CGM.IntTy, flags.getBitMask());
1412
1413 // Reserved
1414 fields.addInt(CGM.IntTy, 0);
1415 } else {
1416 fields.addInt(CGM.IntTy, blockInfo.BlockSize.getQuantity());
1417 fields.addInt(CGM.IntTy, blockInfo.BlockAlign.getQuantity());
1418 }
John McCall351762c2011-02-07 10:33:21 +00001419
Sven van Haastregtda3b6322018-10-02 13:02:24 +00001420 // Function
1421 fields.add(blockFn);
1422
Yaxun Liu10712d92017-10-04 20:32:17 +00001423 if (!IsOpenCL) {
1424 // Descriptor
1425 fields.add(buildBlockDescriptor(CGM, blockInfo));
1426 } else if (auto *Helper =
1427 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1428 for (auto I : Helper->getCustomFieldValues(CGM, blockInfo)) {
1429 fields.add(I);
1430 }
1431 }
John McCall351762c2011-02-07 10:33:21 +00001432
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001433 unsigned AddrSpace = 0;
1434 if (CGM.getContext().getLangOpts().OpenCL)
1435 AddrSpace = CGM.getContext().getTargetAddressSpace(LangAS::opencl_global);
1436
Akira Hatanaka6cb2d9d2019-06-14 22:06:28 +00001437 llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001438 "__block_literal_global", blockInfo.BlockAlign,
David Chisnallc5a458c2018-08-09 08:02:42 +00001439 /*constant*/ !IsWindows, llvm::GlobalVariable::InternalLinkage, AddrSpace);
1440
Akira Hatanaka6cb2d9d2019-06-14 22:06:28 +00001441 literal->addAttribute("objc_arc_inert");
1442
David Chisnallc5a458c2018-08-09 08:02:42 +00001443 // Windows does not allow globals to be initialised to point to globals in
1444 // different DLLs. Any such variables must run code to initialise them.
1445 if (IsWindows) {
1446 auto *Init = llvm::Function::Create(llvm::FunctionType::get(CGM.VoidTy,
1447 {}), llvm::GlobalValue::InternalLinkage, ".block_isa_init",
1448 &CGM.getModule());
1449 llvm::IRBuilder<> b(llvm::BasicBlock::Create(CGM.getLLVMContext(), "entry",
1450 Init));
1451 b.CreateAlignedStore(CGM.getNSConcreteGlobalBlock(),
1452 b.CreateStructGEP(literal, 0), CGM.getPointerAlign().getQuantity());
1453 b.CreateRetVoid();
1454 // We can't use the normal LLVM global initialisation array, because we
1455 // need to specify that this runs early in library initialisation.
1456 auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1457 /*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1458 Init, ".block_isa_init_ptr");
1459 InitVar->setSection(".CRT$XCLa");
1460 CGM.addUsedGlobal(InitVar);
1461 }
John McCall351762c2011-02-07 10:33:21 +00001462
1463 // Return a constant of the appropriately-casted type.
George Burgess IVe3763372016-12-22 02:50:20 +00001464 llvm::Type *RequiredType =
John McCall351762c2011-02-07 10:33:21 +00001465 CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType());
George Burgess IVe3763372016-12-22 02:50:20 +00001466 llvm::Constant *Result =
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001467 llvm::ConstantExpr::getPointerCast(literal, RequiredType);
George Burgess IVe3763372016-12-22 02:50:20 +00001468 CGM.setAddrOfGlobalBlock(blockInfo.BlockExpression, Result);
Yaxun Liufa13d012018-02-15 16:39:19 +00001469 if (CGM.getContext().getLangOpts().OpenCL)
1470 CGM.getOpenCLRuntime().recordBlockInfo(
1471 blockInfo.BlockExpression,
1472 cast<llvm::Function>(blockFn->stripPointerCasts()), Result);
George Burgess IVe3763372016-12-22 02:50:20 +00001473 return Result;
Mike Stumpcb2fbcb2009-02-21 20:00:35 +00001474}
1475
John McCall7f416cc2015-09-08 08:05:57 +00001476void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1477 unsigned argNum,
1478 llvm::Value *arg) {
1479 assert(BlockInfo && "not emitting prologue of block invocation function?!");
1480
Adrian Prantl356347b2017-10-26 20:08:52 +00001481 // Allocate a stack slot like for any local variable to guarantee optimal
1482 // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1483 Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
1484 Builder.CreateStore(arg, alloc);
John McCall7f416cc2015-09-08 08:05:57 +00001485 if (CGDebugInfo *DI = getDebugInfo()) {
Benjamin Kramer8c305922016-02-02 11:06:51 +00001486 if (CGM.getCodeGenOpts().getDebugInfo() >=
1487 codegenoptions::LimitedDebugInfo) {
John McCall7f416cc2015-09-08 08:05:57 +00001488 DI->setLocation(D->getLocation());
Adrian Prantl356347b2017-10-26 20:08:52 +00001489 DI->EmitDeclareOfBlockLiteralArgVariable(
1490 *BlockInfo, D->getName(), argNum,
1491 cast<llvm::AllocaInst>(alloc.getPointer()), Builder);
John McCall7f416cc2015-09-08 08:05:57 +00001492 }
1493 }
1494
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001495 SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
John McCall7f416cc2015-09-08 08:05:57 +00001496 ApplyDebugLocation Scope(*this, StartLoc);
1497
1498 // Instead of messing around with LocalDeclMap, just set the value
1499 // directly as BlockPointer.
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001500 BlockPointer = Builder.CreatePointerCast(
1501 arg,
1502 BlockInfo->StructureType->getPointerTo(
1503 getContext().getLangOpts().OpenCL
1504 ? getContext().getTargetAddressSpace(LangAS::opencl_generic)
1505 : 0),
1506 "block");
John McCall7f416cc2015-09-08 08:05:57 +00001507}
1508
1509Address CodeGenFunction::LoadBlockStruct() {
1510 assert(BlockInfo && "not in a block invocation function!");
1511 assert(BlockPointer && "no block pointer set!");
1512 return Address(BlockPointer, BlockInfo->BlockAlign);
1513}
1514
Mike Stump4446dcf2009-03-05 08:32:30 +00001515llvm::Function *
John McCall351762c2011-02-07 10:33:21 +00001516CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
1517 const CGBlockInfo &blockInfo,
Eli Friedman2495ab02012-02-25 02:48:22 +00001518 const DeclMapTy &ldm,
Akira Hatanakaba0367a2017-09-22 21:32:06 +00001519 bool IsLambdaConversionToBlock,
1520 bool BuildGlobalBlock) {
John McCall351762c2011-02-07 10:33:21 +00001521 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
Devang Patel9074ed82009-04-15 21:51:44 +00001522
Fariborz Jahanian63628032012-06-26 16:06:38 +00001523 CurGD = GD;
David Blaikie1ae04912015-01-13 23:06:27 +00001524
Stephen Kelly1c301dc2018-08-09 21:09:38 +00001525 CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
Fangrui Song6907ce22018-07-30 19:24:48 +00001526
John McCall351762c2011-02-07 10:33:21 +00001527 BlockInfo = &blockInfo;
Mike Stump11289f42009-09-09 15:08:12 +00001528
Mike Stump5469f292009-03-13 23:34:28 +00001529 // Arrange for local static and local extern declarations to appear
John McCall351762c2011-02-07 10:33:21 +00001530 // to be local to this function as well, in case they're directly
1531 // referenced in a block.
1532 for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) {
Rafael Espindola2ae250c2014-05-09 00:08:36 +00001533 const auto *var = dyn_cast<VarDecl>(i->first);
John McCall351762c2011-02-07 10:33:21 +00001534 if (var && !var->hasLocalStorage())
John McCall7f416cc2015-09-08 08:05:57 +00001535 setAddrOfLocalVar(var, i->second);
Mike Stump5469f292009-03-13 23:34:28 +00001536 }
1537
John McCall351762c2011-02-07 10:33:21 +00001538 // Begin building the function declaration.
Eli Friedman09a9b6e2009-03-28 03:24:54 +00001539
John McCall351762c2011-02-07 10:33:21 +00001540 // Build the argument list.
1541 FunctionArgList args;
Mike Stumpb7074c02009-02-13 15:32:32 +00001542
John McCall351762c2011-02-07 10:33:21 +00001543 // The first argument is the block pointer. Just take it as a void*
1544 // and cast it later.
1545 QualType selfTy = getContext().VoidPtrTy;
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001546
1547 // For OpenCL passed block pointer can be private AS local variable or
1548 // global AS program scope variable (for the case with and without captures).
Hiroshi Inouec5e54dd2017-07-03 08:49:44 +00001549 // Generic AS is used therefore to be able to accommodate both private and
Anastasia Stulovaaf0a7bb2017-01-27 15:11:34 +00001550 // generic AS in one implementation.
1551 if (getLangOpts().OpenCL)
1552 selfTy = getContext().getPointerType(getContext().getAddrSpaceQualType(
1553 getContext().VoidTy, LangAS::opencl_generic));
1554
Mike Stump7fe9cc12009-10-21 03:49:08 +00001555 IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor");
Mike Stumpd0153282009-10-20 02:12:22 +00001556
Alexey Bataev56223232017-06-09 13:40:18 +00001557 ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl),
1558 SourceLocation(), II, selfTy,
1559 ImplicitParamDecl::ObjCSelf);
1560 args.push_back(&SelfDecl);
Mike Stump7fe9cc12009-10-21 03:49:08 +00001561
John McCall351762c2011-02-07 10:33:21 +00001562 // Now add the rest of the parameters.
Benjamin Kramerf9890422015-02-17 16:48:30 +00001563 args.append(blockDecl->param_begin(), blockDecl->param_end());
John McCall87fe5d52010-05-20 01:18:31 +00001564
John McCall351762c2011-02-07 10:33:21 +00001565 // Create the function declaration.
John McCalla729c622012-02-17 03:33:10 +00001566 const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
John McCallc56a8b32016-03-11 04:30:31 +00001567 const CGFunctionInfo &fnInfo =
1568 CGM.getTypes().arrangeBlockFunctionDeclaration(fnType, args);
Tim Northovere77cc392014-03-29 13:28:05 +00001569 if (CGM.ReturnSlotInterferesWithArgs(fnInfo))
John McCall85915252011-03-09 08:39:33 +00001570 blockInfo.UsesStret = true;
1571
John McCalla729c622012-02-17 03:33:10 +00001572 llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(fnInfo);
Mike Stumpb7074c02009-02-13 15:32:32 +00001573
Alp Tokerfb8d02b2014-06-05 22:10:59 +00001574 StringRef name = CGM.getBlockMangledName(GD, blockDecl);
Alp Toker0e64e0d2014-06-03 02:13:57 +00001575 llvm::Function *fn = llvm::Function::Create(
1576 fnLLVMType, llvm::GlobalValue::InternalLinkage, name, &CGM.getModule());
John McCall351762c2011-02-07 10:33:21 +00001577 CGM.SetInternalFunctionAttributes(blockDecl, fn, fnInfo);
Mike Stumpb7074c02009-02-13 15:32:32 +00001578
Yaxun Liu10712d92017-10-04 20:32:17 +00001579 if (BuildGlobalBlock) {
1580 auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1581 ? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1582 : VoidPtrTy;
Akira Hatanakaba0367a2017-09-22 21:32:06 +00001583 buildGlobalBlock(CGM, blockInfo,
Yaxun Liu10712d92017-10-04 20:32:17 +00001584 llvm::ConstantExpr::getPointerCast(fn, GenVoidPtrTy));
1585 }
Akira Hatanakaba0367a2017-09-22 21:32:06 +00001586
John McCall351762c2011-02-07 10:33:21 +00001587 // Begin generating the function.
Alp Toker314cc812014-01-25 16:55:45 +00001588 StartFunction(blockDecl, fnType->getReturnType(), fn, fnInfo, args,
Adrian Prantl42d71b92014-04-10 23:21:53 +00001589 blockDecl->getLocation(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001590 blockInfo.getBlockExpr()->getBody()->getBeginLoc());
Mike Stumpb7074c02009-02-13 15:32:32 +00001591
John McCall147d0212011-02-22 22:38:33 +00001592 // Okay. Undo some of what StartFunction did.
John McCall7f416cc2015-09-08 08:05:57 +00001593
Adrian Prantl0f6df002013-03-29 19:20:35 +00001594 // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1595 // won't delete the dbg.declare intrinsics for captured variables.
1596 llvm::Value *BlockPointerDbgLoc = BlockPointer;
1597 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1598 // Allocate a stack slot for it, so we can point the debugger to it
John McCall7f416cc2015-09-08 08:05:57 +00001599 Address Alloca = CreateTempAlloca(BlockPointer->getType(),
1600 getPointerAlign(),
1601 "block.addr");
Adrian Prantl2832b4e2013-04-02 01:00:48 +00001602 // Set the DebugLocation to empty, so the store is recognized as a
1603 // frame setup instruction by llvm::DwarfDebug::beginFunction().
Adrian Prantl95b24e92015-02-03 20:00:54 +00001604 auto NL = ApplyDebugLocation::CreateEmpty(*this);
John McCall7f416cc2015-09-08 08:05:57 +00001605 Builder.CreateStore(BlockPointer, Alloca);
1606 BlockPointerDbgLoc = Alloca.getPointer();
Adrian Prantl0f6df002013-03-29 19:20:35 +00001607 }
Anders Carlsson6a60fa22009-02-12 17:55:02 +00001608
John McCall87fe5d52010-05-20 01:18:31 +00001609 // If we have a C++ 'this' reference, go ahead and force it into
1610 // existence now.
John McCall351762c2011-02-07 10:33:21 +00001611 if (blockDecl->capturesCXXThis()) {
James Y Knight751fe282019-02-09 22:22:28 +00001612 Address addr = Builder.CreateStructGEP(
1613 LoadBlockStruct(), blockInfo.CXXThisIndex, "block.captured-this");
John McCall351762c2011-02-07 10:33:21 +00001614 CXXThisValue = Builder.CreateLoad(addr, "this");
John McCall87fe5d52010-05-20 01:18:31 +00001615 }
1616
John McCall351762c2011-02-07 10:33:21 +00001617 // Also force all the constant captures.
Aaron Ballman9371dd22014-03-14 18:34:04 +00001618 for (const auto &CI : blockDecl->captures()) {
1619 const VarDecl *variable = CI.getVariable();
John McCall351762c2011-02-07 10:33:21 +00001620 const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
1621 if (!capture.isConstant()) continue;
1622
John McCall7f416cc2015-09-08 08:05:57 +00001623 CharUnits align = getContext().getDeclAlign(variable);
1624 Address alloca =
1625 CreateMemTemp(variable->getType(), align, "block.captured-const");
John McCall351762c2011-02-07 10:33:21 +00001626
John McCall7f416cc2015-09-08 08:05:57 +00001627 Builder.CreateStore(capture.getConstant(), alloca);
John McCall351762c2011-02-07 10:33:21 +00001628
John McCall7f416cc2015-09-08 08:05:57 +00001629 setAddrOfLocalVar(variable, alloca);
John McCall9d42f0f2010-05-21 04:11:14 +00001630 }
1631
John McCall113bee02012-03-10 09:33:50 +00001632 // Save a spot to insert the debug information for all the DeclRefExprs.
Mike Stump017460a2009-10-01 22:29:41 +00001633 llvm::BasicBlock *entry = Builder.GetInsertBlock();
1634 llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1635 --entry_ptr;
1636
Eli Friedman2495ab02012-02-25 02:48:22 +00001637 if (IsLambdaConversionToBlock)
1638 EmitLambdaBlockInvokeBody();
Bob Wilsonc845c002014-03-06 20:24:27 +00001639 else {
Serge Pavlov3a561452015-12-06 14:32:39 +00001640 PGO.assignRegionCounters(GlobalDecl(blockDecl), fn);
Justin Bogner66242d62015-04-23 23:06:47 +00001641 incrementProfileCounter(blockDecl->getBody());
Eli Friedman2495ab02012-02-25 02:48:22 +00001642 EmitStmt(blockDecl->getBody());
Bob Wilsonc845c002014-03-06 20:24:27 +00001643 }
Mike Stump017460a2009-10-01 22:29:41 +00001644
Mike Stump7d699112009-10-01 00:27:30 +00001645 // Remember where we were...
1646 llvm::BasicBlock *resume = Builder.GetInsertBlock();
Mike Stump017460a2009-10-01 22:29:41 +00001647
Mike Stump7d699112009-10-01 00:27:30 +00001648 // Go back to the entry.
Mike Stump017460a2009-10-01 22:29:41 +00001649 ++entry_ptr;
1650 Builder.SetInsertPoint(entry, entry_ptr);
1651
John McCall113bee02012-03-10 09:33:50 +00001652 // Emit debug information for all the DeclRefExprs.
John McCall351762c2011-02-07 10:33:21 +00001653 // FIXME: also for 'this'
Mike Stump2e722b92009-09-30 02:43:10 +00001654 if (CGDebugInfo *DI = getDebugInfo()) {
Aaron Ballman9371dd22014-03-14 18:34:04 +00001655 for (const auto &CI : blockDecl->captures()) {
1656 const VarDecl *variable = CI.getVariable();
Eric Christopher7cdf9482011-10-13 21:45:18 +00001657 DI->EmitLocation(Builder, variable->getLocation());
John McCall351762c2011-02-07 10:33:21 +00001658
Benjamin Kramer8c305922016-02-02 11:06:51 +00001659 if (CGM.getCodeGenOpts().getDebugInfo() >=
1660 codegenoptions::LimitedDebugInfo) {
Alexey Samsonov74a38682012-05-04 07:39:27 +00001661 const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
1662 if (capture.isConstant()) {
John McCall7f416cc2015-09-08 08:05:57 +00001663 auto addr = LocalDeclMap.find(variable)->second;
Sander de Smalen891af03a2018-02-03 13:55:59 +00001664 (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(),
1665 Builder);
Alexey Samsonov74a38682012-05-04 07:39:27 +00001666 continue;
1667 }
John McCall351762c2011-02-07 10:33:21 +00001668
Duncan P. N. Exon Smith9f5260a2015-11-06 23:00:41 +00001669 DI->EmitDeclareOfBlockDeclRefVariable(
1670 variable, BlockPointerDbgLoc, Builder, blockInfo,
1671 entry_ptr == entry->end() ? nullptr : &*entry_ptr);
Alexey Samsonov74a38682012-05-04 07:39:27 +00001672 }
Mike Stump2e722b92009-09-30 02:43:10 +00001673 }
Manman Renab08a9a2013-01-04 18:51:35 +00001674 // Recover location if it was changed in the above loop.
1675 DI->EmitLocation(Builder,
Adrian Prantl83e30fd2013-04-08 20:52:12 +00001676 cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc());
Mike Stump2e722b92009-09-30 02:43:10 +00001677 }
John McCall351762c2011-02-07 10:33:21 +00001678
Mike Stump7d699112009-10-01 00:27:30 +00001679 // And resume where we left off.
Craig Topper8a13c412014-05-21 05:09:00 +00001680 if (resume == nullptr)
Mike Stump7d699112009-10-01 00:27:30 +00001681 Builder.ClearInsertionPoint();
1682 else
1683 Builder.SetInsertPoint(resume);
Mike Stump2e722b92009-09-30 02:43:10 +00001684
John McCall351762c2011-02-07 10:33:21 +00001685 FinishFunction(cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc());
Anders Carlsson6a60fa22009-02-12 17:55:02 +00001686
John McCall351762c2011-02-07 10:33:21 +00001687 return fn;
Anders Carlsson6a60fa22009-02-12 17:55:02 +00001688}
Mike Stump1db7d042009-02-28 09:07:16 +00001689
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001690static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1691computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1692 const LangOptions &LangOpts) {
1693 if (CI.getCopyExpr()) {
1694 assert(!CI.isByRef());
1695 // don't bother computing flags
1696 return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags());
1697 }
1698 BlockFieldFlags Flags;
Akira Hatanaka8e57b072018-10-01 21:51:28 +00001699 if (CI.isEscapingByref()) {
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001700 Flags = BLOCK_FIELD_IS_BYREF;
1701 if (T.isObjCGCWeak())
1702 Flags |= BLOCK_FIELD_IS_WEAK;
1703 return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);
1704 }
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001705
1706 Flags = BLOCK_FIELD_IS_OBJECT;
1707 bool isBlockPointer = T->isBlockPointerType();
1708 if (isBlockPointer)
1709 Flags = BLOCK_FIELD_IS_BLOCK;
1710
Akira Hatanaka7275da02018-02-28 07:15:55 +00001711 switch (T.isNonTrivialToPrimitiveCopy()) {
1712 case QualType::PCK_Struct:
1713 return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct,
1714 BlockFieldFlags());
Akira Hatanakad791e922018-03-19 17:38:40 +00001715 case QualType::PCK_ARCWeak:
1716 // We need to register __weak direct captures with the runtime.
1717 return std::make_pair(BlockCaptureEntityKind::ARCWeak, Flags);
Akira Hatanaka7275da02018-02-28 07:15:55 +00001718 case QualType::PCK_ARCStrong:
1719 // We need to retain the copied value for __strong direct captures.
1720 // If it's a block pointer, we have to copy the block and assign that to
1721 // the destination pointer, so we might as well use _Block_object_assign.
1722 // Otherwise we can avoid that.
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001723 return std::make_pair(!isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1724 : BlockCaptureEntityKind::BlockObject,
1725 Flags);
Akira Hatanaka7275da02018-02-28 07:15:55 +00001726 case QualType::PCK_Trivial:
1727 case QualType::PCK_VolatileTrivial: {
1728 if (!T->isObjCRetainableType())
1729 // For all other types, the memcpy is fine.
1730 return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());
1731
1732 // Special rules for ARC captures:
1733 Qualifiers QS = T.getQualifiers();
1734
Akira Hatanaka7275da02018-02-28 07:15:55 +00001735 // Non-ARC captures of retainable pointers are strong and
1736 // therefore require a call to _Block_object_assign.
1737 if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1738 return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);
1739
1740 // Otherwise the memcpy is fine.
1741 return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001742 }
Akira Hatanaka7275da02018-02-28 07:15:55 +00001743 }
Nico Weberb3897eb2018-02-28 19:28:47 +00001744 llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001745}
1746
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001747static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1748computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1749 const LangOptions &LangOpts);
1750
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001751/// Find the set of block captures that need to be explicitly copied or destroy.
1752static void findBlockCapturedManagedEntities(
1753 const CGBlockInfo &BlockInfo, const LangOptions &LangOpts,
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001754 SmallVectorImpl<BlockCaptureManagedEntity> &ManagedCaptures) {
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001755 for (const auto &CI : BlockInfo.getBlockDecl()->captures()) {
1756 const VarDecl *Variable = CI.getVariable();
1757 const CGBlockInfo::Capture &Capture = BlockInfo.getCapture(Variable);
1758 if (Capture.isConstant())
1759 continue;
1760
Akira Hatanaka2a5e4632018-08-22 13:41:19 +00001761 QualType VT = Capture.fieldType();
1762 auto CopyInfo = computeCopyInfoForBlockCapture(CI, VT, LangOpts);
1763 auto DisposeInfo = computeDestroyInfoForBlockCapture(CI, VT, LangOpts);
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001764 if (CopyInfo.first != BlockCaptureEntityKind::None ||
1765 DisposeInfo.first != BlockCaptureEntityKind::None)
1766 ManagedCaptures.emplace_back(CopyInfo.first, DisposeInfo.first,
1767 CopyInfo.second, DisposeInfo.second, CI,
1768 Capture);
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001769 }
Akira Hatanaka9978da32018-08-10 15:09:24 +00001770
1771 // Sort the captures by offset.
Fangrui Song55fab262018-09-26 22:16:28 +00001772 llvm::sort(ManagedCaptures);
Alex Lorenze08e5bc2017-03-06 16:23:04 +00001773}
1774
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001775namespace {
1776/// Release a __block variable.
1777struct CallBlockRelease final : EHScopeStack::Cleanup {
1778 Address Addr;
1779 BlockFieldFlags FieldFlags;
Akira Hatanaka9978da32018-08-10 15:09:24 +00001780 bool LoadBlockVarAddr, CanThrow;
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001781
Akira Hatanaka9978da32018-08-10 15:09:24 +00001782 CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1783 bool CT)
1784 : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1785 CanThrow(CT) {}
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001786
1787 void Emit(CodeGenFunction &CGF, Flags flags) override {
1788 llvm::Value *BlockVarAddr;
1789 if (LoadBlockVarAddr) {
1790 BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1791 BlockVarAddr = CGF.Builder.CreateBitCast(BlockVarAddr, CGF.VoidPtrTy);
1792 } else {
1793 BlockVarAddr = Addr.getPointer();
1794 }
1795
Akira Hatanaka9978da32018-08-10 15:09:24 +00001796 CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow);
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001797 }
1798};
1799} // end anonymous namespace
1800
Akira Hatanaka9978da32018-08-10 15:09:24 +00001801/// Check if \p T is a C++ class that has a destructor that can throw.
1802bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1803 if (const auto *RD = T->getAsCXXRecordDecl())
1804 if (const CXXDestructorDecl *DD = RD->getDestructor())
Simon Pilgrim0abbb152019-10-04 15:01:54 +00001805 return DD->getType()->castAs<FunctionProtoType>()->canThrow();
Akira Hatanaka9978da32018-08-10 15:09:24 +00001806 return false;
1807}
1808
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001809// Return a string that has the information about a capture.
1810static std::string getBlockCaptureStr(const BlockCaptureManagedEntity &E,
1811 CaptureStrKind StrKind,
1812 CharUnits BlockAlignment,
1813 CodeGenModule &CGM) {
1814 std::string Str;
Akira Hatanaka9978da32018-08-10 15:09:24 +00001815 ASTContext &Ctx = CGM.getContext();
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001816 const BlockDecl::Capture &CI = *E.CI;
1817 QualType CaptureTy = CI.getVariable()->getType();
Akira Hatanaka9978da32018-08-10 15:09:24 +00001818
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001819 BlockCaptureEntityKind Kind;
1820 BlockFieldFlags Flags;
1821
1822 // CaptureStrKind::Merged should be passed only when the operations and the
1823 // flags are the same for copy and dispose.
1824 assert((StrKind != CaptureStrKind::Merged ||
1825 (E.CopyKind == E.DisposeKind && E.CopyFlags == E.DisposeFlags)) &&
1826 "different operations and flags");
1827
1828 if (StrKind == CaptureStrKind::DisposeHelper) {
1829 Kind = E.DisposeKind;
1830 Flags = E.DisposeFlags;
1831 } else {
1832 Kind = E.CopyKind;
1833 Flags = E.CopyFlags;
1834 }
1835
1836 switch (Kind) {
1837 case BlockCaptureEntityKind::CXXRecord: {
1838 Str += "c";
1839 SmallString<256> TyStr;
1840 llvm::raw_svector_ostream Out(TyStr);
Akira Hatanaka32e0a582018-10-20 05:45:01 +00001841 CGM.getCXXABI().getMangleContext().mangleTypeName(CaptureTy, Out);
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001842 Str += llvm::to_string(TyStr.size()) + TyStr.c_str();
1843 break;
1844 }
1845 case BlockCaptureEntityKind::ARCWeak:
1846 Str += "w";
1847 break;
1848 case BlockCaptureEntityKind::ARCStrong:
1849 Str += "s";
1850 break;
1851 case BlockCaptureEntityKind::BlockObject: {
1852 const VarDecl *Var = CI.getVariable();
1853 unsigned F = Flags.getBitMask();
1854 if (F & BLOCK_FIELD_IS_BYREF) {
1855 Str += "r";
1856 if (F & BLOCK_FIELD_IS_WEAK)
1857 Str += "w";
1858 else {
1859 // If CaptureStrKind::Merged is passed, check both the copy expression
1860 // and the destructor.
1861 if (StrKind != CaptureStrKind::DisposeHelper) {
1862 if (Ctx.getBlockVarCopyInit(Var).canThrow())
1863 Str += "c";
1864 }
1865 if (StrKind != CaptureStrKind::CopyHelper) {
1866 if (CodeGenFunction::cxxDestructorCanThrow(CaptureTy))
1867 Str += "d";
1868 }
1869 }
1870 } else {
1871 assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1872 if (F == BLOCK_FIELD_IS_BLOCK)
1873 Str += "b";
1874 else
1875 Str += "o";
1876 }
1877 break;
1878 }
1879 case BlockCaptureEntityKind::NonTrivialCStruct: {
1880 bool IsVolatile = CaptureTy.isVolatileQualified();
1881 CharUnits Alignment =
1882 BlockAlignment.alignmentAtOffset(E.Capture->getOffset());
1883
1884 Str += "n";
1885 std::string FuncStr;
1886 if (StrKind == CaptureStrKind::DisposeHelper)
1887 FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1888 CaptureTy, Alignment, IsVolatile, Ctx);
1889 else
1890 // If CaptureStrKind::Merged is passed, use the copy constructor string.
1891 // It has all the information that the destructor string has.
1892 FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1893 CaptureTy, Alignment, IsVolatile, Ctx);
1894 // The underscore is necessary here because non-trivial copy constructor
1895 // and destructor strings can start with a number.
1896 Str += llvm::to_string(FuncStr.size()) + "_" + FuncStr;
1897 break;
1898 }
1899 case BlockCaptureEntityKind::None:
1900 break;
1901 }
1902
1903 return Str;
1904}
1905
1906static std::string getCopyDestroyHelperFuncName(
1907 const SmallVectorImpl<BlockCaptureManagedEntity> &Captures,
1908 CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1909 assert((StrKind == CaptureStrKind::CopyHelper ||
1910 StrKind == CaptureStrKind::DisposeHelper) &&
1911 "unexpected CaptureStrKind");
1912 std::string Name = StrKind == CaptureStrKind::CopyHelper
1913 ? "__copy_helper_block_"
1914 : "__destroy_helper_block_";
Akira Hatanaka9978da32018-08-10 15:09:24 +00001915 if (CGM.getLangOpts().Exceptions)
1916 Name += "e";
1917 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1918 Name += "a";
1919 Name += llvm::to_string(BlockAlignment.getQuantity()) + "_";
1920
1921 for (const BlockCaptureManagedEntity &E : Captures) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00001922 Name += llvm::to_string(E.Capture->getOffset().getQuantity());
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001923 Name += getBlockCaptureStr(E, StrKind, BlockAlignment, CGM);
Akira Hatanaka9978da32018-08-10 15:09:24 +00001924 }
1925
1926 return Name;
1927}
1928
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001929static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1930 Address Field, QualType CaptureType,
Akira Hatanaka9978da32018-08-10 15:09:24 +00001931 BlockFieldFlags Flags, bool ForCopyHelper,
1932 VarDecl *Var, CodeGenFunction &CGF) {
1933 bool EHOnly = ForCopyHelper;
1934
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001935 switch (CaptureKind) {
1936 case BlockCaptureEntityKind::CXXRecord:
1937 case BlockCaptureEntityKind::ARCWeak:
1938 case BlockCaptureEntityKind::NonTrivialCStruct:
1939 case BlockCaptureEntityKind::ARCStrong: {
1940 if (CaptureType.isDestructedType() &&
1941 (!EHOnly || CGF.needsEHCleanup(CaptureType.isDestructedType()))) {
1942 CodeGenFunction::Destroyer *Destroyer =
1943 CaptureKind == BlockCaptureEntityKind::ARCStrong
1944 ? CodeGenFunction::destroyARCStrongImprecise
1945 : CGF.getDestroyer(CaptureType.isDestructedType());
1946 CleanupKind Kind =
1947 EHOnly ? EHCleanup
1948 : CGF.getCleanupKind(CaptureType.isDestructedType());
1949 CGF.pushDestroy(Kind, Field, CaptureType, Destroyer, Kind & EHCleanup);
1950 }
1951 break;
1952 }
1953 case BlockCaptureEntityKind::BlockObject: {
1954 if (!EHOnly || CGF.getLangOpts().Exceptions) {
1955 CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
Akira Hatanaka9978da32018-08-10 15:09:24 +00001956 // Calls to _Block_object_dispose along the EH path in the copy helper
1957 // function don't throw as newly-copied __block variables always have a
1958 // reference count of 2.
1959 bool CanThrow =
1960 !ForCopyHelper && CGF.cxxDestructorCanThrow(CaptureType);
1961 CGF.enterByrefCleanup(Kind, Field, Flags, /*LoadBlockVarAddr*/ true,
1962 CanThrow);
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001963 }
1964 break;
1965 }
1966 case BlockCaptureEntityKind::None:
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001967 break;
Akira Hatanakacb6a9332018-07-26 16:51:21 +00001968 }
1969}
1970
Akira Hatanaka9978da32018-08-10 15:09:24 +00001971static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1972 llvm::Function *Fn,
1973 const CGFunctionInfo &FI,
1974 CodeGenModule &CGM) {
1975 if (CapturesNonExternalType) {
1976 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
1977 } else {
1978 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1979 Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
Erich Keanede6480a32018-11-13 15:48:08 +00001980 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn);
Akira Hatanaka9978da32018-08-10 15:09:24 +00001981 CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
1982 }
1983}
John McCallf593b102013-01-22 03:56:22 +00001984/// Generate the copy-helper function for a block closure object:
1985/// static void block_copy_helper(block_t *dst, block_t *src);
1986/// The runtime will have previously initialized 'dst' by doing a
1987/// bit-copy of 'src'.
1988///
1989/// Note that this copies an entire block closure object to the heap;
1990/// it should not be confused with a 'byref copy helper', which moves
1991/// the contents of an individual __block variable to the heap.
John McCall351762c2011-02-07 10:33:21 +00001992llvm::Constant *
John McCallad7c5c12011-02-08 08:22:06 +00001993CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00001994 SmallVector<BlockCaptureManagedEntity, 4> CopiedCaptures;
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001995 findBlockCapturedManagedEntities(blockInfo, getLangOpts(), CopiedCaptures);
Akira Hatanaka9978da32018-08-10 15:09:24 +00001996 std::string FuncName =
1997 getCopyDestroyHelperFuncName(CopiedCaptures, blockInfo.BlockAlign,
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00001998 CaptureStrKind::CopyHelper, CGM);
Akira Hatanaka9978da32018-08-10 15:09:24 +00001999
2000 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))
Akira Hatanaka936240c2018-08-14 00:15:42 +00002001 return llvm::ConstantExpr::getBitCast(Func, VoidPtrTy);
Akira Hatanaka9978da32018-08-10 15:09:24 +00002002
John McCall351762c2011-02-07 10:33:21 +00002003 ASTContext &C = getContext();
2004
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002005 QualType ReturnTy = C.VoidTy;
2006
John McCall351762c2011-02-07 10:33:21 +00002007 FunctionArgList args;
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002008 ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev56223232017-06-09 13:40:18 +00002009 args.push_back(&DstDecl);
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002010 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev56223232017-06-09 13:40:18 +00002011 args.push_back(&SrcDecl);
Mike Stump11289f42009-09-09 15:08:12 +00002012
John McCallc56a8b32016-03-11 04:30:31 +00002013 const CGFunctionInfo &FI =
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002014 CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);
Mike Stump0c743272009-03-06 01:33:24 +00002015
John McCall351762c2011-02-07 10:33:21 +00002016 // FIXME: it would be nice if these were mergeable with things with
2017 // identical semantics.
John McCalla729c622012-02-17 03:33:10 +00002018 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);
Mike Stump0c743272009-03-06 01:33:24 +00002019
2020 llvm::Function *Fn =
Akira Hatanaka9978da32018-08-10 15:09:24 +00002021 llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,
2022 FuncName, &CGM.getModule());
Saleem Abdulrasool89628922019-02-22 16:29:50 +00002023 if (CGM.supportsCOMDAT())
2024 Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName));
Mike Stump0c743272009-03-06 01:33:24 +00002025
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002026 IdentifierInfo *II = &C.Idents.get(FuncName);
Mike Stump0c743272009-03-06 01:33:24 +00002027
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002028 SmallVector<QualType, 2> ArgTys;
2029 ArgTys.push_back(C.VoidPtrTy);
2030 ArgTys.push_back(C.VoidPtrTy);
2031 QualType FunctionTy = C.getFunctionType(ReturnTy, ArgTys, {});
2032
2033 FunctionDecl *FD = FunctionDecl::Create(
2034 C, C.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
2035 FunctionTy, nullptr, SC_Static, false, false);
Akira Hatanakaaec6b2c2015-10-08 20:26:34 +00002036
Akira Hatanaka9978da32018-08-10 15:09:24 +00002037 setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,
2038 CGM);
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002039 StartFunction(FD, ReturnTy, Fn, FI, args);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002040 ApplyDebugLocation NL{*this, blockInfo.getBlockExpr()->getBeginLoc()};
Chris Lattner2192fe52011-07-18 04:24:23 +00002041 llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
Mike Stumpaeb0ffd2009-03-07 02:35:30 +00002042
Alexey Bataev56223232017-06-09 13:40:18 +00002043 Address src = GetAddrOfLocalVar(&SrcDecl);
John McCall7f416cc2015-09-08 08:05:57 +00002044 src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
John McCallad7c5c12011-02-08 08:22:06 +00002045 src = Builder.CreateBitCast(src, structPtrTy, "block.source");
Mike Stumpaeb0ffd2009-03-07 02:35:30 +00002046
Alexey Bataev56223232017-06-09 13:40:18 +00002047 Address dst = GetAddrOfLocalVar(&DstDecl);
John McCall7f416cc2015-09-08 08:05:57 +00002048 dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign);
John McCallad7c5c12011-02-08 08:22:06 +00002049 dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
Mike Stumpaeb0ffd2009-03-07 02:35:30 +00002050
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002051 for (const auto &CopiedCapture : CopiedCaptures) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00002052 const BlockDecl::Capture &CI = *CopiedCapture.CI;
2053 const CGBlockInfo::Capture &capture = *CopiedCapture.Capture;
Akira Hatanakacb6a9332018-07-26 16:51:21 +00002054 QualType captureType = CI.getVariable()->getType();
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002055 BlockFieldFlags flags = CopiedCapture.CopyFlags;
John McCall351762c2011-02-07 10:33:21 +00002056
2057 unsigned index = capture.getIndex();
James Y Knight751fe282019-02-09 22:22:28 +00002058 Address srcField = Builder.CreateStructGEP(src, index);
2059 Address dstField = Builder.CreateStructGEP(dst, index);
John McCall351762c2011-02-07 10:33:21 +00002060
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002061 switch (CopiedCapture.CopyKind) {
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002062 case BlockCaptureEntityKind::CXXRecord:
2063 // If there's an explicit copy expression, we do that.
2064 assert(CI.getCopyExpr() && "copy expression for variable is missing");
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002065 EmitSynthesizedCXXCopyCtor(dstField, srcField, CI.getCopyExpr());
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002066 break;
2067 case BlockCaptureEntityKind::ARCWeak:
John McCall31168b02011-06-15 23:02:42 +00002068 EmitARCCopyWeak(dstField, srcField);
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002069 break;
2070 case BlockCaptureEntityKind::NonTrivialCStruct: {
2071 // If this is a C struct that requires non-trivial copy construction,
2072 // emit a call to its copy constructor.
Akira Hatanaka7275da02018-02-28 07:15:55 +00002073 QualType varType = CI.getVariable()->getType();
2074 callCStructCopyConstructor(MakeAddrLValue(dstField, varType),
2075 MakeAddrLValue(srcField, varType));
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002076 break;
2077 }
2078 case BlockCaptureEntityKind::ARCStrong: {
John McCall351762c2011-02-07 10:33:21 +00002079 llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002080 // At -O0, store null into the destination field (so that the
2081 // storeStrong doesn't over-release) and then call storeStrong.
2082 // This is a workaround to not having an initStrong call.
2083 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2084 auto *ty = cast<llvm::PointerType>(srcValue->getType());
2085 llvm::Value *null = llvm::ConstantPointerNull::get(ty);
2086 Builder.CreateStore(null, dstField);
2087 EmitARCStoreStrongCall(dstField, srcValue, true);
John McCalle68b8f42012-10-17 02:28:37 +00002088
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002089 // With optimization enabled, take advantage of the fact that
2090 // the blocks runtime guarantees a memcpy of the block data, and
2091 // just emit a retain of the src field.
John McCalle68b8f42012-10-17 02:28:37 +00002092 } else {
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002093 EmitARCRetainNonBlock(srcValue);
John McCall882987f2013-02-28 19:01:20 +00002094
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002095 // Unless EH cleanup is required, we don't need this anymore, so kill
2096 // it. It's not quite worth the annoyance to avoid creating it in the
2097 // first place.
2098 if (!needsEHCleanup(captureType.isDestructedType()))
2099 cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent();
John McCalle68b8f42012-10-17 02:28:37 +00002100 }
Akira Hatanaka4a6f1902018-08-13 20:59:57 +00002101 break;
2102 }
2103 case BlockCaptureEntityKind::BlockObject: {
2104 llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
2105 srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy);
2106 llvm::Value *dstAddr =
2107 Builder.CreateBitCast(dstField.getPointer(), VoidPtrTy);
2108 llvm::Value *args[] = {
2109 dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
2110 };
2111
2112 if (CI.isByRef() && C.getBlockVarCopyInit(CI.getVariable()).canThrow())
2113 EmitRuntimeCallOrInvoke(CGM.getBlockObjectAssign(), args);
2114 else
2115 EmitNounwindRuntimeCall(CGM.getBlockObjectAssign(), args);
2116 break;
2117 }
2118 case BlockCaptureEntityKind::None:
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002119 continue;
Mike Stumpaeb0ffd2009-03-07 02:35:30 +00002120 }
Akira Hatanakacb6a9332018-07-26 16:51:21 +00002121
2122 // Ensure that we destroy the copied object if an exception is thrown later
2123 // in the helper function.
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002124 pushCaptureCleanup(CopiedCapture.CopyKind, dstField, captureType, flags,
Akira Hatanaka9978da32018-08-10 15:09:24 +00002125 /*ForCopyHelper*/ true, CI.getVariable(), *this);
Mike Stumpaeb0ffd2009-03-07 02:35:30 +00002126 }
2127
John McCallad7c5c12011-02-08 08:22:06 +00002128 FinishFunction();
Mike Stump0c743272009-03-06 01:33:24 +00002129
John McCalle3dc1702011-02-15 09:22:45 +00002130 return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
Mike Stump97d01d52009-03-04 03:23:46 +00002131}
2132
Akira Hatanaka7275da02018-02-28 07:15:55 +00002133static BlockFieldFlags
2134getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2135 QualType T) {
2136 BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2137 if (T->isBlockPointerType())
2138 Flags = BLOCK_FIELD_IS_BLOCK;
2139 return Flags;
2140}
2141
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002142static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2143computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2144 const LangOptions &LangOpts) {
Akira Hatanaka8e57b072018-10-01 21:51:28 +00002145 if (CI.isEscapingByref()) {
Akira Hatanaka7275da02018-02-28 07:15:55 +00002146 BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002147 if (T.isObjCGCWeak())
2148 Flags |= BLOCK_FIELD_IS_WEAK;
2149 return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);
2150 }
2151
Akira Hatanaka7275da02018-02-28 07:15:55 +00002152 switch (T.isDestructedType()) {
2153 case QualType::DK_cxx_destructor:
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002154 return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags());
Akira Hatanaka7275da02018-02-28 07:15:55 +00002155 case QualType::DK_objc_strong_lifetime:
2156 // Use objc_storeStrong for __strong direct captures; the
2157 // dynamic tools really like it when we do this.
2158 return std::make_pair(BlockCaptureEntityKind::ARCStrong,
2159 getBlockFieldFlagsForObjCObjectPointer(CI, T));
2160 case QualType::DK_objc_weak_lifetime:
2161 // Support __weak direct captures.
2162 return std::make_pair(BlockCaptureEntityKind::ARCWeak,
2163 getBlockFieldFlagsForObjCObjectPointer(CI, T));
2164 case QualType::DK_nontrivial_c_struct:
2165 return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct,
2166 BlockFieldFlags());
2167 case QualType::DK_none: {
2168 // Non-ARC captures are strong, and we need to use _Block_object_dispose.
2169 if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2170 !LangOpts.ObjCAutoRefCount)
2171 return std::make_pair(BlockCaptureEntityKind::BlockObject,
2172 getBlockFieldFlagsForObjCObjectPointer(CI, T));
2173 // Otherwise, we have nothing to do.
2174 return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002175 }
Akira Hatanaka7275da02018-02-28 07:15:55 +00002176 }
Nico Weberb3897eb2018-02-28 19:28:47 +00002177 llvm_unreachable("after exhaustive DestructionKind switch");
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002178}
2179
John McCallf593b102013-01-22 03:56:22 +00002180/// Generate the destroy-helper function for a block closure object:
2181/// static void block_destroy_helper(block_t *theBlock);
2182///
2183/// Note that this destroys a heap-allocated block closure object;
2184/// it should not be confused with a 'byref destroy helper', which
2185/// destroys the heap-allocated contents of an individual __block
2186/// variable.
John McCall351762c2011-02-07 10:33:21 +00002187llvm::Constant *
John McCallad7c5c12011-02-08 08:22:06 +00002188CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00002189 SmallVector<BlockCaptureManagedEntity, 4> DestroyedCaptures;
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002190 findBlockCapturedManagedEntities(blockInfo, getLangOpts(), DestroyedCaptures);
Akira Hatanaka9978da32018-08-10 15:09:24 +00002191 std::string FuncName =
2192 getCopyDestroyHelperFuncName(DestroyedCaptures, blockInfo.BlockAlign,
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002193 CaptureStrKind::DisposeHelper, CGM);
Akira Hatanaka9978da32018-08-10 15:09:24 +00002194
2195 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))
Akira Hatanaka936240c2018-08-14 00:15:42 +00002196 return llvm::ConstantExpr::getBitCast(Func, VoidPtrTy);
Akira Hatanaka9978da32018-08-10 15:09:24 +00002197
John McCall351762c2011-02-07 10:33:21 +00002198 ASTContext &C = getContext();
Mike Stump0c743272009-03-06 01:33:24 +00002199
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002200 QualType ReturnTy = C.VoidTy;
2201
John McCall351762c2011-02-07 10:33:21 +00002202 FunctionArgList args;
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002203 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev56223232017-06-09 13:40:18 +00002204 args.push_back(&SrcDecl);
Mike Stump11289f42009-09-09 15:08:12 +00002205
John McCallc56a8b32016-03-11 04:30:31 +00002206 const CGFunctionInfo &FI =
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002207 CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);
Mike Stump0c743272009-03-06 01:33:24 +00002208
Mike Stumpcbc2bca2009-06-05 23:26:36 +00002209 // FIXME: We'd like to put these into a mergable by content, with
2210 // internal linkage.
John McCalla729c622012-02-17 03:33:10 +00002211 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);
Mike Stump0c743272009-03-06 01:33:24 +00002212
2213 llvm::Function *Fn =
Akira Hatanaka9978da32018-08-10 15:09:24 +00002214 llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,
2215 FuncName, &CGM.getModule());
Saleem Abdulrasool89628922019-02-22 16:29:50 +00002216 if (CGM.supportsCOMDAT())
2217 Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName));
Mike Stump0c743272009-03-06 01:33:24 +00002218
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002219 IdentifierInfo *II = &C.Idents.get(FuncName);
Mike Stump0c743272009-03-06 01:33:24 +00002220
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002221 SmallVector<QualType, 1> ArgTys;
2222 ArgTys.push_back(C.VoidPtrTy);
2223 QualType FunctionTy = C.getFunctionType(ReturnTy, ArgTys, {});
2224
2225 FunctionDecl *FD = FunctionDecl::Create(
2226 C, C.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
2227 FunctionTy, nullptr, SC_Static, false, false);
Akira Hatanakaaec6b2c2015-10-08 20:26:34 +00002228
Akira Hatanaka9978da32018-08-10 15:09:24 +00002229 setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,
2230 CGM);
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002231 StartFunction(FD, ReturnTy, Fn, FI, args);
Akira Hatanaka9978da32018-08-10 15:09:24 +00002232 markAsIgnoreThreadCheckingAtRuntime(Fn);
2233
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002234 ApplyDebugLocation NL{*this, blockInfo.getBlockExpr()->getBeginLoc()};
Mike Stump6f7d9f82009-03-07 02:53:18 +00002235
Chris Lattner2192fe52011-07-18 04:24:23 +00002236 llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
Mike Stump6f7d9f82009-03-07 02:53:18 +00002237
Alexey Bataev56223232017-06-09 13:40:18 +00002238 Address src = GetAddrOfLocalVar(&SrcDecl);
John McCall7f416cc2015-09-08 08:05:57 +00002239 src = Address(Builder.CreateLoad(src), blockInfo.BlockAlign);
John McCallad7c5c12011-02-08 08:22:06 +00002240 src = Builder.CreateBitCast(src, structPtrTy, "block");
Mike Stump6f7d9f82009-03-07 02:53:18 +00002241
John McCallad7c5c12011-02-08 08:22:06 +00002242 CodeGenFunction::RunCleanupsScope cleanups(*this);
John McCall351762c2011-02-07 10:33:21 +00002243
Alex Lorenze08e5bc2017-03-06 16:23:04 +00002244 for (const auto &DestroyedCapture : DestroyedCaptures) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00002245 const BlockDecl::Capture &CI = *DestroyedCapture.CI;
2246 const CGBlockInfo::Capture &capture = *DestroyedCapture.Capture;
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002247 BlockFieldFlags flags = DestroyedCapture.DisposeFlags;
John McCall351762c2011-02-07 10:33:21 +00002248
James Y Knight751fe282019-02-09 22:22:28 +00002249 Address srcField = Builder.CreateStructGEP(src, capture.getIndex());
John McCall351762c2011-02-07 10:33:21 +00002250
Akira Hatanaka2ec36f02018-08-17 15:46:07 +00002251 pushCaptureCleanup(DestroyedCapture.DisposeKind, srcField,
Akira Hatanaka9978da32018-08-10 15:09:24 +00002252 CI.getVariable()->getType(), flags,
2253 /*ForCopyHelper*/ false, CI.getVariable(), *this);
Mike Stump6f7d9f82009-03-07 02:53:18 +00002254 }
2255
John McCall351762c2011-02-07 10:33:21 +00002256 cleanups.ForceCleanup();
2257
John McCallad7c5c12011-02-08 08:22:06 +00002258 FinishFunction();
Mike Stump0c743272009-03-06 01:33:24 +00002259
John McCalle3dc1702011-02-15 09:22:45 +00002260 return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
Mike Stump0c743272009-03-06 01:33:24 +00002261}
2262
John McCallf9b056b2011-03-31 08:03:29 +00002263namespace {
2264
2265/// Emits the copy/dispose helper functions for a __block object of id type.
John McCall7f416cc2015-09-08 08:05:57 +00002266class ObjectByrefHelpers final : public BlockByrefHelpers {
John McCallf9b056b2011-03-31 08:03:29 +00002267 BlockFieldFlags Flags;
2268
2269public:
2270 ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
John McCall7f416cc2015-09-08 08:05:57 +00002271 : BlockByrefHelpers(alignment), Flags(flags) {}
John McCallf9b056b2011-03-31 08:03:29 +00002272
John McCall7f416cc2015-09-08 08:05:57 +00002273 void emitCopy(CodeGenFunction &CGF, Address destField,
2274 Address srcField) override {
John McCallf9b056b2011-03-31 08:03:29 +00002275 destField = CGF.Builder.CreateBitCast(destField, CGF.VoidPtrTy);
2276
2277 srcField = CGF.Builder.CreateBitCast(srcField, CGF.VoidPtrPtrTy);
2278 llvm::Value *srcValue = CGF.Builder.CreateLoad(srcField);
2279
2280 unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2281
2282 llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);
James Y Knight9871db02019-02-05 16:42:33 +00002283 llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
John McCall882987f2013-02-28 19:01:20 +00002284
John McCall7f416cc2015-09-08 08:05:57 +00002285 llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal };
John McCall882987f2013-02-28 19:01:20 +00002286 CGF.EmitNounwindRuntimeCall(fn, args);
John McCallf9b056b2011-03-31 08:03:29 +00002287 }
2288
John McCall7f416cc2015-09-08 08:05:57 +00002289 void emitDispose(CodeGenFunction &CGF, Address field) override {
John McCallf9b056b2011-03-31 08:03:29 +00002290 field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0));
2291 llvm::Value *value = CGF.Builder.CreateLoad(field);
2292
Akira Hatanaka9978da32018-08-10 15:09:24 +00002293 CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER, false);
John McCallf9b056b2011-03-31 08:03:29 +00002294 }
2295
Craig Topper4f12f102014-03-12 06:41:41 +00002296 void profileImpl(llvm::FoldingSetNodeID &id) const override {
John McCallf9b056b2011-03-31 08:03:29 +00002297 id.AddInteger(Flags.getBitMask());
2298 }
2299};
2300
John McCall31168b02011-06-15 23:02:42 +00002301/// Emits the copy/dispose helpers for an ARC __block __weak variable.
John McCall7f416cc2015-09-08 08:05:57 +00002302class ARCWeakByrefHelpers final : public BlockByrefHelpers {
John McCall31168b02011-06-15 23:02:42 +00002303public:
John McCall7f416cc2015-09-08 08:05:57 +00002304 ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
John McCall31168b02011-06-15 23:02:42 +00002305
John McCall7f416cc2015-09-08 08:05:57 +00002306 void emitCopy(CodeGenFunction &CGF, Address destField,
2307 Address srcField) override {
John McCall31168b02011-06-15 23:02:42 +00002308 CGF.EmitARCMoveWeak(destField, srcField);
2309 }
2310
John McCall7f416cc2015-09-08 08:05:57 +00002311 void emitDispose(CodeGenFunction &CGF, Address field) override {
John McCall31168b02011-06-15 23:02:42 +00002312 CGF.EmitARCDestroyWeak(field);
2313 }
2314
Craig Topper4f12f102014-03-12 06:41:41 +00002315 void profileImpl(llvm::FoldingSetNodeID &id) const override {
John McCall31168b02011-06-15 23:02:42 +00002316 // 0 is distinguishable from all pointers and byref flags
2317 id.AddInteger(0);
2318 }
2319};
2320
2321/// Emits the copy/dispose helpers for an ARC __block __strong variable
2322/// that's not of block-pointer type.
John McCall7f416cc2015-09-08 08:05:57 +00002323class ARCStrongByrefHelpers final : public BlockByrefHelpers {
John McCall31168b02011-06-15 23:02:42 +00002324public:
John McCall7f416cc2015-09-08 08:05:57 +00002325 ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
John McCall31168b02011-06-15 23:02:42 +00002326
John McCall7f416cc2015-09-08 08:05:57 +00002327 void emitCopy(CodeGenFunction &CGF, Address destField,
2328 Address srcField) override {
John McCall31168b02011-06-15 23:02:42 +00002329 // Do a "move" by copying the value and then zeroing out the old
2330 // variable.
2331
John McCall7f416cc2015-09-08 08:05:57 +00002332 llvm::Value *value = CGF.Builder.CreateLoad(srcField);
Fangrui Song6907ce22018-07-30 19:24:48 +00002333
John McCall31168b02011-06-15 23:02:42 +00002334 llvm::Value *null =
2335 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(value->getType()));
John McCall3a237aa2011-11-09 03:17:26 +00002336
Fariborz Jahaniana82e9262013-01-04 23:32:24 +00002337 if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
John McCall7f416cc2015-09-08 08:05:57 +00002338 CGF.Builder.CreateStore(null, destField);
Fariborz Jahaniana82e9262013-01-04 23:32:24 +00002339 CGF.EmitARCStoreStrongCall(destField, value, /*ignored*/ true);
2340 CGF.EmitARCStoreStrongCall(srcField, null, /*ignored*/ true);
2341 return;
2342 }
John McCall7f416cc2015-09-08 08:05:57 +00002343 CGF.Builder.CreateStore(value, destField);
2344 CGF.Builder.CreateStore(null, srcField);
John McCall31168b02011-06-15 23:02:42 +00002345 }
2346
John McCall7f416cc2015-09-08 08:05:57 +00002347 void emitDispose(CodeGenFunction &CGF, Address field) override {
John McCallcdda29c2013-03-13 03:10:54 +00002348 CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);
John McCall31168b02011-06-15 23:02:42 +00002349 }
2350
Craig Topper4f12f102014-03-12 06:41:41 +00002351 void profileImpl(llvm::FoldingSetNodeID &id) const override {
John McCall31168b02011-06-15 23:02:42 +00002352 // 1 is distinguishable from all pointers and byref flags
2353 id.AddInteger(1);
2354 }
2355};
2356
John McCall3a237aa2011-11-09 03:17:26 +00002357/// Emits the copy/dispose helpers for an ARC __block __strong
2358/// variable that's of block-pointer type.
John McCall7f416cc2015-09-08 08:05:57 +00002359class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
John McCall3a237aa2011-11-09 03:17:26 +00002360public:
John McCall7f416cc2015-09-08 08:05:57 +00002361 ARCStrongBlockByrefHelpers(CharUnits alignment)
2362 : BlockByrefHelpers(alignment) {}
John McCall3a237aa2011-11-09 03:17:26 +00002363
John McCall7f416cc2015-09-08 08:05:57 +00002364 void emitCopy(CodeGenFunction &CGF, Address destField,
2365 Address srcField) override {
John McCall3a237aa2011-11-09 03:17:26 +00002366 // Do the copy with objc_retainBlock; that's all that
2367 // _Block_object_assign would do anyway, and we'd have to pass the
2368 // right arguments to make sure it doesn't get no-op'ed.
John McCall7f416cc2015-09-08 08:05:57 +00002369 llvm::Value *oldValue = CGF.Builder.CreateLoad(srcField);
John McCall3a237aa2011-11-09 03:17:26 +00002370 llvm::Value *copy = CGF.EmitARCRetainBlock(oldValue, /*mandatory*/ true);
John McCall7f416cc2015-09-08 08:05:57 +00002371 CGF.Builder.CreateStore(copy, destField);
John McCall3a237aa2011-11-09 03:17:26 +00002372 }
2373
John McCall7f416cc2015-09-08 08:05:57 +00002374 void emitDispose(CodeGenFunction &CGF, Address field) override {
John McCallcdda29c2013-03-13 03:10:54 +00002375 CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);
John McCall3a237aa2011-11-09 03:17:26 +00002376 }
2377
Craig Topper4f12f102014-03-12 06:41:41 +00002378 void profileImpl(llvm::FoldingSetNodeID &id) const override {
John McCall3a237aa2011-11-09 03:17:26 +00002379 // 2 is distinguishable from all pointers and byref flags
2380 id.AddInteger(2);
2381 }
2382};
2383
John McCallf9b056b2011-03-31 08:03:29 +00002384/// Emits the copy/dispose helpers for a __block variable with a
2385/// nontrivial copy constructor or destructor.
John McCall7f416cc2015-09-08 08:05:57 +00002386class CXXByrefHelpers final : public BlockByrefHelpers {
John McCallf9b056b2011-03-31 08:03:29 +00002387 QualType VarType;
2388 const Expr *CopyExpr;
2389
2390public:
2391 CXXByrefHelpers(CharUnits alignment, QualType type,
2392 const Expr *copyExpr)
John McCall7f416cc2015-09-08 08:05:57 +00002393 : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
John McCallf9b056b2011-03-31 08:03:29 +00002394
Craig Topper8a13c412014-05-21 05:09:00 +00002395 bool needsCopy() const override { return CopyExpr != nullptr; }
John McCall7f416cc2015-09-08 08:05:57 +00002396 void emitCopy(CodeGenFunction &CGF, Address destField,
2397 Address srcField) override {
John McCallf9b056b2011-03-31 08:03:29 +00002398 if (!CopyExpr) return;
2399 CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr);
2400 }
2401
John McCall7f416cc2015-09-08 08:05:57 +00002402 void emitDispose(CodeGenFunction &CGF, Address field) override {
John McCallf9b056b2011-03-31 08:03:29 +00002403 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2404 CGF.PushDestructorCleanup(VarType, field);
2405 CGF.PopCleanupBlocks(cleanupDepth);
2406 }
2407
Craig Topper4f12f102014-03-12 06:41:41 +00002408 void profileImpl(llvm::FoldingSetNodeID &id) const override {
John McCallf9b056b2011-03-31 08:03:29 +00002409 id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());
2410 }
2411};
Akira Hatanaka7275da02018-02-28 07:15:55 +00002412
2413/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2414/// C struct.
2415class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2416 QualType VarType;
2417
2418public:
2419 NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2420 : BlockByrefHelpers(alignment), VarType(type) {}
2421
2422 void emitCopy(CodeGenFunction &CGF, Address destField,
2423 Address srcField) override {
2424 CGF.callCStructMoveConstructor(CGF.MakeAddrLValue(destField, VarType),
2425 CGF.MakeAddrLValue(srcField, VarType));
2426 }
2427
2428 bool needsDispose() const override {
2429 return VarType.isDestructedType();
2430 }
2431
2432 void emitDispose(CodeGenFunction &CGF, Address field) override {
2433 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2434 CGF.pushDestroy(VarType.isDestructedType(), field, VarType);
2435 CGF.PopCleanupBlocks(cleanupDepth);
2436 }
2437
2438 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2439 id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());
2440 }
2441};
John McCallf9b056b2011-03-31 08:03:29 +00002442} // end anonymous namespace
2443
2444static llvm::Constant *
John McCall7f416cc2015-09-08 08:05:57 +00002445generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2446 BlockByrefHelpers &generator) {
John McCallf9b056b2011-03-31 08:03:29 +00002447 ASTContext &Context = CGF.getContext();
2448
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002449 QualType ReturnTy = Context.VoidTy;
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002450
John McCalla738c252011-03-09 04:27:21 +00002451 FunctionArgList args;
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002452 ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev56223232017-06-09 13:40:18 +00002453 args.push_back(&Dst);
Mike Stumpf89230d2009-03-06 06:12:24 +00002454
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002455 ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamDecl::Other);
Alexey Bataev56223232017-06-09 13:40:18 +00002456 args.push_back(&Src);
Mike Stump11289f42009-09-09 15:08:12 +00002457
John McCallc56a8b32016-03-11 04:30:31 +00002458 const CGFunctionInfo &FI =
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002459 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002460
John McCall7f416cc2015-09-08 08:05:57 +00002461 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002462
Mike Stumpcbc2bca2009-06-05 23:26:36 +00002463 // FIXME: We'd like to put these into a mergable by content, with
2464 // internal linkage.
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002465 llvm::Function *Fn =
2466 llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
John McCallf9b056b2011-03-31 08:03:29 +00002467 "__Block_byref_object_copy_", &CGF.CGM.getModule());
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002468
2469 IdentifierInfo *II
John McCallf9b056b2011-03-31 08:03:29 +00002470 = &Context.Idents.get("__Block_byref_object_copy_");
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002471
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002472 SmallVector<QualType, 2> ArgTys;
2473 ArgTys.push_back(Context.VoidPtrTy);
2474 ArgTys.push_back(Context.VoidPtrTy);
2475 QualType FunctionTy = Context.getFunctionType(ReturnTy, ArgTys, {});
2476
2477 FunctionDecl *FD = FunctionDecl::Create(
2478 Context, Context.getTranslationUnitDecl(), SourceLocation(),
2479 SourceLocation(), II, FunctionTy, nullptr, SC_Static, false, false);
John McCall31168b02011-06-15 23:02:42 +00002480
Rafael Espindola51ec5a92018-02-28 23:46:35 +00002481 CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
Akira Hatanakaaec6b2c2015-10-08 20:26:34 +00002482
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002483 CGF.StartFunction(FD, ReturnTy, Fn, FI, args);
Mike Stumpf89230d2009-03-06 06:12:24 +00002484
John McCall7f416cc2015-09-08 08:05:57 +00002485 if (generator.needsCopy()) {
2486 llvm::Type *byrefPtrType = byrefInfo.Type->getPointerTo(0);
Mike Stumpf89230d2009-03-06 06:12:24 +00002487
John McCallf9b056b2011-03-31 08:03:29 +00002488 // dst->x
Alexey Bataev56223232017-06-09 13:40:18 +00002489 Address destField = CGF.GetAddrOfLocalVar(&Dst);
John McCall7f416cc2015-09-08 08:05:57 +00002490 destField = Address(CGF.Builder.CreateLoad(destField),
2491 byrefInfo.ByrefAlignment);
John McCallf9b056b2011-03-31 08:03:29 +00002492 destField = CGF.Builder.CreateBitCast(destField, byrefPtrType);
John McCall7f416cc2015-09-08 08:05:57 +00002493 destField = CGF.emitBlockByrefAddress(destField, byrefInfo, false,
2494 "dest-object");
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002495
John McCallf9b056b2011-03-31 08:03:29 +00002496 // src->x
Alexey Bataev56223232017-06-09 13:40:18 +00002497 Address srcField = CGF.GetAddrOfLocalVar(&Src);
John McCall7f416cc2015-09-08 08:05:57 +00002498 srcField = Address(CGF.Builder.CreateLoad(srcField),
2499 byrefInfo.ByrefAlignment);
John McCallf9b056b2011-03-31 08:03:29 +00002500 srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType);
John McCall7f416cc2015-09-08 08:05:57 +00002501 srcField = CGF.emitBlockByrefAddress(srcField, byrefInfo, false,
2502 "src-object");
John McCallf9b056b2011-03-31 08:03:29 +00002503
John McCall7f416cc2015-09-08 08:05:57 +00002504 generator.emitCopy(CGF, destField, srcField);
Fangrui Song6907ce22018-07-30 19:24:48 +00002505 }
John McCallf9b056b2011-03-31 08:03:29 +00002506
2507 CGF.FinishFunction();
2508
2509 return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002510}
2511
John McCallf9b056b2011-03-31 08:03:29 +00002512/// Build the copy helper for a __block variable.
2513static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
John McCall7f416cc2015-09-08 08:05:57 +00002514 const BlockByrefInfo &byrefInfo,
2515 BlockByrefHelpers &generator) {
John McCallf9b056b2011-03-31 08:03:29 +00002516 CodeGenFunction CGF(CGM);
John McCall7f416cc2015-09-08 08:05:57 +00002517 return generateByrefCopyHelper(CGF, byrefInfo, generator);
John McCallf9b056b2011-03-31 08:03:29 +00002518}
2519
2520/// Generate code for a __block variable's dispose helper.
2521static llvm::Constant *
2522generateByrefDisposeHelper(CodeGenFunction &CGF,
John McCall7f416cc2015-09-08 08:05:57 +00002523 const BlockByrefInfo &byrefInfo,
2524 BlockByrefHelpers &generator) {
John McCallf9b056b2011-03-31 08:03:29 +00002525 ASTContext &Context = CGF.getContext();
2526 QualType R = Context.VoidTy;
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002527
John McCalla738c252011-03-09 04:27:21 +00002528 FunctionArgList args;
Alexey Bataev56223232017-06-09 13:40:18 +00002529 ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy,
2530 ImplicitParamDecl::Other);
2531 args.push_back(&Src);
Mike Stump11289f42009-09-09 15:08:12 +00002532
John McCallc56a8b32016-03-11 04:30:31 +00002533 const CGFunctionInfo &FI =
2534 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(R, args);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002535
John McCall7f416cc2015-09-08 08:05:57 +00002536 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002537
Mike Stumpcbc2bca2009-06-05 23:26:36 +00002538 // FIXME: We'd like to put these into a mergable by content, with
2539 // internal linkage.
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002540 llvm::Function *Fn =
2541 llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
Fariborz Jahanian50198092010-12-02 17:02:11 +00002542 "__Block_byref_object_dispose_",
John McCallf9b056b2011-03-31 08:03:29 +00002543 &CGF.CGM.getModule());
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002544
2545 IdentifierInfo *II
John McCallf9b056b2011-03-31 08:03:29 +00002546 = &Context.Idents.get("__Block_byref_object_dispose_");
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002547
Jonas Devlieghere64a26302018-11-11 00:56:15 +00002548 SmallVector<QualType, 1> ArgTys;
2549 ArgTys.push_back(Context.VoidPtrTy);
2550 QualType FunctionTy = Context.getFunctionType(R, ArgTys, {});
2551
2552 FunctionDecl *FD = FunctionDecl::Create(
2553 Context, Context.getTranslationUnitDecl(), SourceLocation(),
2554 SourceLocation(), II, FunctionTy, nullptr, SC_Static, false, false);
Akira Hatanakaaec6b2c2015-10-08 20:26:34 +00002555
Rafael Espindola51ec5a92018-02-28 23:46:35 +00002556 CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
Akira Hatanakaaec6b2c2015-10-08 20:26:34 +00002557
Adrian Prantl22e66b42014-04-11 01:13:04 +00002558 CGF.StartFunction(FD, R, Fn, FI, args);
Mike Stumpfbe25dd2009-03-06 04:53:30 +00002559
John McCall7f416cc2015-09-08 08:05:57 +00002560 if (generator.needsDispose()) {
Alexey Bataev56223232017-06-09 13:40:18 +00002561 Address addr = CGF.GetAddrOfLocalVar(&Src);
John McCall7f416cc2015-09-08 08:05:57 +00002562 addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.ByrefAlignment);
2563 auto byrefPtrType = byrefInfo.Type->getPointerTo(0);
2564 addr = CGF.Builder.CreateBitCast(addr, byrefPtrType);
2565 addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object");
John McCallad7c5c12011-02-08 08:22:06 +00002566
John McCall7f416cc2015-09-08 08:05:57 +00002567 generator.emitDispose(CGF, addr);
Fariborz Jahanian50198092010-12-02 17:02:11 +00002568 }
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002569
John McCallf9b056b2011-03-31 08:03:29 +00002570 CGF.FinishFunction();
John McCallad7c5c12011-02-08 08:22:06 +00002571
John McCallf9b056b2011-03-31 08:03:29 +00002572 return llvm::ConstantExpr::getBitCast(Fn, CGF.Int8PtrTy);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002573}
2574
John McCallf9b056b2011-03-31 08:03:29 +00002575/// Build the dispose helper for a __block variable.
2576static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
John McCall7f416cc2015-09-08 08:05:57 +00002577 const BlockByrefInfo &byrefInfo,
2578 BlockByrefHelpers &generator) {
John McCallf9b056b2011-03-31 08:03:29 +00002579 CodeGenFunction CGF(CGM);
John McCall7f416cc2015-09-08 08:05:57 +00002580 return generateByrefDisposeHelper(CGF, byrefInfo, generator);
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002581}
2582
John McCallf593b102013-01-22 03:56:22 +00002583/// Lazily build the copy and dispose helpers for a __block variable
2584/// with the given information.
David Blaikie92551612015-08-13 23:53:09 +00002585template <class T>
John McCall7f416cc2015-09-08 08:05:57 +00002586static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2587 T &&generator) {
John McCallf9b056b2011-03-31 08:03:29 +00002588 llvm::FoldingSetNodeID id;
John McCall7f416cc2015-09-08 08:05:57 +00002589 generator.Profile(id);
John McCallf9b056b2011-03-31 08:03:29 +00002590
2591 void *insertPos;
John McCall7f416cc2015-09-08 08:05:57 +00002592 BlockByrefHelpers *node
John McCallf9b056b2011-03-31 08:03:29 +00002593 = CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos);
2594 if (node) return static_cast<T*>(node);
2595
John McCall7f416cc2015-09-08 08:05:57 +00002596 generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2597 generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
John McCallf9b056b2011-03-31 08:03:29 +00002598
Malcolm Parsonsf92d44c2016-12-06 14:49:18 +00002599 T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
John McCallf9b056b2011-03-31 08:03:29 +00002600 CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
2601 return copy;
2602}
2603
John McCallf593b102013-01-22 03:56:22 +00002604/// Build the copy and dispose helpers for the given __block variable
2605/// emission. Places the helpers in the global cache. Returns null
2606/// if no helpers are required.
John McCall7f416cc2015-09-08 08:05:57 +00002607BlockByrefHelpers *
Chris Lattner2192fe52011-07-18 04:24:23 +00002608CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
John McCallf9b056b2011-03-31 08:03:29 +00002609 const AutoVarEmission &emission) {
2610 const VarDecl &var = *emission.Variable;
Akira Hatanaka8e57b072018-10-01 21:51:28 +00002611 assert(var.isEscapingByref() &&
2612 "only escaping __block variables need byref helpers");
2613
John McCallf9b056b2011-03-31 08:03:29 +00002614 QualType type = var.getType();
2615
John McCall7f416cc2015-09-08 08:05:57 +00002616 auto &byrefInfo = getBlockByrefInfo(&var);
2617
2618 // The alignment we care about for the purposes of uniquing byref
2619 // helpers is the alignment of the actual byref value field.
2620 CharUnits valueAlignment =
2621 byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset);
John McCallf593b102013-01-22 03:56:22 +00002622
John McCallf9b056b2011-03-31 08:03:29 +00002623 if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
Akira Hatanaka9978da32018-08-10 15:09:24 +00002624 const Expr *copyExpr =
2625 CGM.getContext().getBlockVarCopyInit(&var).getCopyExpr();
Craig Topper8a13c412014-05-21 05:09:00 +00002626 if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
John McCallf9b056b2011-03-31 08:03:29 +00002627
David Blaikie92551612015-08-13 23:53:09 +00002628 return ::buildByrefHelpers(
John McCall7f416cc2015-09-08 08:05:57 +00002629 CGM, byrefInfo, CXXByrefHelpers(valueAlignment, type, copyExpr));
John McCallf9b056b2011-03-31 08:03:29 +00002630 }
2631
Akira Hatanaka7275da02018-02-28 07:15:55 +00002632 // If type is a non-trivial C struct type that is non-trivial to
2633 // destructly move or destroy, build the copy and dispose helpers.
2634 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2635 type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2636 return ::buildByrefHelpers(
2637 CGM, byrefInfo, NonTrivialCStructByrefHelpers(valueAlignment, type));
2638
John McCall31168b02011-06-15 23:02:42 +00002639 // Otherwise, if we don't have a retainable type, there's nothing to do.
2640 // that the runtime does extra copies.
Craig Topper8a13c412014-05-21 05:09:00 +00002641 if (!type->isObjCRetainableType()) return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002642
2643 Qualifiers qs = type.getQualifiers();
2644
2645 // If we have lifetime, that dominates.
2646 if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
John McCall31168b02011-06-15 23:02:42 +00002647 switch (lifetime) {
2648 case Qualifiers::OCL_None: llvm_unreachable("impossible");
2649
2650 // These are just bits as far as the runtime is concerned.
2651 case Qualifiers::OCL_ExplicitNone:
2652 case Qualifiers::OCL_Autoreleasing:
Craig Topper8a13c412014-05-21 05:09:00 +00002653 return nullptr;
John McCall31168b02011-06-15 23:02:42 +00002654
2655 // Tell the runtime that this is ARC __weak, called by the
2656 // byref routines.
David Blaikie92551612015-08-13 23:53:09 +00002657 case Qualifiers::OCL_Weak:
John McCall7f416cc2015-09-08 08:05:57 +00002658 return ::buildByrefHelpers(CGM, byrefInfo,
2659 ARCWeakByrefHelpers(valueAlignment));
John McCall31168b02011-06-15 23:02:42 +00002660
2661 // ARC __strong __block variables need to be retained.
2662 case Qualifiers::OCL_Strong:
John McCall3a237aa2011-11-09 03:17:26 +00002663 // Block pointers need to be copied, and there's no direct
2664 // transfer possible.
John McCall31168b02011-06-15 23:02:42 +00002665 if (type->isBlockPointerType()) {
John McCall7f416cc2015-09-08 08:05:57 +00002666 return ::buildByrefHelpers(CGM, byrefInfo,
2667 ARCStrongBlockByrefHelpers(valueAlignment));
John McCall31168b02011-06-15 23:02:42 +00002668
2669 // Otherwise, we transfer ownership of the retain from the stack
2670 // to the heap.
2671 } else {
John McCall7f416cc2015-09-08 08:05:57 +00002672 return ::buildByrefHelpers(CGM, byrefInfo,
2673 ARCStrongByrefHelpers(valueAlignment));
John McCall31168b02011-06-15 23:02:42 +00002674 }
2675 }
2676 llvm_unreachable("fell out of lifetime switch!");
2677 }
2678
John McCallf9b056b2011-03-31 08:03:29 +00002679 BlockFieldFlags flags;
2680 if (type->isBlockPointerType()) {
2681 flags |= BLOCK_FIELD_IS_BLOCK;
Fangrui Song6907ce22018-07-30 19:24:48 +00002682 } else if (CGM.getContext().isObjCNSObjectType(type) ||
John McCallf9b056b2011-03-31 08:03:29 +00002683 type->isObjCObjectPointerType()) {
2684 flags |= BLOCK_FIELD_IS_OBJECT;
2685 } else {
Craig Topper8a13c412014-05-21 05:09:00 +00002686 return nullptr;
John McCallf9b056b2011-03-31 08:03:29 +00002687 }
2688
2689 if (type.isObjCGCWeak())
2690 flags |= BLOCK_FIELD_IS_WEAK;
2691
John McCall7f416cc2015-09-08 08:05:57 +00002692 return ::buildByrefHelpers(CGM, byrefInfo,
2693 ObjectByrefHelpers(valueAlignment, flags));
Mike Stumpee2a5ee2009-03-06 02:29:21 +00002694}
2695
John McCall7f416cc2015-09-08 08:05:57 +00002696Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2697 const VarDecl *var,
2698 bool followForward) {
2699 auto &info = getBlockByrefInfo(var);
2700 return emitBlockByrefAddress(baseAddr, info, followForward, var->getName());
John McCall73064872011-03-31 01:59:53 +00002701}
2702
John McCall7f416cc2015-09-08 08:05:57 +00002703Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2704 const BlockByrefInfo &info,
2705 bool followForward,
2706 const llvm::Twine &name) {
2707 // Chase the forwarding address if requested.
2708 if (followForward) {
James Y Knight751fe282019-02-09 22:22:28 +00002709 Address forwardingAddr = Builder.CreateStructGEP(baseAddr, 1, "forwarding");
John McCall7f416cc2015-09-08 08:05:57 +00002710 baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.ByrefAlignment);
2711 }
2712
James Y Knight751fe282019-02-09 22:22:28 +00002713 return Builder.CreateStructGEP(baseAddr, info.FieldIndex, name);
John McCall73064872011-03-31 01:59:53 +00002714}
2715
John McCall7f416cc2015-09-08 08:05:57 +00002716/// BuildByrefInfo - This routine changes a __block variable declared as T x
John McCall73064872011-03-31 01:59:53 +00002717/// into:
2718///
2719/// struct {
2720/// void *__isa;
2721/// void *__forwarding;
2722/// int32_t __flags;
2723/// int32_t __size;
2724/// void *__copy_helper; // only if needed
2725/// void *__destroy_helper; // only if needed
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002726/// void *__byref_variable_layout;// only if needed
John McCall73064872011-03-31 01:59:53 +00002727/// char padding[X]; // only if needed
2728/// T x;
2729/// } x
2730///
John McCall7f416cc2015-09-08 08:05:57 +00002731const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2732 auto it = BlockByrefInfos.find(D);
2733 if (it != BlockByrefInfos.end())
2734 return it->second;
John McCall73064872011-03-31 01:59:53 +00002735
John McCall7f416cc2015-09-08 08:05:57 +00002736 llvm::StructType *byrefType =
Chris Lattner5ec04a52011-08-12 17:43:31 +00002737 llvm::StructType::create(getLLVMContext(),
2738 "struct.__block_byref_" + D->getNameAsString());
Fangrui Song6907ce22018-07-30 19:24:48 +00002739
John McCall7f416cc2015-09-08 08:05:57 +00002740 QualType Ty = D->getType();
2741
2742 CharUnits size;
2743 SmallVector<llvm::Type *, 8> types;
Fangrui Song6907ce22018-07-30 19:24:48 +00002744
John McCall73064872011-03-31 01:59:53 +00002745 // void *__isa;
John McCall9dc0db22011-05-15 01:53:33 +00002746 types.push_back(Int8PtrTy);
John McCall7f416cc2015-09-08 08:05:57 +00002747 size += getPointerSize();
Fangrui Song6907ce22018-07-30 19:24:48 +00002748
John McCall73064872011-03-31 01:59:53 +00002749 // void *__forwarding;
John McCall7f416cc2015-09-08 08:05:57 +00002750 types.push_back(llvm::PointerType::getUnqual(byrefType));
2751 size += getPointerSize();
Fangrui Song6907ce22018-07-30 19:24:48 +00002752
John McCall73064872011-03-31 01:59:53 +00002753 // int32_t __flags;
John McCall9dc0db22011-05-15 01:53:33 +00002754 types.push_back(Int32Ty);
John McCall7f416cc2015-09-08 08:05:57 +00002755 size += CharUnits::fromQuantity(4);
Fangrui Song6907ce22018-07-30 19:24:48 +00002756
John McCall73064872011-03-31 01:59:53 +00002757 // int32_t __size;
John McCall9dc0db22011-05-15 01:53:33 +00002758 types.push_back(Int32Ty);
John McCall7f416cc2015-09-08 08:05:57 +00002759 size += CharUnits::fromQuantity(4);
2760
Fariborz Jahanian998f0a32012-11-28 23:12:17 +00002761 // Note that this must match *exactly* the logic in buildByrefHelpers.
John McCall7f416cc2015-09-08 08:05:57 +00002762 bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2763 if (hasCopyAndDispose) {
John McCall73064872011-03-31 01:59:53 +00002764 /// void *__copy_helper;
John McCall9dc0db22011-05-15 01:53:33 +00002765 types.push_back(Int8PtrTy);
John McCall7f416cc2015-09-08 08:05:57 +00002766 size += getPointerSize();
Fangrui Song6907ce22018-07-30 19:24:48 +00002767
John McCall73064872011-03-31 01:59:53 +00002768 /// void *__destroy_helper;
John McCall9dc0db22011-05-15 01:53:33 +00002769 types.push_back(Int8PtrTy);
John McCall7f416cc2015-09-08 08:05:57 +00002770 size += getPointerSize();
John McCall73064872011-03-31 01:59:53 +00002771 }
John McCall7f416cc2015-09-08 08:05:57 +00002772
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002773 bool HasByrefExtendedLayout = false;
2774 Qualifiers::ObjCLifetime Lifetime;
2775 if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
John McCall7f416cc2015-09-08 08:05:57 +00002776 HasByrefExtendedLayout) {
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002777 /// void *__byref_variable_layout;
2778 types.push_back(Int8PtrTy);
John McCall7f416cc2015-09-08 08:05:57 +00002779 size += CharUnits::fromQuantity(PointerSizeInBytes);
John McCall73064872011-03-31 01:59:53 +00002780 }
2781
2782 // T x;
John McCall7f416cc2015-09-08 08:05:57 +00002783 llvm::Type *varTy = ConvertTypeForMem(Ty);
2784
2785 bool packed = false;
2786 CharUnits varAlign = getContext().getDeclAlign(D);
Rui Ueyama83aa9792016-01-14 21:00:27 +00002787 CharUnits varOffset = size.alignTo(varAlign);
John McCall7f416cc2015-09-08 08:05:57 +00002788
2789 // We may have to insert padding.
2790 if (varOffset != size) {
2791 llvm::Type *paddingTy =
2792 llvm::ArrayType::get(Int8Ty, (varOffset - size).getQuantity());
2793
2794 types.push_back(paddingTy);
2795 size = varOffset;
2796
2797 // Conversely, we might have to prevent LLVM from inserting padding.
2798 } else if (CGM.getDataLayout().getABITypeAlignment(varTy)
2799 > varAlign.getQuantity()) {
2800 packed = true;
2801 }
2802 types.push_back(varTy);
2803
2804 byrefType->setBody(types, packed);
2805
2806 BlockByrefInfo info;
2807 info.Type = byrefType;
2808 info.FieldIndex = types.size() - 1;
2809 info.FieldOffset = varOffset;
2810 info.ByrefAlignment = std::max(varAlign, getPointerAlign());
2811
2812 auto pair = BlockByrefInfos.insert({D, info});
2813 assert(pair.second && "info was inserted recursively?");
2814 return pair.first->second;
John McCall73064872011-03-31 01:59:53 +00002815}
2816
2817/// Initialize the structural components of a __block variable, i.e.
2818/// everything but the actual object.
2819void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
John McCallf9b056b2011-03-31 08:03:29 +00002820 // Find the address of the local.
John McCall7f416cc2015-09-08 08:05:57 +00002821 Address addr = emission.Addr;
John McCall73064872011-03-31 01:59:53 +00002822
John McCallf9b056b2011-03-31 08:03:29 +00002823 // That's an alloca of the byref structure type.
Chris Lattner2192fe52011-07-18 04:24:23 +00002824 llvm::StructType *byrefType = cast<llvm::StructType>(
John McCall7f416cc2015-09-08 08:05:57 +00002825 cast<llvm::PointerType>(addr.getPointer()->getType())->getElementType());
2826
2827 unsigned nextHeaderIndex = 0;
2828 CharUnits nextHeaderOffset;
2829 auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2830 const Twine &name) {
James Y Knight751fe282019-02-09 22:22:28 +00002831 auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, name);
John McCall7f416cc2015-09-08 08:05:57 +00002832 Builder.CreateStore(value, fieldAddr);
2833
2834 nextHeaderIndex++;
2835 nextHeaderOffset += fieldSize;
2836 };
John McCallf9b056b2011-03-31 08:03:29 +00002837
2838 // Build the byref helpers if necessary. This is null if we don't need any.
John McCall7f416cc2015-09-08 08:05:57 +00002839 BlockByrefHelpers *helpers = buildByrefHelpers(*byrefType, emission);
John McCall73064872011-03-31 01:59:53 +00002840
2841 const VarDecl &D = *emission.Variable;
2842 QualType type = D.getType();
2843
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002844 bool HasByrefExtendedLayout;
2845 Qualifiers::ObjCLifetime ByrefLifetime;
2846 bool ByRefHasLifetime =
2847 getContext().getByrefLifetime(type, ByrefLifetime, HasByrefExtendedLayout);
John McCall7f416cc2015-09-08 08:05:57 +00002848
John McCallf9b056b2011-03-31 08:03:29 +00002849 llvm::Value *V;
John McCall73064872011-03-31 01:59:53 +00002850
2851 // Initialize the 'isa', which is just 0 or 1.
2852 int isa = 0;
John McCallf9b056b2011-03-31 08:03:29 +00002853 if (type.isObjCGCWeak())
John McCall73064872011-03-31 01:59:53 +00002854 isa = 1;
2855 V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");
John McCall7f416cc2015-09-08 08:05:57 +00002856 storeHeaderField(V, getPointerSize(), "byref.isa");
John McCall73064872011-03-31 01:59:53 +00002857
2858 // Store the address of the variable into its own forwarding pointer.
John McCall7f416cc2015-09-08 08:05:57 +00002859 storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding");
John McCall73064872011-03-31 01:59:53 +00002860
2861 // Blocks ABI:
2862 // c) the flags field is set to either 0 if no helper functions are
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002863 // needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
John McCall73064872011-03-31 01:59:53 +00002864 BlockFlags flags;
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002865 if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2866 if (ByRefHasLifetime) {
2867 if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2868 else switch (ByrefLifetime) {
2869 case Qualifiers::OCL_Strong:
2870 flags |= BLOCK_BYREF_LAYOUT_STRONG;
2871 break;
2872 case Qualifiers::OCL_Weak:
2873 flags |= BLOCK_BYREF_LAYOUT_WEAK;
2874 break;
2875 case Qualifiers::OCL_ExplicitNone:
2876 flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2877 break;
2878 case Qualifiers::OCL_None:
2879 if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2880 flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2881 break;
2882 default:
2883 break;
2884 }
2885 if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2886 printf("\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2887 if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2888 printf(" BLOCK_BYREF_HAS_COPY_DISPOSE");
2889 if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2890 BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2891 if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2892 printf(" BLOCK_BYREF_LAYOUT_EXTENDED");
2893 if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2894 printf(" BLOCK_BYREF_LAYOUT_STRONG");
2895 if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2896 printf(" BLOCK_BYREF_LAYOUT_WEAK");
2897 if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2898 printf(" BLOCK_BYREF_LAYOUT_UNRETAINED");
2899 if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2900 printf(" BLOCK_BYREF_LAYOUT_NON_OBJECT");
2901 }
2902 printf("\n");
2903 }
2904 }
John McCall7f416cc2015-09-08 08:05:57 +00002905 storeHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
2906 getIntSize(), "byref.flags");
John McCall73064872011-03-31 01:59:53 +00002907
John McCallf9b056b2011-03-31 08:03:29 +00002908 CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType);
2909 V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity());
John McCall7f416cc2015-09-08 08:05:57 +00002910 storeHeaderField(V, getIntSize(), "byref.size");
John McCall73064872011-03-31 01:59:53 +00002911
John McCallf9b056b2011-03-31 08:03:29 +00002912 if (helpers) {
John McCall7f416cc2015-09-08 08:05:57 +00002913 storeHeaderField(helpers->CopyHelper, getPointerSize(),
2914 "byref.copyHelper");
2915 storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2916 "byref.disposeHelper");
John McCall73064872011-03-31 01:59:53 +00002917 }
John McCall7f416cc2015-09-08 08:05:57 +00002918
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002919 if (ByRefHasLifetime && HasByrefExtendedLayout) {
John McCall7f416cc2015-09-08 08:05:57 +00002920 auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type);
2921 storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
Fariborz Jahaniana9d44642012-11-14 17:15:51 +00002922 }
John McCall73064872011-03-31 01:59:53 +00002923}
2924
Akira Hatanaka9978da32018-08-10 15:09:24 +00002925void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2926 bool CanThrow) {
James Y Knight9871db02019-02-05 16:42:33 +00002927 llvm::FunctionCallee F = CGM.getBlockObjectDispose();
John McCall882987f2013-02-28 19:01:20 +00002928 llvm::Value *args[] = {
2929 Builder.CreateBitCast(V, Int8PtrTy),
2930 llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
2931 };
Akira Hatanaka9978da32018-08-10 15:09:24 +00002932
2933 if (CanThrow)
2934 EmitRuntimeCallOrInvoke(F, args);
2935 else
2936 EmitNounwindRuntimeCall(F, args);
Mike Stump626aecc2009-03-05 01:23:13 +00002937}
John McCall73064872011-03-31 01:59:53 +00002938
Akira Hatanakacb6a9332018-07-26 16:51:21 +00002939void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2940 BlockFieldFlags Flags,
Akira Hatanaka9978da32018-08-10 15:09:24 +00002941 bool LoadBlockVarAddr, bool CanThrow) {
2942 EHStack.pushCleanup<CallBlockRelease>(Kind, Addr, Flags, LoadBlockVarAddr,
2943 CanThrow);
John McCall73064872011-03-31 01:59:53 +00002944}
John McCall7959fee2011-09-09 20:41:01 +00002945
2946/// Adjust the declaration of something from the blocks API.
2947static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2948 llvm::Constant *C) {
Rafael Espindola2ae250c2014-05-09 00:08:36 +00002949 auto *GV = cast<llvm::GlobalValue>(C->stripPointerCasts());
Saleem Abdulrasool442b88b2016-05-28 19:41:35 +00002950
2951 if (CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2952 IdentifierInfo &II = CGM.getContext().Idents.get(C->getName());
2953 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2954 DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
2955
Saleem Abdulrasool7bae9ad2016-06-03 23:26:30 +00002956 assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2957 isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2958 "expected Function or GlobalVariable");
Saleem Abdulrasool442b88b2016-05-28 19:41:35 +00002959
2960 const NamedDecl *ND = nullptr;
2961 for (const auto &Result : DC->lookup(&II))
2962 if ((ND = dyn_cast<FunctionDecl>(Result)) ||
2963 (ND = dyn_cast<VarDecl>(Result)))
2964 break;
2965
2966 // TODO: support static blocks runtime
2967 if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2968 GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2969 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2970 } else {
2971 GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2972 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2973 }
2974 }
2975
Rafael Espindola3c8a39c2018-03-14 18:19:26 +00002976 if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2977 GV->hasExternalLinkage())
John McCall7959fee2011-09-09 20:41:01 +00002978 GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
Rafael Espindola3c8a39c2018-03-14 18:19:26 +00002979
2980 CGM.setDSOLocal(GV);
John McCall7959fee2011-09-09 20:41:01 +00002981}
2982
James Y Knight9871db02019-02-05 16:42:33 +00002983llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
John McCall7959fee2011-09-09 20:41:01 +00002984 if (BlockObjectDispose)
2985 return BlockObjectDispose;
2986
2987 llvm::Type *args[] = { Int8PtrTy, Int32Ty };
2988 llvm::FunctionType *fty
2989 = llvm::FunctionType::get(VoidTy, args, false);
2990 BlockObjectDispose = CreateRuntimeFunction(fty, "_Block_object_dispose");
James Y Knight9871db02019-02-05 16:42:33 +00002991 configureBlocksRuntimeObject(
2992 *this, cast<llvm::Constant>(BlockObjectDispose.getCallee()));
John McCall7959fee2011-09-09 20:41:01 +00002993 return BlockObjectDispose;
2994}
2995
James Y Knight9871db02019-02-05 16:42:33 +00002996llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
John McCall7959fee2011-09-09 20:41:01 +00002997 if (BlockObjectAssign)
2998 return BlockObjectAssign;
2999
3000 llvm::Type *args[] = { Int8PtrTy, Int8PtrTy, Int32Ty };
3001 llvm::FunctionType *fty
3002 = llvm::FunctionType::get(VoidTy, args, false);
3003 BlockObjectAssign = CreateRuntimeFunction(fty, "_Block_object_assign");
James Y Knight9871db02019-02-05 16:42:33 +00003004 configureBlocksRuntimeObject(
3005 *this, cast<llvm::Constant>(BlockObjectAssign.getCallee()));
John McCall7959fee2011-09-09 20:41:01 +00003006 return BlockObjectAssign;
3007}
3008
3009llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
3010 if (NSConcreteGlobalBlock)
3011 return NSConcreteGlobalBlock;
3012
3013 NSConcreteGlobalBlock = GetOrCreateLLVMGlobal("_NSConcreteGlobalBlock",
Craig Topper8a13c412014-05-21 05:09:00 +00003014 Int8PtrTy->getPointerTo(),
3015 nullptr);
John McCall7959fee2011-09-09 20:41:01 +00003016 configureBlocksRuntimeObject(*this, NSConcreteGlobalBlock);
3017 return NSConcreteGlobalBlock;
3018}
3019
3020llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
3021 if (NSConcreteStackBlock)
3022 return NSConcreteStackBlock;
3023
3024 NSConcreteStackBlock = GetOrCreateLLVMGlobal("_NSConcreteStackBlock",
Craig Topper8a13c412014-05-21 05:09:00 +00003025 Int8PtrTy->getPointerTo(),
3026 nullptr);
John McCall7959fee2011-09-09 20:41:01 +00003027 configureBlocksRuntimeObject(*this, NSConcreteStackBlock);
Saleem Abdulrasool442b88b2016-05-28 19:41:35 +00003028 return NSConcreteStackBlock;
John McCall7959fee2011-09-09 20:41:01 +00003029}