Promote atomic type sizes up to a power of two, capped by
MaxAtomicPromoteWidth. Fix a ton of terrible bugs with
_Atomic types and (non-intrinsic-mediated) loads and stores
thereto.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@176658 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/CGAtomic.cpp b/lib/CodeGen/CGAtomic.cpp
index f17e48d..817d5c4 100644
--- a/lib/CodeGen/CGAtomic.cpp
+++ b/lib/CodeGen/CGAtomic.cpp
@@ -17,10 +17,169 @@
#include "clang/AST/ASTContext.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Operator.h"
using namespace clang;
using namespace CodeGen;
+// The ABI values for various atomic memory orderings.
+enum AtomicOrderingKind {
+ AO_ABI_memory_order_relaxed = 0,
+ AO_ABI_memory_order_consume = 1,
+ AO_ABI_memory_order_acquire = 2,
+ AO_ABI_memory_order_release = 3,
+ AO_ABI_memory_order_acq_rel = 4,
+ AO_ABI_memory_order_seq_cst = 5
+};
+
+namespace {
+ class AtomicInfo {
+ CodeGenFunction &CGF;
+ QualType AtomicTy;
+ QualType ValueTy;
+ uint64_t AtomicSizeInBits;
+ uint64_t ValueSizeInBits;
+ CharUnits AtomicAlign;
+ CharUnits ValueAlign;
+ CharUnits LValueAlign;
+ TypeEvaluationKind EvaluationKind;
+ bool UseLibcall;
+ public:
+ AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) {
+ assert(lvalue.isSimple());
+
+ AtomicTy = lvalue.getType();
+ ValueTy = AtomicTy->castAs<AtomicType>()->getValueType();
+ EvaluationKind = CGF.getEvaluationKind(ValueTy);
+
+ ASTContext &C = CGF.getContext();
+
+ uint64_t valueAlignInBits;
+ llvm::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy);
+
+ uint64_t atomicAlignInBits;
+ llvm::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy);
+
+ assert(ValueSizeInBits <= AtomicSizeInBits);
+ assert(valueAlignInBits <= atomicAlignInBits);
+
+ AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits);
+ ValueAlign = C.toCharUnitsFromBits(valueAlignInBits);
+ if (lvalue.getAlignment().isZero())
+ lvalue.setAlignment(AtomicAlign);
+
+ UseLibcall =
+ (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) ||
+ AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth());
+ }
+
+ QualType getAtomicType() const { return AtomicTy; }
+ QualType getValueType() const { return ValueTy; }
+ CharUnits getAtomicAlignment() const { return AtomicAlign; }
+ CharUnits getValueAlignment() const { return ValueAlign; }
+ uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
+ uint64_t getValueSizeInBits() const { return AtomicSizeInBits; }
+ TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
+ bool shouldUseLibcall() const { return UseLibcall; }
+
+ /// Is the atomic size larger than the underlying value type?
+ ///
+ /// Note that the absence of padding does not mean that atomic
+ /// objects are completely interchangeable with non-atomic
+ /// objects: we might have promoted the alignment of a type
+ /// without making it bigger.
+ bool hasPadding() const {
+ return (ValueSizeInBits != AtomicSizeInBits);
+ }
+
+ void emitMemSetZeroIfNecessary(LValue dest) const;
+
+ llvm::Value *getAtomicSizeValue() const {
+ CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
+ return CGF.CGM.getSize(size);
+ }
+
+ /// Cast the given pointer to an integer pointer suitable for
+ /// atomic operations.
+ llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
+
+ /// Turn an atomic-layout object into an r-value.
+ RValue convertTempToRValue(llvm::Value *addr,
+ AggValueSlot resultSlot) const;
+
+ /// Copy an atomic r-value into atomic-layout memory.
+ void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const;
+
+ /// Project an l-value down to the value field.
+ LValue projectValue(LValue lvalue) const {
+ llvm::Value *addr = lvalue.getAddress();
+ if (hasPadding())
+ addr = CGF.Builder.CreateStructGEP(addr, 0);
+
+ return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(),
+ CGF.getContext(), lvalue.getTBAAInfo());
+ }
+
+ /// Materialize an atomic r-value in atomic-layout memory.
+ llvm::Value *materializeRValue(RValue rvalue) const;
+
+ private:
+ bool requiresMemSetZero(llvm::Type *type) const;
+ };
+}
+
+static RValue emitAtomicLibcall(CodeGenFunction &CGF,
+ StringRef fnName,
+ QualType resultType,
+ CallArgList &args) {
+ const CGFunctionInfo &fnInfo =
+ CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
+ FunctionType::ExtInfo(), RequiredArgs::All);
+ llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
+ llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
+ return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
+}
+
+/// Does a store of the given IR type modify the full expected width?
+static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
+ uint64_t expectedSize) {
+ return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
+}
+
+/// Does the atomic type require memsetting to zero before initialization?
+///
+/// The IR type is provided as a way of making certain queries faster.
+bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
+ // If the atomic type has size padding, we definitely need a memset.
+ if (hasPadding()) return true;
+
+ // Otherwise, do some simple heuristics to try to avoid it:
+ switch (getEvaluationKind()) {
+ // For scalars and complexes, check whether the store size of the
+ // type uses the full size.
+ case TEK_Scalar:
+ return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
+ case TEK_Complex:
+ return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
+ AtomicSizeInBits / 2);
+
+ // Just be pessimistic about aggregates.
+ case TEK_Aggregate:
+ return true;
+ }
+ llvm_unreachable("bad evaluation kind");
+}
+
+void AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const {
+ llvm::Value *addr = dest.getAddress();
+ if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
+ return;
+
+ CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
+ AtomicSizeInBits / 8,
+ dest.getAlignment().getQuantity());
+}
+
static void
EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
@@ -177,24 +336,9 @@
if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
assert(!Dest && "Init does not return a value");
- LValue LV = MakeAddrLValue(Ptr, AtomicTy, alignChars);
- switch (getEvaluationKind(E->getVal1()->getType())) {
- case TEK_Scalar:
- EmitScalarInit(EmitScalarExpr(E->getVal1()), LV);
- return RValue::get(0);
- case TEK_Complex:
- EmitComplexExprIntoLValue(E->getVal1(), LV, /*isInit*/ true);
- return RValue::get(0);
- case TEK_Aggregate: {
- AggValueSlot Slot = AggValueSlot::forLValue(LV,
- AggValueSlot::IsNotDestructed,
- AggValueSlot::DoesNotNeedGCBarriers,
- AggValueSlot::IsNotAliased);
- EmitAggExpr(E->getVal1(), Slot);
- return RValue::get(0);
- }
- }
- llvm_unreachable("bad evaluation kind");
+ LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
+ EmitAtomicInit(E->getVal1(), lvalue);
+ return RValue::get(0);
}
Order = EmitScalarExpr(E->getOrder());
@@ -385,30 +529,30 @@
if (isa<llvm::ConstantInt>(Order)) {
int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
switch (ord) {
- case 0: // memory_order_relaxed
+ case AO_ABI_memory_order_relaxed:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::Monotonic);
break;
- case 1: // memory_order_consume
- case 2: // memory_order_acquire
+ case AO_ABI_memory_order_consume:
+ case AO_ABI_memory_order_acquire:
if (IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::Acquire);
break;
- case 3: // memory_order_release
+ case AO_ABI_memory_order_release:
if (IsLoad)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::Release);
break;
- case 4: // memory_order_acq_rel
+ case AO_ABI_memory_order_acq_rel:
if (IsLoad || IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::AcquireRelease);
break;
- case 5: // memory_order_seq_cst
+ case AO_ABI_memory_order_seq_cst:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
llvm::SequentiallyConsistent);
break;
@@ -483,3 +627,316 @@
return RValue::get(0);
return convertTempToRValue(OrigDest, E->getType());
}
+
+llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
+ unsigned addrspace =
+ cast<llvm::PointerType>(addr->getType())->getAddressSpace();
+ llvm::IntegerType *ty =
+ llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
+ return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
+}
+
+RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
+ AggValueSlot resultSlot) const {
+ if (EvaluationKind == TEK_Aggregate) {
+ // Nothing to do if the result is ignored.
+ if (resultSlot.isIgnored()) return resultSlot.asRValue();
+
+ assert(resultSlot.getAddr() == addr || hasPadding());
+
+ // In these cases, we should have emitted directly into the result slot.
+ if (!hasPadding() || resultSlot.isValueOfAtomic())
+ return resultSlot.asRValue();
+
+ // Otherwise, fall into the common path.
+ }
+
+ // Drill into the padding structure if we have one.
+ if (hasPadding())
+ addr = CGF.Builder.CreateStructGEP(addr, 0);
+
+ // If we're emitting to an aggregate, copy into the result slot.
+ if (EvaluationKind == TEK_Aggregate) {
+ CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(),
+ resultSlot.isVolatile());
+ return resultSlot.asRValue();
+ }
+
+ // Otherwise, just convert the temporary to an r-value using the
+ // normal conversion routine.
+ return CGF.convertTempToRValue(addr, getValueType());
+}
+
+/// Emit a load from an l-value of atomic type. Note that the r-value
+/// we produce is an r-value of the atomic *value* type.
+RValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) {
+ AtomicInfo atomics(*this, src);
+
+ // Check whether we should use a library call.
+ if (atomics.shouldUseLibcall()) {
+ llvm::Value *tempAddr;
+ if (resultSlot.isValueOfAtomic()) {
+ assert(atomics.getEvaluationKind() == TEK_Aggregate);
+ tempAddr = resultSlot.getPaddedAtomicAddr();
+ } else if (!resultSlot.isIgnored() && !atomics.hasPadding()) {
+ assert(atomics.getEvaluationKind() == TEK_Aggregate);
+ tempAddr = resultSlot.getAddr();
+ } else {
+ tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
+ }
+
+ // void __atomic_load(size_t size, void *mem, void *return, int order);
+ CallArgList args;
+ args.add(RValue::get(atomics.getAtomicSizeValue()),
+ getContext().getSizeType());
+ args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())),
+ getContext().VoidPtrTy);
+ args.add(RValue::get(EmitCastToVoidPtr(tempAddr)),
+ getContext().VoidPtrTy);
+ args.add(RValue::get(llvm::ConstantInt::get(IntTy,
+ AO_ABI_memory_order_seq_cst)),
+ getContext().IntTy);
+ emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args);
+
+ // Produce the r-value.
+ return atomics.convertTempToRValue(tempAddr, resultSlot);
+ }
+
+ // Okay, we're doing this natively.
+ llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress());
+ llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load");
+ load->setAtomic(llvm::SequentiallyConsistent);
+
+ // Other decoration.
+ load->setAlignment(src.getAlignment().getQuantity());
+ if (src.isVolatileQualified())
+ load->setVolatile(true);
+ if (src.getTBAAInfo())
+ CGM.DecorateInstruction(load, src.getTBAAInfo());
+
+ // Okay, turn that back into the original value type.
+ QualType valueType = atomics.getValueType();
+ llvm::Value *result = load;
+
+ // If we're ignoring an aggregate return, don't do anything.
+ if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored())
+ return RValue::getAggregate(0, false);
+
+ // The easiest way to do this this is to go through memory, but we
+ // try not to in some easy cases.
+ if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) {
+ llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType);
+ if (isa<llvm::IntegerType>(resultTy)) {
+ assert(result->getType() == resultTy);
+ result = EmitFromMemory(result, valueType);
+ } else if (isa<llvm::PointerType>(resultTy)) {
+ result = Builder.CreateIntToPtr(result, resultTy);
+ } else {
+ result = Builder.CreateBitCast(result, resultTy);
+ }
+ return RValue::get(result);
+ }
+
+ // Create a temporary. This needs to be big enough to hold the
+ // atomic integer.
+ llvm::Value *temp;
+ bool tempIsVolatile = false;
+ CharUnits tempAlignment;
+ if (atomics.getEvaluationKind() == TEK_Aggregate &&
+ (!atomics.hasPadding() || resultSlot.isValueOfAtomic())) {
+ assert(!resultSlot.isIgnored());
+ if (resultSlot.isValueOfAtomic()) {
+ temp = resultSlot.getPaddedAtomicAddr();
+ tempAlignment = atomics.getAtomicAlignment();
+ } else {
+ temp = resultSlot.getAddr();
+ tempAlignment = atomics.getValueAlignment();
+ }
+ tempIsVolatile = resultSlot.isVolatile();
+ } else {
+ temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp");
+ tempAlignment = atomics.getAtomicAlignment();
+ }
+
+ // Slam the integer into the temporary.
+ llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp);
+ Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity())
+ ->setVolatile(tempIsVolatile);
+
+ return atomics.convertTempToRValue(temp, resultSlot);
+}
+
+
+
+/// Copy an r-value into memory as part of storing to an atomic type.
+/// This needs to create a bit-pattern suitable for atomic operations.
+void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const {
+ // If we have an r-value, the rvalue should be of the atomic type,
+ // which means that the caller is responsible for having zeroed
+ // any padding. Just do an aggregate copy of that type.
+ if (rvalue.isAggregate()) {
+ CGF.EmitAggregateCopy(dest.getAddress(),
+ rvalue.getAggregateAddr(),
+ getAtomicType(),
+ (rvalue.isVolatileQualified()
+ || dest.isVolatileQualified()),
+ dest.getAlignment());
+ return;
+ }
+
+ // Okay, otherwise we're copying stuff.
+
+ // Zero out the buffer if necessary.
+ emitMemSetZeroIfNecessary(dest);
+
+ // Drill past the padding if present.
+ dest = projectValue(dest);
+
+ // Okay, store the rvalue in.
+ if (rvalue.isScalar()) {
+ CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true);
+ } else {
+ CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true);
+ }
+}
+
+
+/// Materialize an r-value into memory for the purposes of storing it
+/// to an atomic type.
+llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
+ // Aggregate r-values are already in memory, and EmitAtomicStore
+ // requires them to be values of the atomic type.
+ if (rvalue.isAggregate())
+ return rvalue.getAggregateAddr();
+
+ // Otherwise, make a temporary and materialize into it.
+ llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp");
+ LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment());
+ emitCopyIntoMemory(rvalue, tempLV);
+ return temp;
+}
+
+/// Emit a store to an l-value of atomic type.
+///
+/// Note that the r-value is expected to be an r-value *of the atomic
+/// type*; this means that for aggregate r-values, it should include
+/// storage for any padding that was necessary.
+void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
+ bool isInit) {
+ // If this is an aggregate r-value, it should agree in type except
+ // maybe for address-space qualification.
+ assert(!rvalue.isAggregate() ||
+ rvalue.getAggregateAddr()->getType()->getPointerElementType()
+ == dest.getAddress()->getType()->getPointerElementType());
+
+ AtomicInfo atomics(*this, dest);
+
+ // If this is an initialization, just put the value there normally.
+ if (isInit) {
+ atomics.emitCopyIntoMemory(rvalue, dest);
+ return;
+ }
+
+ // Check whether we should use a library call.
+ if (atomics.shouldUseLibcall()) {
+ // Produce a source address.
+ llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
+
+ // void __atomic_store(size_t size, void *mem, void *val, int order)
+ CallArgList args;
+ args.add(RValue::get(atomics.getAtomicSizeValue()),
+ getContext().getSizeType());
+ args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())),
+ getContext().VoidPtrTy);
+ args.add(RValue::get(EmitCastToVoidPtr(srcAddr)),
+ getContext().VoidPtrTy);
+ args.add(RValue::get(llvm::ConstantInt::get(IntTy,
+ AO_ABI_memory_order_seq_cst)),
+ getContext().IntTy);
+ emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
+ return;
+ }
+
+ // Okay, we're doing this natively.
+ llvm::Value *intValue;
+
+ // If we've got a scalar value of the right size, try to avoid going
+ // through memory.
+ if (rvalue.isScalar() && !atomics.hasPadding()) {
+ llvm::Value *value = rvalue.getScalarVal();
+ if (isa<llvm::IntegerType>(value->getType())) {
+ intValue = value;
+ } else {
+ llvm::IntegerType *inputIntTy =
+ llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits());
+ if (isa<llvm::PointerType>(value->getType())) {
+ intValue = Builder.CreatePtrToInt(value, inputIntTy);
+ } else {
+ intValue = Builder.CreateBitCast(value, inputIntTy);
+ }
+ }
+
+ // Otherwise, we need to go through memory.
+ } else {
+ // Put the r-value in memory.
+ llvm::Value *addr = atomics.materializeRValue(rvalue);
+
+ // Cast the temporary to the atomic int type and pull a value out.
+ addr = atomics.emitCastToAtomicIntPointer(addr);
+ intValue = Builder.CreateAlignedLoad(addr,
+ atomics.getAtomicAlignment().getQuantity());
+ }
+
+ // Do the atomic store.
+ llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress());
+ llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
+
+ // Initializations don't need to be atomic.
+ if (!isInit) store->setAtomic(llvm::SequentiallyConsistent);
+
+ // Other decoration.
+ store->setAlignment(dest.getAlignment().getQuantity());
+ if (dest.isVolatileQualified())
+ store->setVolatile(true);
+ if (dest.getTBAAInfo())
+ CGM.DecorateInstruction(store, dest.getTBAAInfo());
+}
+
+void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
+ AtomicInfo atomics(*this, dest);
+
+ switch (atomics.getEvaluationKind()) {
+ case TEK_Scalar: {
+ llvm::Value *value = EmitScalarExpr(init);
+ atomics.emitCopyIntoMemory(RValue::get(value), dest);
+ return;
+ }
+
+ case TEK_Complex: {
+ ComplexPairTy value = EmitComplexExpr(init);
+ atomics.emitCopyIntoMemory(RValue::getComplex(value), dest);
+ return;
+ }
+
+ case TEK_Aggregate: {
+ // Memset the buffer first if there's any possibility of
+ // uninitialized internal bits.
+ atomics.emitMemSetZeroIfNecessary(dest);
+
+ // HACK: whether the initializer actually has an atomic type
+ // doesn't really seem reliable right now.
+ if (!init->getType()->isAtomicType()) {
+ dest = atomics.projectValue(dest);
+ }
+
+ // Evaluate the expression directly into the destination.
+ AggValueSlot slot = AggValueSlot::forLValue(dest,
+ AggValueSlot::IsNotDestructed,
+ AggValueSlot::DoesNotNeedGCBarriers,
+ AggValueSlot::IsNotAliased);
+ EmitAggExpr(init, slot);
+ return;
+ }
+ }
+ llvm_unreachable("bad evaluation kind");
+}