Promote atomic type sizes up to a power of two, capped by
MaxAtomicPromoteWidth. Fix a ton of terrible bugs with
_Atomic types and (non-intrinsic-mediated) loads and stores
thereto.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@176658 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index f8921db..1ac13c0 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -29,6 +29,14 @@
// Aggregate Expression Emitter
//===----------------------------------------------------------------------===//
+llvm::Value *AggValueSlot::getPaddedAtomicAddr() const {
+ assert(isValueOfAtomic());
+ llvm::GEPOperator *op = cast<llvm::GEPOperator>(getAddr());
+ assert(op->getNumIndices() == 2);
+ assert(op->hasAllZeroIndices());
+ return op->getPointerOperand();
+}
+
namespace {
class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
CodeGenFunction &CGF;
@@ -190,6 +198,38 @@
CGF.EmitAtomicExpr(E, EnsureSlot(E->getType()).getAddr());
}
};
+
+/// A helper class for emitting expressions into the value sub-object
+/// of a padded atomic type.
+class ValueDestForAtomic {
+ AggValueSlot Dest;
+public:
+ ValueDestForAtomic(CodeGenFunction &CGF, AggValueSlot dest, QualType type)
+ : Dest(dest) {
+ assert(!Dest.isValueOfAtomic());
+ if (!Dest.isIgnored() && CGF.CGM.isPaddedAtomicType(type)) {
+ llvm::Value *valueAddr = CGF.Builder.CreateStructGEP(Dest.getAddr(), 0);
+ Dest = AggValueSlot::forAddr(valueAddr,
+ Dest.getAlignment(),
+ Dest.getQualifiers(),
+ Dest.isExternallyDestructed(),
+ Dest.requiresGCollection(),
+ Dest.isPotentiallyAliased(),
+ Dest.isZeroed(),
+ AggValueSlot::IsValueOfAtomic);
+ }
+ }
+
+ const AggValueSlot &getDest() const { return Dest; }
+
+ ~ValueDestForAtomic() {
+ // Kill the GEP if we made one and it didn't end up used.
+ if (Dest.isValueOfAtomic()) {
+ llvm::Instruction *addr = cast<llvm::GetElementPtrInst>(Dest.getAddr());
+ if (addr->use_empty()) addr->eraseFromParent();
+ }
+ }
+};
} // end anonymous namespace.
//===----------------------------------------------------------------------===//
@@ -201,6 +241,14 @@
/// then loads the result into DestPtr.
void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
LValue LV = CGF.EmitLValue(E);
+
+ // If the type of the l-value is atomic, then do an atomic load.
+ if (LV.getType()->isAtomicType()) {
+ ValueDestForAtomic valueDest(CGF, Dest, LV.getType());
+ CGF.EmitAtomicLoad(LV, valueDest.getDest());
+ return;
+ }
+
EmitFinalDestCopy(E->getType(), LV);
}
@@ -543,6 +591,20 @@
CGF.EmitAggExpr(E->getInitializer(), Slot);
}
+/// Attempt to look through various unimportant expressions to find a
+/// cast of the given kind.
+static Expr *findPeephole(Expr *op, CastKind kind) {
+ while (true) {
+ op = op->IgnoreParens();
+ if (CastExpr *castE = dyn_cast<CastExpr>(op)) {
+ if (castE->getCastKind() == kind)
+ return castE->getSubExpr();
+ if (castE->getCastKind() == CK_NoOp)
+ continue;
+ }
+ return 0;
+ }
+}
void AggExprEmitter::VisitCastExpr(CastExpr *E) {
switch (E->getCastKind()) {
@@ -582,6 +644,75 @@
"should have been unpacked before we got here");
}
+ case CK_NonAtomicToAtomic:
+ case CK_AtomicToNonAtomic: {
+ bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
+
+ // Determine the atomic and value types.
+ QualType atomicType = E->getSubExpr()->getType();
+ QualType valueType = E->getType();
+ if (isToAtomic) std::swap(atomicType, valueType);
+
+ assert(atomicType->isAtomicType());
+ assert(CGF.getContext().hasSameUnqualifiedType(valueType,
+ atomicType->castAs<AtomicType>()->getValueType()));
+
+ // Just recurse normally if we're ignoring the result or the
+ // atomic type doesn't change representation.
+ if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) {
+ return Visit(E->getSubExpr());
+ }
+
+ CastKind peepholeTarget =
+ (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
+
+ // These two cases are reverses of each other; try to peephole them.
+ if (Expr *op = findPeephole(E->getSubExpr(), peepholeTarget)) {
+ assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
+ E->getType()) &&
+ "peephole significantly changed types?");
+ return Visit(op);
+ }
+
+ // If we're converting an r-value of non-atomic type to an r-value
+ // of atomic type, just make an atomic temporary, emit into that,
+ // and then copy the value out. (FIXME: do we need to
+ // zero-initialize it first?)
+ if (isToAtomic) {
+ ValueDestForAtomic valueDest(CGF, Dest, atomicType);
+ CGF.EmitAggExpr(E->getSubExpr(), valueDest.getDest());
+ return;
+ }
+
+ // Otherwise, we're converting an atomic type to a non-atomic type.
+
+ // If the dest is a value-of-atomic subobject, drill back out.
+ if (Dest.isValueOfAtomic()) {
+ AggValueSlot atomicSlot =
+ AggValueSlot::forAddr(Dest.getPaddedAtomicAddr(),
+ Dest.getAlignment(),
+ Dest.getQualifiers(),
+ Dest.isExternallyDestructed(),
+ Dest.requiresGCollection(),
+ Dest.isPotentiallyAliased(),
+ Dest.isZeroed(),
+ AggValueSlot::IsNotValueOfAtomic);
+ CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
+ return;
+ }
+
+ // Otherwise, make an atomic temporary, emit into that, and then
+ // copy the value out.
+ AggValueSlot atomicSlot =
+ CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");
+ CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
+
+ llvm::Value *valueAddr =
+ Builder.CreateStructGEP(atomicSlot.getAddr(), 0);
+ RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
+ return EmitFinalDestCopy(valueType, rvalue);
+ }
+
case CK_LValueToRValue:
// If we're loading from a volatile type, force the destination
// into existence.
@@ -589,11 +720,10 @@
EnsureDest(E->getType());
return Visit(E->getSubExpr());
}
+
// fallthrough
case CK_NoOp:
- case CK_AtomicToNonAtomic:
- case CK_NonAtomicToAtomic:
case CK_UserDefinedConversion:
case CK_ConstructorConversion:
assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
@@ -775,6 +905,12 @@
// Now emit the LHS and copy into it.
LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
+ // That copy is an atomic copy if the LHS is atomic.
+ if (LHS.getType()->isAtomicType()) {
+ CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
+ return;
+ }
+
EmitCopy(E->getLHS()->getType(),
AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,
needsGC(E->getLHS()->getType()),
@@ -785,6 +921,15 @@
LValue LHS = CGF.EmitLValue(E->getLHS());
+ // If we have an atomic type, evaluate into the destination and then
+ // do an atomic copy.
+ if (LHS.getType()->isAtomicType()) {
+ EnsureDest(E->getRHS()->getType());
+ Visit(E->getRHS());
+ CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
+ return;
+ }
+
// Codegen the RHS so that it stores directly into the LHS.
AggValueSlot LHSSlot =
AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,