[ARM] Add __bf16 as new Bfloat16 C Type
Summary:
This patch upstreams support for a new storage only bfloat16 C type.
This type is used to implement primitive support for bfloat16 data, in
line with the Bfloat16 extension of the Armv8.6-a architecture, as
detailed here:
https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/arm-architecture-developments-armv8-6-a
The bfloat type, and its properties are specified in the Arm Architecture
Reference Manual:
https://developer.arm.com/docs/ddi0487/latest/arm-architecture-reference-manual-armv8-for-armv8-a-architecture-profile
In detail this patch:
- introduces an opaque, storage-only C-type __bf16, which introduces a new bfloat IR type.
This is part of a patch series, starting with command-line and Bfloat16
assembly support. The subsequent patches will upstream intrinsics
support for BFloat16, followed by Matrix Multiplication and the
remaining Virtualization features of the armv8.6-a architecture.
The following people contributed to this patch:
- Luke Cheeseman
- Momchil Velikov
- Alexandros Lamprineas
- Luke Geeson
- Simon Tatham
- Ties Stuij
Reviewers: SjoerdMeijer, rjmccall, rsmith, liutianle, RKSimon, craig.topper, jfb, LukeGeeson, fpetrogalli
Reviewed By: SjoerdMeijer
Subscribers: labrinea, majnemer, asmith, dexonsmith, kristof.beyls, arphaman, danielkiss, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D76077
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index eccc3e2..60482ea 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -100,7 +100,7 @@
using namespace clang;
enum FloatingRank {
- Float16Rank, HalfRank, FloatRank, DoubleRank, LongDoubleRank, Float128Rank
+ BFloat16Rank, Float16Rank, HalfRank, FloatRank, DoubleRank, LongDoubleRank, Float128Rank
};
/// \returns location that is relevant when searching for Doc comments related
@@ -1448,6 +1448,8 @@
// half type (OpenCL 6.1.1.1) / ARM NEON __fp16
InitBuiltinType(HalfTy, BuiltinType::Half);
+ InitBuiltinType(BFloat16Ty, BuiltinType::BFloat16);
+
// Builtin type used to help define __builtin_va_list.
VaListTagDecl = nullptr;
@@ -1651,6 +1653,8 @@
switch (T->castAs<BuiltinType>()->getKind()) {
default:
llvm_unreachable("Not a floating point type!");
+ case BuiltinType::BFloat16:
+ return Target->getBFloat16Format();
case BuiltinType::Float16:
case BuiltinType::Half:
return Target->getHalfFormat();
@@ -2045,6 +2049,10 @@
Width = Target->getLongFractWidth();
Align = Target->getLongFractAlign();
break;
+ case BuiltinType::BFloat16:
+ Width = Target->getBFloat16Width();
+ Align = Target->getBFloat16Align();
+ break;
case BuiltinType::Float16:
case BuiltinType::Half:
if (Target->hasFloat16Type() || !getLangOpts().OpenMP ||
@@ -5984,6 +5992,7 @@
case BuiltinType::Double: return DoubleRank;
case BuiltinType::LongDouble: return LongDoubleRank;
case BuiltinType::Float128: return Float128Rank;
+ case BuiltinType::BFloat16: return BFloat16Rank;
}
}
@@ -5996,6 +6005,7 @@
FloatingRank EltRank = getFloatingRank(Size);
if (Domain->isComplexType()) {
switch (EltRank) {
+ case BFloat16Rank: llvm_unreachable("Complex bfloat16 is not supported");
case Float16Rank:
case HalfRank: llvm_unreachable("Complex half is not supported");
case FloatRank: return FloatComplexTy;
@@ -6008,6 +6018,7 @@
assert(Domain->isRealFloatingType() && "Unknown domain!");
switch (EltRank) {
case Float16Rank: return HalfTy;
+ case BFloat16Rank: return BFloat16Ty;
case HalfRank: return HalfTy;
case FloatRank: return FloatTy;
case DoubleRank: return DoubleTy;
@@ -6985,6 +6996,7 @@
case BuiltinType::LongDouble: return 'D';
case BuiltinType::NullPtr: return '*'; // like char*
+ case BuiltinType::BFloat16:
case BuiltinType::Float16:
case BuiltinType::Float128:
case BuiltinType::Half:
@@ -9892,6 +9904,11 @@
// Read the base type.
switch (*Str++) {
default: llvm_unreachable("Unknown builtin type letter!");
+ case 'y':
+ assert(HowLong == 0 && !Signed && !Unsigned &&
+ "Bad modifiers used with 'y'!");
+ Type = Context.BFloat16Ty;
+ break;
case 'v':
assert(HowLong == 0 && !Signed && !Unsigned &&
"Bad modifiers used with 'v'!");
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 487ff8f..aae33c5 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -2764,6 +2764,11 @@
Out << TI->getFloat128Mangling();
break;
}
+ case BuiltinType::BFloat16: {
+ const TargetInfo *TI = &getASTContext().getTargetInfo();
+ Out << TI->getBFloat16Mangling();
+ break;
+ }
case BuiltinType::NullPtr:
Out << "Dn";
break;
@@ -3179,7 +3184,8 @@
case BuiltinType::ULongLong: EltName = "uint64_t"; break;
case BuiltinType::Double: EltName = "float64_t"; break;
case BuiltinType::Float: EltName = "float32_t"; break;
- case BuiltinType::Half: EltName = "float16_t";break;
+ case BuiltinType::Half: EltName = "float16_t"; break;
+ case BuiltinType::BFloat16: EltName = "bfloat16_t"; break;
default:
llvm_unreachable("unexpected Neon vector element type");
}
@@ -3231,6 +3237,8 @@
return "Float32";
case BuiltinType::Double:
return "Float64";
+ case BuiltinType::BFloat16:
+ return "BFloat16";
default:
llvm_unreachable("Unexpected vector element base type");
}
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index e3796ac..ca628b3 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -2114,6 +2114,7 @@
case BuiltinType::SatUShortFract:
case BuiltinType::SatUFract:
case BuiltinType::SatULongFract:
+ case BuiltinType::BFloat16:
case BuiltinType::Float128: {
DiagnosticsEngine &Diags = Context.getDiags();
unsigned DiagID = Diags.getCustomDiagID(
diff --git a/clang/lib/AST/NSAPI.cpp b/clang/lib/AST/NSAPI.cpp
index ac06d33..ace7f1c 100644
--- a/clang/lib/AST/NSAPI.cpp
+++ b/clang/lib/AST/NSAPI.cpp
@@ -486,6 +486,7 @@
case BuiltinType::OMPArraySection:
case BuiltinType::OMPArrayShaping:
case BuiltinType::OMPIterator:
+ case BuiltinType::BFloat16:
break;
}
diff --git a/clang/lib/AST/PrintfFormatString.cpp b/clang/lib/AST/PrintfFormatString.cpp
index d13ff0a..f3ac181 100644
--- a/clang/lib/AST/PrintfFormatString.cpp
+++ b/clang/lib/AST/PrintfFormatString.cpp
@@ -752,6 +752,7 @@
case BuiltinType::UInt128:
case BuiltinType::Int128:
case BuiltinType::Half:
+ case BuiltinType::BFloat16:
case BuiltinType::Float16:
case BuiltinType::Float128:
case BuiltinType::ShortAccum:
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index deb2186..7dd85d1 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2137,7 +2137,8 @@
bool Type::isArithmeticType() const {
if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
return BT->getKind() >= BuiltinType::Bool &&
- BT->getKind() <= BuiltinType::Float128;
+ BT->getKind() <= BuiltinType::Float128 &&
+ BT->getKind() != BuiltinType::BFloat16;
if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
// GCC allows forward declaration of enum types (forbid by C99 6.7.2.3p2).
// If a body isn't seen by the time we get here, return false.
@@ -2922,6 +2923,8 @@
return "unsigned __int128";
case Half:
return Policy.Half ? "half" : "__fp16";
+ case BFloat16:
+ return "__bf16";
case Float:
return "float";
case Double:
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 366f4d8..57c11ca 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -375,6 +375,7 @@
case BuiltinType::SatUShortFract:
case BuiltinType::SatUFract:
case BuiltinType::SatULongFract:
+ case BuiltinType::BFloat16:
llvm_unreachable("Builtin type needs extra local data!");
// Fall through, if the impossible happens.
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index a3c8da5..7f360b7 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -36,6 +36,7 @@
HasLegalHalfType = false;
HasFloat128 = false;
HasFloat16 = false;
+ HasBFloat16 = false;
PointerWidth = PointerAlign = 32;
BoolWidth = BoolAlign = 8;
IntWidth = IntAlign = 32;
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 5357d31..080571c 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -70,6 +70,9 @@
LongDoubleWidth = LongDoubleAlign = SuitableAlign = 128;
LongDoubleFormat = &llvm::APFloat::IEEEquad();
+ BFloat16Width = BFloat16Align = 16;
+ BFloat16Format = &llvm::APFloat::BFloat();
+
// Make __builtin_ms_va_list available.
HasBuiltinMSVaList = true;
@@ -360,6 +363,7 @@
HasMTE = false;
HasTME = false;
HasMatMul = false;
+ HasBFloat16 = false;
ArchKind = llvm::AArch64::ArchKind::ARMV8A;
for (const auto &Feature : Features) {
@@ -397,6 +401,8 @@
HasTME = true;
if (Feature == "+i8mm")
HasMatMul = true;
+ if (Feature == "+bf16")
+ HasBFloat16 = true;
}
setDataLayout();
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 6af44d1..4a58c66 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -119,6 +119,7 @@
int getEHDataRegisterNumber(unsigned RegNo) const override;
+ const char *getBFloat16Mangling() const override { return "u6__bf16"; };
bool hasInt128Type() const override;
bool hasExtIntType() const override { return true; }
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 08b6a9c..5e605ab 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -25,6 +25,9 @@
IsAAPCS = true;
DoubleAlign = LongLongAlign = LongDoubleAlign = SuitableAlign = 64;
+ BFloat16Width = BFloat16Align = 16;
+ BFloat16Format = &llvm::APFloat::BFloat();
+
const llvm::Triple &T = getTriple();
bool IsNetBSD = T.isOSNetBSD();
@@ -74,6 +77,8 @@
DoubleAlign = LongLongAlign = LongDoubleAlign = SuitableAlign = 64;
else
DoubleAlign = LongLongAlign = LongDoubleAlign = SuitableAlign = 32;
+ BFloat16Width = BFloat16Align = 16;
+ BFloat16Format = &llvm::APFloat::BFloat();
WCharType = SignedInt;
@@ -428,6 +433,7 @@
HasMatMul = 0;
HasFloat16 = true;
ARMCDECoprocMask = 0;
+ HasBFloat16 = false;
// This does not diagnose illegal cases like having both
// "+vfpv2" and "+vfpv3" or having "+neon" and "-fp64".
@@ -498,6 +504,8 @@
Feature <= "+cdecp7") {
unsigned Coproc = Feature.back() - '0';
ARMCDECoprocMask |= (1U << Coproc);
+ } else if (Feature == "+bf16") {
+ HasBFloat16 = true;
}
}
@@ -547,6 +555,10 @@
.Default(false);
}
+bool ARMTargetInfo::hasBFloat16Type() const {
+ return HasBFloat16 && !SoftFloat;
+}
+
bool ARMTargetInfo::isValidCPUName(StringRef Name) const {
return Name == "generic" ||
llvm::ARM::parseCPUArch(Name) != llvm::ARM::ArchKind::INVALID;
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index d2f4e69..1e80f74 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -137,6 +137,8 @@
bool hasFeature(StringRef Feature) const override;
+ bool hasBFloat16Type() const override;
+
bool isValidCPUName(StringRef Name) const override;
void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
@@ -184,6 +186,8 @@
bool hasSjLjLowering() const override;
bool hasExtIntType() const override { return true; }
+
+ const char *getBFloat16Mangling() const override { return "u6__bf16"; };
};
class LLVM_LIBRARY_VISIBILITY ARMleTargetInfo : public ARMTargetInfo {
diff --git a/clang/lib/CodeGen/ABIInfo.h b/clang/lib/CodeGen/ABIInfo.h
index 4e9d444..bb40dac 100644
--- a/clang/lib/CodeGen/ABIInfo.h
+++ b/clang/lib/CodeGen/ABIInfo.h
@@ -60,6 +60,8 @@
virtual bool supportsSwift() const { return false; }
+ virtual bool allowBFloatArgsAndRet() const { return false; }
+
CodeGen::CGCXXABI &getCXXABI() const;
ASTContext &getContext() const;
llvm::LLVMContext &getVMContext() const;
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index a73245a..6b7863a 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -4480,8 +4480,9 @@
static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
NeonTypeFlags TypeFlags,
- bool HasLegalHalfType=true,
- bool V1Ty=false) {
+ bool HasLegalHalfType = true,
+ bool V1Ty = false,
+ bool AllowBFloatArgsAndRet = true) {
int IsQuad = TypeFlags.isQuad();
switch (TypeFlags.getEltType()) {
case NeonTypeFlags::Int8:
@@ -4490,6 +4491,11 @@
case NeonTypeFlags::Int16:
case NeonTypeFlags::Poly16:
return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
+ case NeonTypeFlags::BFloat16:
+ if (AllowBFloatArgsAndRet)
+ return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
+ else
+ return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
case NeonTypeFlags::Float16:
if (HasLegalHalfType)
return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
@@ -5505,8 +5511,11 @@
bool Usgn = Type.isUnsigned();
bool Quad = Type.isQuad();
const bool HasLegalHalfType = getTarget().hasLegalHalfType();
+ const bool AllowBFloatArgsAndRet =
+ getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
- llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType);
+ llvm::VectorType *VTy = GetNeonType(this, Type, HasLegalHalfType, false,
+ AllowBFloatArgsAndRet);
llvm::Type *Ty = VTy;
if (!Ty)
return nullptr;
@@ -6975,7 +6984,9 @@
bool rightShift = false;
llvm::VectorType *VTy = GetNeonType(this, Type,
- getTarget().hasLegalHalfType());
+ getTarget().hasLegalHalfType(),
+ false,
+ getTarget().hasBFloat16Type());
llvm::Type *Ty = VTy;
if (!Ty)
return nullptr;
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 5be8e77..cc50ec6 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -772,6 +772,7 @@
case BuiltinType::Float:
case BuiltinType::LongDouble:
case BuiltinType::Float16:
+ case BuiltinType::BFloat16:
case BuiltinType::Float128:
case BuiltinType::Double:
// FIXME: For targets where long double and __float128 have the same size,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 096e0bd..3d7a8730 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -112,6 +112,7 @@
Int32Ty = llvm::Type::getInt32Ty(LLVMContext);
Int64Ty = llvm::Type::getInt64Ty(LLVMContext);
HalfTy = llvm::Type::getHalfTy(LLVMContext);
+ BFloatTy = llvm::Type::getBFloatTy(LLVMContext);
FloatTy = llvm::Type::getFloatTy(LLVMContext);
DoubleTy = llvm::Type::getDoubleTy(LLVMContext);
PointerWidthInBits = C.getTargetInfo().getPointerWidth(0);
diff --git a/clang/lib/CodeGen/CodeGenTypeCache.h b/clang/lib/CodeGen/CodeGenTypeCache.h
index ed4b773..20a3263 100644
--- a/clang/lib/CodeGen/CodeGenTypeCache.h
+++ b/clang/lib/CodeGen/CodeGenTypeCache.h
@@ -35,8 +35,8 @@
/// i8, i16, i32, and i64
llvm::IntegerType *Int8Ty, *Int16Ty, *Int32Ty, *Int64Ty;
- /// float, double
- llvm::Type *HalfTy, *FloatTy, *DoubleTy;
+ /// half, bfloat, float, double
+ llvm::Type *HalfTy, *BFloatTy, *FloatTy, *DoubleTy;
/// int
llvm::IntegerType *IntTy;
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index a9ca5e2..bae934b 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -300,6 +300,8 @@
else
return llvm::Type::getInt16Ty(VMContext);
}
+ if (&format == &llvm::APFloat::BFloat())
+ return llvm::Type::getBFloatTy(VMContext);
if (&format == &llvm::APFloat::IEEEsingle())
return llvm::Type::getFloatTy(VMContext);
if (&format == &llvm::APFloat::IEEEdouble())
@@ -498,6 +500,7 @@
Context.getLangOpts().NativeHalfType ||
!Context.getTargetInfo().useFP16ConversionIntrinsics());
break;
+ case BuiltinType::BFloat16:
case BuiltinType::Float:
case BuiltinType::Double:
case BuiltinType::LongDouble:
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index af3714b..677d652 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3027,6 +3027,7 @@
case BuiltinType::SatUShortFract:
case BuiltinType::SatUFract:
case BuiltinType::SatULongFract:
+ case BuiltinType::BFloat16:
return false;
case BuiltinType::Dependent:
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 6747862..564eb09 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -5375,6 +5375,10 @@
bool isLegalVectorTypeForSwift(CharUnits totalSize, llvm::Type *eltTy,
unsigned elts) const override;
+
+ bool allowBFloatArgsAndRet() const override {
+ return getTarget().hasBFloat16Type();
+ }
};
class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
@@ -5980,11 +5984,14 @@
private:
ABIKind Kind;
+ bool IsFloatABISoftFP;
public:
ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind)
: SwiftABIInfo(CGT), Kind(_Kind) {
setCCs();
+ IsFloatABISoftFP = CGT.getCodeGenOpts().FloatABI == "softfp" ||
+ CGT.getCodeGenOpts().FloatABI == ""; // default
}
bool isEABI() const {
@@ -6015,6 +6022,10 @@
ABIKind getABIKind() const { return Kind; }
+ bool allowBFloatArgsAndRet() const override {
+ return !IsFloatABISoftFP && getTarget().hasBFloat16Type();
+ }
+
private:
ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic,
unsigned functionCallConv) const;
@@ -6254,9 +6265,9 @@
if (isIllegalVectorType(Ty))
return coerceIllegalVector(Ty);
- // _Float16 and __fp16 get passed as if it were an int or float, but with
- // the top 16 bits unspecified. This is not done for OpenCL as it handles the
- // half type natively, and does not need to interwork with AAPCS code.
+ // _Float16 and __fp16 get passed as if it were an int or float, but
+ // with the top 16 bits unspecified. This is not done for OpenCL as it handles
+ // the half type natively, and does not need to interwork with AAPCS code.
if ((Ty->isFloat16Type() || Ty->isHalfType()) &&
!getContext().getLangOpts().NativeHalfArgsAndReturns) {
llvm::Type *ResType = IsAAPCS_VFP ?
@@ -6265,6 +6276,13 @@
return ABIArgInfo::getDirect(ResType);
}
+ // __bf16 gets passed using the bfloat IR type, or using i32 but
+ // with the top 16 bits unspecified.
+ if (Ty->isBFloat16Type() && IsFloatABISoftFP) {
+ llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());
+ return ABIArgInfo::getDirect(ResType);
+ }
+
if (!isAggregateTypeForABI(Ty)) {
// Treat an enum type as its underlying type.
if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
@@ -6458,10 +6476,13 @@
// Large vector types should be returned via memory.
if (getContext().getTypeSize(RetTy) > 128)
return getNaturalAlignIndirect(RetTy);
- // FP16 vectors should be converted to integer vectors
- if (!getTarget().hasLegalHalfType() &&
+ // TODO: FP16/BF16 vectors should be converted to integer vectors
+ // This check is similar to isIllegalVectorType - refactor?
+ if ((!getTarget().hasLegalHalfType() &&
(VT->getElementType()->isFloat16Type() ||
- VT->getElementType()->isHalfType()))
+ VT->getElementType()->isHalfType())) ||
+ (IsFloatABISoftFP &&
+ VT->getElementType()->isBFloat16Type()))
return coerceIllegalVector(RetTy);
}
@@ -6476,6 +6497,15 @@
return ABIArgInfo::getDirect(ResType);
}
+ // if we're using the softfp float abi, __bf16 get returned as if it were an
+ // int but with the top 16 bits unspecified.
+ if (RetTy->isBFloat16Type()) {
+ llvm::Type *ResType = IsAAPCS_VFP ?
+ llvm::Type::getBFloatTy(getVMContext()) :
+ llvm::Type::getInt32Ty(getVMContext());
+ return ABIArgInfo::getDirect(ResType);
+ }
+
if (!isAggregateTypeForABI(RetTy)) {
// Treat an enum type as its underlying type.
if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
@@ -6562,12 +6592,17 @@
/// isIllegalVector - check whether Ty is an illegal vector type.
bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
if (const VectorType *VT = Ty->getAs<VectorType> ()) {
- // On targets that don't support FP16, FP16 is expanded into float, and we
- // don't want the ABI to depend on whether or not FP16 is supported in
- // hardware. Thus return false to coerce FP16 vectors into integer vectors.
- if (!getTarget().hasLegalHalfType() &&
+ // On targets that don't support half, fp16 or bfloat, they are expanded
+ // into float, and we don't want the ABI to depend on whether or not they
+ // are supported in hardware. Thus return false to coerce vectors of these
+ // types into integer vectors.
+ // We do not depend on hasLegalHalfType for bfloat as it is a
+ // separate IR type.
+ if ((!getTarget().hasLegalHalfType() &&
(VT->getElementType()->isFloat16Type() ||
- VT->getElementType()->isHalfType()))
+ VT->getElementType()->isHalfType())) ||
+ (IsFloatABISoftFP &&
+ VT->getElementType()->isBFloat16Type()))
return true;
if (isAndroid()) {
// Android shipped using Clang 3.1, which supported a slightly different
@@ -6619,6 +6654,7 @@
} else {
if (const VectorType *VT = Ty->getAs<VectorType>())
return (VT->getElementType()->isFloat16Type() ||
+ VT->getElementType()->isBFloat16Type() ||
VT->getElementType()->isHalfType());
return false;
}
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index ee35f3a..7d79297 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -50,6 +50,7 @@
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
+ case tok::kw___bf16:
case tok::kw__Float16:
case tok::kw___float128:
case tok::kw_wchar_t:
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index e503f1e..0d1e812 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -753,6 +753,7 @@
case BuiltinType::SatUShortFract:
case BuiltinType::SatUFract:
case BuiltinType::SatULongFract:
+ case BuiltinType::BFloat16:
IgnoreResults = true;
return;
case BuiltinType::ObjCId:
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index d5d4993..7e76197 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3842,6 +3842,10 @@
isInvalid = DS.SetTypeSpecType(DeclSpec::TST_half, Loc, PrevSpec,
DiagID, Policy);
break;
+ case tok::kw___bf16:
+ isInvalid = DS.SetTypeSpecType(DeclSpec::TST_BFloat16, Loc, PrevSpec,
+ DiagID, Policy);
+ break;
case tok::kw_float:
isInvalid = DS.SetTypeSpecType(DeclSpec::TST_float, Loc, PrevSpec,
DiagID, Policy);
@@ -4942,6 +4946,7 @@
case tok::kw_char32_t:
case tok::kw_int:
case tok::kw__ExtInt:
+ case tok::kw___bf16:
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
@@ -5023,6 +5028,7 @@
case tok::kw_int:
case tok::kw__ExtInt:
case tok::kw_half:
+ case tok::kw___bf16:
case tok::kw_float:
case tok::kw_double:
case tok::kw__Accum:
@@ -5190,6 +5196,7 @@
case tok::kw_int:
case tok::kw__ExtInt:
case tok::kw_half:
+ case tok::kw___bf16:
case tok::kw_float:
case tok::kw_double:
case tok::kw__Accum:
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 1ef352e..bd00320 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1514,6 +1514,7 @@
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
+ case tok::kw___bf16:
case tok::kw__Float16:
case tok::kw___float128:
case tok::kw_void:
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 454b725..bf7ada3 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -2201,6 +2201,9 @@
case tok::kw___int128:
DS.SetTypeSpecType(DeclSpec::TST_int128, Loc, PrevSpec, DiagID, Policy);
break;
+ case tok::kw___bf16:
+ DS.SetTypeSpecType(DeclSpec::TST_BFloat16, Loc, PrevSpec, DiagID, Policy);
+ break;
case tok::kw_half:
DS.SetTypeSpecType(DeclSpec::TST_half, Loc, PrevSpec, DiagID, Policy);
break;
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index 9179cc9..948162c 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -1640,6 +1640,7 @@
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
+ case tok::kw___bf16:
case tok::kw__Float16:
case tok::kw___float128:
case tok::kw_void:
@@ -1753,6 +1754,7 @@
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
+ case tok::kw___bf16:
case tok::kw__Float16:
case tok::kw___float128:
case tok::kw_void:
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index 834e253..6ad50e1 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -368,6 +368,7 @@
case TST_unspecified:
case TST_void:
case TST_wchar:
+ case TST_BFloat16:
#define GENERIC_IMAGE_TYPE(ImgType, Id) case TST_##ImgType##_t:
#include "clang/Basic/OpenCLImageTypes.def"
return false;
@@ -566,6 +567,7 @@
case DeclSpec::TST_underlyingType: return "__underlying_type";
case DeclSpec::TST_unknown_anytype: return "__unknown_anytype";
case DeclSpec::TST_atomic: return "_Atomic";
+ case DeclSpec::TST_BFloat16: return "__bf16";
#define GENERIC_IMAGE_TYPE(ImgType, Id) \
case DeclSpec::TST_##ImgType##_t: \
return #ImgType "_t";
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index f5bc4bd..f483f73 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -2789,6 +2789,20 @@
return;
}
+ // Can't cast to or from bfloat
+ if (DestType->isBFloat16Type() && !SrcType->isBFloat16Type()) {
+ Self.Diag(SrcExpr.get()->getExprLoc(), diag::err_cast_to_bfloat16)
+ << SrcExpr.get()->getSourceRange();
+ SrcExpr = ExprError();
+ return;
+ }
+ if (SrcType->isBFloat16Type() && !DestType->isBFloat16Type()) {
+ Self.Diag(SrcExpr.get()->getExprLoc(), diag::err_cast_from_bfloat16)
+ << SrcExpr.get()->getSourceRange();
+ SrcExpr = ExprError();
+ return;
+ }
+
// If either type is a pointer, the other type has to be either an
// integer or a pointer.
if (!DestType->isArithmeticType()) {
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 64df274..86b18e2 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1961,6 +1961,9 @@
case NeonTypeFlags::Float64:
assert(!shift && "cannot shift float types!");
return (1 << IsQuad) - 1;
+ case NeonTypeFlags::BFloat16:
+ assert(!shift && "cannot shift float types!");
+ return (4 << IsQuad) - 1;
}
llvm_unreachable("Invalid NeonTypeFlag!");
}
@@ -2000,6 +2003,8 @@
return Context.FloatTy;
case NeonTypeFlags::Float64:
return Context.DoubleTy;
+ case NeonTypeFlags::BFloat16:
+ return Context.BFloat16Ty;
}
llvm_unreachable("Invalid NeonTypeFlag!");
}
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index aec3d55..6e52c95 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -138,6 +138,7 @@
case tok::kw_half:
case tok::kw_float:
case tok::kw_double:
+ case tok::kw___bf16:
case tok::kw__Float16:
case tok::kw___float128:
case tok::kw_wchar_t:
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 3c07b7d..d7a39cf 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8133,6 +8133,11 @@
return ResTy;
}
+ // And if they're both bfloat (which isn't arithmetic), that's fine too.
+ if (LHSTy->isBFloat16Type() && RHSTy->isBFloat16Type()) {
+ return LHSTy;
+ }
+
// If both operands are the same structure or union type, the result is that
// type.
if (const RecordType *LHSRT = LHSTy->getAs<RecordType>()) { // C99 6.5.15p3
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 2180a9c..d7fc966 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1870,6 +1870,10 @@
// FIXME: disable conversions between long double and __float128 if
// their representation is different until there is back end support
// We of course allow this conversion if long double is really double.
+
+ // Conversions between bfloat and other floats are not permitted.
+ if (FromType == S.Context.BFloat16Ty || ToType == S.Context.BFloat16Ty)
+ return false;
if (&S.Context.getFloatTypeSemantics(FromType) !=
&S.Context.getFloatTypeSemantics(ToType)) {
bool Float128AndLongDouble = ((FromType == S.Context.Float128Ty &&
@@ -1888,6 +1892,10 @@
ToType->isIntegralType(S.Context)) ||
(FromType->isIntegralOrUnscopedEnumerationType() &&
ToType->isRealFloatingType())) {
+ // Conversions between bfloat and int are not permitted.
+ if (FromType->isBFloat16Type() || ToType->isBFloat16Type())
+ return false;
+
// Floating-integral conversions (C++ 4.9).
SCS.Second = ICK_Floating_Integral;
FromType = ToType.getUnqualifiedType();
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index 466cb08..7b77d1c 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -881,6 +881,7 @@
case TST_auto:
case TST_auto_type:
case TST_decltype_auto:
+ case TST_BFloat16:
#define GENERIC_IMAGE_TYPE(ImgType, Id) case TST_##ImgType##_t:
#include "clang/Basic/OpenCLImageTypes.def"
case TST_unknown_anytype:
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index fc4a231..93bb2e1 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1521,6 +1521,12 @@
Result = Context.Float16Ty;
break;
case DeclSpec::TST_half: Result = Context.HalfTy; break;
+ case DeclSpec::TST_BFloat16:
+ if (!S.Context.getTargetInfo().hasBFloat16Type())
+ S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported)
+ << "__bf16";
+ Result = Context.BFloat16Ty;
+ break;
case DeclSpec::TST_float: Result = Context.FloatTy; break;
case DeclSpec::TST_double:
if (DS.getTypeSpecWidth() == DeclSpec::TSW_long)
@@ -7673,7 +7679,8 @@
BTy->getKind() == BuiltinType::LongLong ||
BTy->getKind() == BuiltinType::ULongLong ||
BTy->getKind() == BuiltinType::Float ||
- BTy->getKind() == BuiltinType::Half;
+ BTy->getKind() == BuiltinType::Half ||
+ BTy->getKind() == BuiltinType::BFloat16;
}
/// HandleNeonVectorTypeAttr - The "neon_vector_type" and
diff --git a/clang/lib/Serialization/ASTCommon.cpp b/clang/lib/Serialization/ASTCommon.cpp
index a7fd5de..bf583b0 100644
--- a/clang/lib/Serialization/ASTCommon.cpp
+++ b/clang/lib/Serialization/ASTCommon.cpp
@@ -252,6 +252,9 @@
case BuiltinType::OMPIterator:
ID = PREDEF_TYPE_OMP_ITERATOR;
break;
+ case BuiltinType::BFloat16:
+ ID = PREDEF_TYPE_BFLOAT16_ID;
+ break;
}
return TypeIdx(ID);
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 2afa915..8798706 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -6844,6 +6844,9 @@
case PREDEF_TYPE_INT128_ID:
T = Context.Int128Ty;
break;
+ case PREDEF_TYPE_BFLOAT16_ID:
+ T = Context.BFloat16Ty;
+ break;
case PREDEF_TYPE_HALF_ID:
T = Context.HalfTy;
break;