[ARM] Armv8.2-A FP16 code generation (part 3/3)
This adds most of the FP16 codegen support, but these areas need further work:
- FP16 literals and immediates are not properly supported yet (e.g. literal
pool needs work),
- Instructions that are generated from intrinsics (e.g. vabs) haven't been
added.
This will be addressed in follow-up patches.
Differential Revision: https://reviews.llvm.org/D42849
llvm-svn: 324321
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 8e58bd3..72fede7 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -1042,6 +1042,7 @@
setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
setOperationAction(ISD::SETCC, MVT::i32, Expand);
+ setOperationAction(ISD::SETCC, MVT::f16, Expand);
setOperationAction(ISD::SETCC, MVT::f32, Expand);
setOperationAction(ISD::SETCC, MVT::f64, Expand);
setOperationAction(ISD::SELECT, MVT::i32, Custom);
@@ -12746,6 +12747,24 @@
return false;
}
+bool ARMTargetLowering::isFNegFree(EVT VT) const {
+ if (!VT.isSimple())
+ return false;
+
+ // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
+ // negate values directly (fneg is free). So, we don't want to let the DAG
+ // combiner rewrite fneg into xors and some other instructions. For f16 and
+ // FullFP16 argument passing, some bitcast nodes may be introduced,
+ // triggering this DAG combine rewrite, so we are avoiding that with this.
+ switch (VT.getSimpleVT().SimpleTy) {
+ default: break;
+ case MVT::f16:
+ return Subtarget->hasFullFP16();
+ }
+
+ return false;
+}
+
bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
EVT VT = ExtVal.getValueType();
@@ -13842,6 +13861,8 @@
bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
if (!Subtarget->hasVFP3())
return false;
+ if (VT == MVT::f16 && Subtarget->hasFullFP16())
+ return ARM_AM::getFP16Imm(Imm) != -1;
if (VT == MVT::f32)
return ARM_AM::getFP32Imm(Imm) != -1;
if (VT == MVT::f64 && !Subtarget->isFPOnlySP())
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index b096331..b196e23 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -331,6 +331,7 @@
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override;
bool isTruncateFree(EVT SrcVT, EVT DstVT) const override;
bool isZExtFree(SDValue Val, EVT VT2) const override;
+ bool isFNegFree(EVT VT) const override;
bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 48c1a38..29c68f7 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -395,9 +395,9 @@
let TwoOperandAliasConstraint = "$Sn = $Sd" in
def VDIVH : AHbI<0b11101, 0b00, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm),
IIC_fpDIV16, "vdiv", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fdiv HPR:$Sn, HPR:$Sm))]>,
Sched<[WriteFPDIV32]>;
let TwoOperandAliasConstraint = "$Dn = $Dd" in
@@ -420,9 +420,9 @@
let TwoOperandAliasConstraint = "$Sn = $Sd" in
def VMULH : AHbI<0b11100, 0b10, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm),
IIC_fpMUL16, "vmul", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fmul HPR:$Sn, HPR:$Sm))]>,
Sched<[WriteFPMUL32, ReadFPMUL, ReadFPMUL]>;
def VNMULD : ADbI<0b11100, 0b10, 1, 0,
@@ -442,9 +442,9 @@
}
def VNMULH : AHbI<0b11100, 0b10, 1, 0,
- (outs SPR:$Sd), (ins SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sn, HPR:$Sm),
IIC_fpMUL16, "vnmul", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fneg (fmul HPR:$Sn, HPR:$Sm)))]>,
Sched<[WriteFPMUL32, ReadFPMUL, ReadFPMUL]>;
multiclass vsel_inst<string op, bits<2> opc, int CC> {
@@ -525,9 +525,9 @@
}
def VCMPEH : AHuI<0b11101, 0b11, 0b0100, 0b11, 0,
- (outs), (ins SPR:$Sd, SPR:$Sm),
+ (outs), (ins HPR:$Sd, HPR:$Sm),
IIC_fpCMP16, "vcmpe", ".f16\t$Sd, $Sm",
- []>;
+ [(arm_cmpfp HPR:$Sd, HPR:$Sm, (i32 1))]>;
def VCMPD : ADuI<0b11101, 0b11, 0b0100, 0b01, 0,
(outs), (ins DPR:$Dd, DPR:$Dm),
@@ -544,9 +544,9 @@
}
def VCMPH : AHuI<0b11101, 0b11, 0b0100, 0b01, 0,
- (outs), (ins SPR:$Sd, SPR:$Sm),
+ (outs), (ins HPR:$Sd, HPR:$Sm),
IIC_fpCMP16, "vcmp", ".f16\t$Sd, $Sm",
- []>;
+ [(arm_cmpfp HPR:$Sd, HPR:$Sm, (i32 0))]>;
} // Defs = [FPSCR_NZCV]
//===----------------------------------------------------------------------===//
@@ -771,7 +771,7 @@
SDPatternOperator node = null_frag> {
let PostEncoderMethod = "", DecoderNamespace = "VFPV8" in {
def SH : AHuInp<0b11101, 0b11, 0b1100, 0b11, 0,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs SPR:$Sd), (ins HPR:$Sm),
NoItinerary, !strconcat("vcvt", opc, ".s32.f16\t$Sd, $Sm"),
[]>,
Requires<[HasFullFP16]> {
@@ -779,7 +779,7 @@
}
def UH : AHuInp<0b11101, 0b11, 0b1100, 0b01, 0,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs SPR:$Sd), (ins HPR:$Sm),
NoItinerary, !strconcat("vcvt", opc, ".u32.f16\t$Sd, $Sm"),
[]>,
Requires<[HasFullFP16]> {
@@ -834,6 +834,17 @@
}
let Predicates = [HasFPARMv8] in {
+ let Predicates = [HasFullFP16] in {
+ def : Pat<(i32 (fp_to_sint (node HPR:$a))),
+ (COPY_TO_REGCLASS
+ (!cast<Instruction>(NAME#"SH") HPR:$a),
+ GPR)>;
+
+ def : Pat<(i32 (fp_to_uint (node HPR:$a))),
+ (COPY_TO_REGCLASS
+ (!cast<Instruction>(NAME#"UH") HPR:$a),
+ GPR)>;
+ }
def : Pat<(i32 (fp_to_sint (node SPR:$a))),
(COPY_TO_REGCLASS
(!cast<Instruction>(NAME#"SS") SPR:$a),
@@ -875,9 +886,9 @@
}
def VNEGH : AHuI<0b11101, 0b11, 0b0001, 0b01, 0,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sm),
IIC_fpUNA16, "vneg", ".f16\t$Sd, $Sm",
- []>;
+ [(set HPR:$Sd, (fneg HPR:$Sm))]>;
multiclass vrint_inst_zrx<string opc, bit op, bit op2, SDPatternOperator node> {
def H : AHuI<0b11101, 0b11, 0b0110, 0b11, 0,
@@ -1313,13 +1324,16 @@
(VSITOS (VLDRS addrmode5:$a))>;
def VSITOH : AVConv1IHs_Encode<0b11101, 0b11, 0b1000, 0b1001,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs HPR:$Sd), (ins SPR:$Sm),
IIC_fpCVTIH, "vcvt", ".f16.s32\t$Sd, $Sm",
[]>,
Sched<[WriteFPCVT]> {
let Inst{7} = 1; // s32
}
+def : VFPNoNEONPat<(f16 (sint_to_fp GPR:$a)),
+ (VSITOH (COPY_TO_REGCLASS GPR:$a, SPR))>;
+
def VUITOD : AVConv1IDs_Encode<0b11101, 0b11, 0b1000, 0b1011,
(outs DPR:$Dd), (ins SPR:$Sm),
IIC_fpCVTID, "vcvt", ".f64.u32\t$Dd, $Sm",
@@ -1355,13 +1369,16 @@
(VUITOS (VLDRS addrmode5:$a))>;
def VUITOH : AVConv1IHs_Encode<0b11101, 0b11, 0b1000, 0b1001,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs HPR:$Sd), (ins SPR:$Sm),
IIC_fpCVTIH, "vcvt", ".f16.u32\t$Sd, $Sm",
[]>,
Sched<[WriteFPCVT]> {
let Inst{7} = 0; // u32
}
+def : VFPNoNEONPat<(f16 (uint_to_fp GPR:$a)),
+ (VUITOH (COPY_TO_REGCLASS GPR:$a, SPR))>;
+
// FP -> Int:
class AVConv1IsD_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
@@ -1456,13 +1473,16 @@
(VSTRS (VTOSIZS SPR:$a), addrmode5:$ptr)>;
def VTOSIZH : AVConv1IsH_Encode<0b11101, 0b11, 0b1101, 0b1001,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs SPR:$Sd), (ins HPR:$Sm),
IIC_fpCVTHI, "vcvt", ".s32.f16\t$Sd, $Sm",
[]>,
Sched<[WriteFPCVT]> {
let Inst{7} = 1; // Z bit
}
+def : VFPNoNEONPat<(i32 (fp_to_sint HPR:$a)),
+ (COPY_TO_REGCLASS (VTOSIZH HPR:$a), GPR)>;
+
def VTOUIZD : AVConv1IsD_Encode<0b11101, 0b11, 0b1100, 0b1011,
(outs SPR:$Sd), (ins DPR:$Dm),
IIC_fpCVTDI, "vcvt", ".u32.f64\t$Sd, $Dm",
@@ -1499,13 +1519,16 @@
(VSTRS (VTOUIZS SPR:$a), addrmode5:$ptr)>;
def VTOUIZH : AVConv1IsH_Encode<0b11101, 0b11, 0b1100, 0b1001,
- (outs SPR:$Sd), (ins SPR:$Sm),
+ (outs SPR:$Sd), (ins HPR:$Sm),
IIC_fpCVTHI, "vcvt", ".u32.f16\t$Sd, $Sm",
[]>,
Sched<[WriteFPCVT]> {
let Inst{7} = 1; // Z bit
}
+def : VFPNoNEONPat<(i32 (fp_to_uint HPR:$a)),
+ (COPY_TO_REGCLASS (VTOUIZH HPR:$a), GPR)>;
+
// And the Z bit '0' variants, i.e. use the rounding mode specified by FPSCR.
let Uses = [FPSCR] in {
def VTOSIRD : AVConv1IsD_Encode<0b11101, 0b11, 0b1101, 0b1011,
@@ -1789,9 +1812,10 @@
}
def VMLAH : AHbI<0b11100, 0b00, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpMAC16, "vmla", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fadd_mlx (fmul_su HPR:$Sn, HPR:$Sm),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFPVMLx,DontUseFusedMAC]>;
@@ -1801,6 +1825,10 @@
def : Pat<(fadd_mlx SPR:$dstin, (fmul_su SPR:$a, SPR:$b)),
(VMLAS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP2,DontUseNEONForFP, UseFPVMLx,DontUseFusedMAC]>;
+def : Pat<(fadd_mlx HPR:$dstin, (fmul_su HPR:$a, HPR:$b)),
+ (VMLAH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP, UseFPVMLx,DontUseFusedMAC]>;
+
def VMLSD : ADbI<0b11100, 0b00, 1, 0,
(outs DPR:$Dd), (ins DPR:$Ddin, DPR:$Dn, DPR:$Dm),
@@ -1825,9 +1853,10 @@
}
def VMLSH : AHbI<0b11100, 0b00, 1, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpMAC16, "vmls", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fadd_mlx (fneg (fmul_su HPR:$Sn, HPR:$Sm)),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFPVMLx,DontUseFusedMAC]>;
@@ -1837,6 +1866,9 @@
def : Pat<(fsub_mlx SPR:$dstin, (fmul_su SPR:$a, SPR:$b)),
(VMLSS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP2,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
+def : Pat<(fsub_mlx HPR:$dstin, (fmul_su HPR:$a, HPR:$b)),
+ (VMLSH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
def VNMLAD : ADbI<0b11100, 0b01, 1, 0,
(outs DPR:$Dd), (ins DPR:$Ddin, DPR:$Dn, DPR:$Dm),
@@ -1861,9 +1893,10 @@
}
def VNMLAH : AHbI<0b11100, 0b01, 1, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpMAC16, "vnmla", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fsub_mlx (fneg (fmul_su HPR:$Sn, HPR:$Sm)),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFPVMLx,DontUseFusedMAC]>;
@@ -1874,6 +1907,9 @@
def : Pat<(fsub_mlx (fneg (fmul_su SPR:$a, SPR:$b)), SPR:$dstin),
(VNMLAS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP2,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
+def : Pat<(fsub_mlx (fneg (fmul_su HPR:$a, HPR:$b)), HPR:$dstin),
+ (VNMLAH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
// (-dst - (a * b)) -> -(dst + (a * b))
def : Pat<(fsub_mlx (fneg DPR:$dstin), (fmul_su DPR:$a, (f64 DPR:$b))),
@@ -1882,6 +1918,9 @@
def : Pat<(fsub_mlx (fneg SPR:$dstin), (fmul_su SPR:$a, SPR:$b)),
(VNMLAS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP2,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
+def : Pat<(fsub_mlx (fneg HPR:$dstin), (fmul_su HPR:$a, HPR:$b)),
+ (VNMLAH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
def VNMLSD : ADbI<0b11100, 0b01, 0, 0,
(outs DPR:$Dd), (ins DPR:$Ddin, DPR:$Dn, DPR:$Dm),
@@ -1905,9 +1944,9 @@
}
def VNMLSH : AHbI<0b11100, 0b01, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpMAC16, "vnmls", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fsub_mlx (fmul_su HPR:$Sn, HPR:$Sm), HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFPVMLx,DontUseFusedMAC]>;
@@ -1917,6 +1956,9 @@
def : Pat<(fsub_mlx (fmul_su SPR:$a, SPR:$b), SPR:$dstin),
(VNMLSS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP2,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
+def : Pat<(fsub_mlx (fmul_su HPR:$a, HPR:$b), HPR:$dstin),
+ (VNMLSH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFPVMLx,DontUseFusedMAC]>;
//===----------------------------------------------------------------------===//
// Fused FP Multiply-Accumulate Operations.
@@ -1943,9 +1985,10 @@
}
def VFMAH : AHbI<0b11101, 0b10, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpFMAC16, "vfma", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fadd_mlx (fmul_su HPR:$Sn, HPR:$Sm),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFusedMAC]>,
Sched<[WriteFPMAC32, ReadFPMAC, ReadFPMUL, ReadFPMUL]>;
@@ -1956,6 +1999,9 @@
def : Pat<(fadd_mlx SPR:$dstin, (fmul_su SPR:$a, SPR:$b)),
(VFMAS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP4,DontUseNEONForFP,UseFusedMAC]>;
+def : Pat<(fadd_mlx HPR:$dstin, (fmul_su HPR:$a, HPR:$b)),
+ (VFMAH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFusedMAC]>;
// Match @llvm.fma.* intrinsics
// (fma x, y, z) -> (vfms z, x, y)
@@ -1988,9 +2034,10 @@
}
def VFMSH : AHbI<0b11101, 0b10, 1, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpFMAC16, "vfms", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fadd_mlx (fneg (fmul_su HPR:$Sn, HPR:$Sm)),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFusedMAC]>,
Sched<[WriteFPMAC32, ReadFPMAC, ReadFPMUL, ReadFPMUL]>;
@@ -2001,6 +2048,9 @@
def : Pat<(fsub_mlx SPR:$dstin, (fmul_su SPR:$a, SPR:$b)),
(VFMSS SPR:$dstin, SPR:$a, SPR:$b)>,
Requires<[HasVFP4,DontUseNEONForFP,UseFusedMAC]>;
+def : Pat<(fsub_mlx HPR:$dstin, (fmul_su HPR:$a, HPR:$b)),
+ (VFMSH HPR:$dstin, HPR:$a, HPR:$b)>,
+ Requires<[HasFullFP16,DontUseNEONForFP,UseFusedMAC]>;
// Match @llvm.fma.* intrinsics
// (fma (fneg x), y, z) -> (vfms z, x, y)
@@ -2040,9 +2090,10 @@
}
def VFNMAH : AHbI<0b11101, 0b01, 1, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpFMAC16, "vfnma", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fsub_mlx (fneg (fmul_su HPR:$Sn, HPR:$Sm)),
+ HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFusedMAC]>,
Sched<[WriteFPMAC32, ReadFPMAC, ReadFPMUL, ReadFPMUL]>;
@@ -2091,9 +2142,9 @@
}
def VFNMSH : AHbI<0b11101, 0b01, 0, 0,
- (outs SPR:$Sd), (ins SPR:$Sdin, SPR:$Sn, SPR:$Sm),
+ (outs HPR:$Sd), (ins HPR:$Sdin, HPR:$Sn, HPR:$Sm),
IIC_fpFMAC16, "vfnms", ".f16\t$Sd, $Sn, $Sm",
- []>,
+ [(set HPR:$Sd, (fsub_mlx (fmul_su HPR:$Sn, HPR:$Sm), HPR:$Sdin))]>,
RegConstraint<"$Sdin = $Sd">,
Requires<[HasFullFP16,UseFusedMAC]>,
Sched<[WriteFPMAC32, ReadFPMAC, ReadFPMUL, ReadFPMUL]>;