[AVX-512][InstCombine] Add masked scalar FMA intrinsics to SimplifyDemandedVectorElts.
llvm-svn: 289759
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 1b2afe9..cb741fb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1783,6 +1783,16 @@
case Intrinsic::x86_avx512_mask_sub_sd_round:
case Intrinsic::x86_avx512_mask_max_sd_round:
case Intrinsic::x86_avx512_mask_min_sd_round:
+ case Intrinsic::x86_avx512_mask_vfmadd_ss:
+ case Intrinsic::x86_avx512_mask_vfmadd_sd:
+ case Intrinsic::x86_avx512_maskz_vfmadd_ss:
+ case Intrinsic::x86_avx512_maskz_vfmadd_sd:
+ case Intrinsic::x86_avx512_mask3_vfmadd_ss:
+ case Intrinsic::x86_avx512_mask3_vfmadd_sd:
+ case Intrinsic::x86_avx512_mask3_vfmsub_ss:
+ case Intrinsic::x86_avx512_mask3_vfmsub_sd:
+ case Intrinsic::x86_avx512_mask3_vfnmsub_ss:
+ case Intrinsic::x86_avx512_mask3_vfnmsub_sd:
case Intrinsic::x86_fma_vfmadd_ss:
case Intrinsic::x86_fma_vfmsub_ss:
case Intrinsic::x86_fma_vfnmadd_ss:
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index e9d83b3..745f42b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1372,6 +1372,10 @@
case Intrinsic::x86_fma_vfmsub_sd:
case Intrinsic::x86_fma_vfnmadd_sd:
case Intrinsic::x86_fma_vfnmsub_sd:
+ case Intrinsic::x86_avx512_mask_vfmadd_ss:
+ case Intrinsic::x86_avx512_mask_vfmadd_sd:
+ case Intrinsic::x86_avx512_maskz_vfmadd_ss:
+ case Intrinsic::x86_avx512_maskz_vfmadd_sd:
TmpV = SimplifyDemandedVectorElts(II->getArgOperand(0), DemandedElts,
UndefElts, Depth + 1);
if (TmpV) { II->setArgOperand(0, TmpV); MadeChange = true; }
@@ -1396,6 +1400,37 @@
break;
+ case Intrinsic::x86_avx512_mask3_vfmadd_ss:
+ case Intrinsic::x86_avx512_mask3_vfmadd_sd:
+ case Intrinsic::x86_avx512_mask3_vfmsub_ss:
+ case Intrinsic::x86_avx512_mask3_vfmsub_sd:
+ case Intrinsic::x86_avx512_mask3_vfnmsub_ss:
+ case Intrinsic::x86_avx512_mask3_vfnmsub_sd:
+ // These intrinsics get the passthru bits from operand 2.
+ TmpV = SimplifyDemandedVectorElts(II->getArgOperand(2), DemandedElts,
+ UndefElts, Depth + 1);
+ if (TmpV) { II->setArgOperand(2, TmpV); MadeChange = true; }
+
+ // If lowest element of a scalar op isn't used then use Arg2.
+ if (!DemandedElts[0])
+ return II->getArgOperand(2);
+
+ // Only lower element is used for operand 0 and 1.
+ DemandedElts = 1;
+ TmpV = SimplifyDemandedVectorElts(II->getArgOperand(0), DemandedElts,
+ UndefElts2, Depth + 1);
+ if (TmpV) { II->setArgOperand(0, TmpV); MadeChange = true; }
+ TmpV = SimplifyDemandedVectorElts(II->getArgOperand(1), DemandedElts,
+ UndefElts3, Depth + 1);
+ if (TmpV) { II->setArgOperand(1, TmpV); MadeChange = true; }
+
+ // Lower element is undefined if all three lower elements are undefined.
+ // Consider things like undef&0. The result is known zero, not undef.
+ if (!UndefElts2[0] || !UndefElts3[0])
+ UndefElts.clearBit(0);
+
+ break;
+
// SSE4A instructions leave the upper 64-bits of the 128-bit result
// in an undefined state.
case Intrinsic::x86_sse4a_extrq:
diff --git a/llvm/test/Transforms/InstCombine/x86-avx512.ll b/llvm/test/Transforms/InstCombine/x86-avx512.ll
index 7fb7a8f..75955d9 100644
--- a/llvm/test/Transforms/InstCombine/x86-avx512.ll
+++ b/llvm/test/Transforms/InstCombine/x86-avx512.ll
@@ -659,3 +659,393 @@
declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32)
declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32)
declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32)
+
+declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mask_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %res = tail call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %a, <4 x float> %3, <4 x float> %6, i8 %mask, i32 4)
+ ret <4 x float> %res
+}
+
+define float @test_mask_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[TMP2]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %3, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 0
+ ret float %5
+}
+
+define float @test_mask_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %3, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 1
+ ret float %5
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mask_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+ %res = tail call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %a, <2 x double> %1, <2 x double> %2, i8 %mask, i32 4)
+ ret <2 x double> %res
+}
+
+define double @test_mask_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[TMP2]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %1, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_mask_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %1, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 1
+ ret double %3
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_maskz_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %res = tail call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %a, <4 x float> %3, <4 x float> %6, i8 %mask, i32 4)
+ ret <4 x float> %res
+}
+
+define float @test_maskz_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[TMP2]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %3, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 0
+ ret float %5
+}
+
+define float @test_maskz_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %3, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 1
+ ret float %5
+}
+
+declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_maskz_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+ %res = tail call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %a, <2 x double> %1, <2 x double> %2, i8 %mask, i32 4)
+ ret <2 x double> %res
+}
+
+define double @test_maskz_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[TMP2]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %1, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_maskz_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double> %1, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 1
+ ret double %3
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mask3_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %res = tail call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %3, <4 x float> %6, <4 x float> %c, i8 %mask, i32 4)
+ ret <4 x float> %res
+}
+
+define float @test_mask3_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[TMP2]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 0
+ ret float %5
+}
+
+define float @test_mask3_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 1
+ ret float %5
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mask3_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %res = tail call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %1, <2 x double> %2, <2 x double> %c, i8 %mask, i32 4)
+ ret <2 x double> %res
+}
+
+define double @test_mask3_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[TMP2]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_mask3_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 1
+ ret double %3
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mask3_vfmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %res = tail call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %3, <4 x float> %6, <4 x float> %c, i8 %mask, i32 4)
+ ret <4 x float> %res
+}
+
+define float @test_mask3_vfmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[TMP2]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 0
+ ret float %5
+}
+
+define float @test_mask3_vfmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 1
+ ret float %5
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mask3_vfmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %res = tail call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %1, <2 x double> %2, <2 x double> %c, i8 %mask, i32 4)
+ ret <2 x double> %res
+}
+
+define double @test_mask3_vfmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[TMP2]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_mask3_vfmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 1
+ ret double %3
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %res = tail call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %3, <4 x float> %6, <4 x float> %c, i8 %mask, i32 4)
+ ret <4 x float> %res
+}
+
+define float @test_mask3_vfnmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[TMP2]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 0
+ ret float %5
+}
+
+define float @test_mask3_vfnmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ %5 = extractelement <4 x float> %4, i32 1
+ ret float %5
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mask3_vfnmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %res = tail call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %1, <2 x double> %2, <2 x double> %c, i8 %mask, i32 4)
+ ret <2 x double> %res
+}
+
+define double @test_mask3_vfnmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 4)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[TMP2]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_mask3_vfnmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ %3 = extractelement <2 x double> %2, i32 1
+ ret double %3
+}