[ARM] Parallel DSP Pass
Armv6 introduced instructions to perform 32-bit SIMD operations. The purpose of
this pass is to do some straightforward IR pattern matching to create ACLE DSP
intrinsics, which map on these 32-bit SIMD operations.
Currently, only the SMLAD instruction gets recognised. This instruction
performs two multiplications with 16-bit operands, and stores the result in an
accumulator. We will follow this up with patches to recognise SMLAD in more
cases, and also to generate other DSP instructions (like e.g. SADD16).
Patch by: Sam Parker and Sjoerd Meijer
Differential Revision: https://reviews.llvm.org/D48128
llvm-svn: 335850
diff --git a/llvm/test/CodeGen/ARM/smlad0.ll b/llvm/test/CodeGen/ARM/smlad0.ll
new file mode 100644
index 0000000..565e4fa
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad0.ll
@@ -0,0 +1,57 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; The Cortex-M0 does not support unaligned accesses:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+;
+; Check DSP extension:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+;
+; CHECK: %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK: [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK: [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK: [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK: [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK: [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
+;
+; CHECK-UNSUPPORTED-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+ %add11 = add i32 %mul9, %add10
+
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
diff --git a/llvm/test/CodeGen/ARM/smlad1.ll b/llvm/test/CodeGen/ARM/smlad1.ll
new file mode 100644
index 0000000..dc3fa8b
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad1.ll
@@ -0,0 +1,50 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+
+; CHECK: %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK: [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK: [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK: [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK: [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK: [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
+
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+
+; And here the Add is the LHS, the Mul the RHS
+ %add11 = add i32 %add10, %mul9
+
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad10.ll b/llvm/test/CodeGen/ARM/smlad10.ll
new file mode 100644
index 0000000..7c28eb2
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad10.ll
@@ -0,0 +1,47 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Reduction statement is an i64 type: we only support i32 so check that the
+; rewrite isn't triggered.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i64 @test(i64 %arg, i64* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i64 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+ ret i64 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i64 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i64 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i64 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i64 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i64
+ %conv4 = sext i16 %0 to i64
+ %mul = mul nsw i64 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i64 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i64
+ %conv8 = sext i16 %1 to i64
+ %mul9 = mul nsw i64 %conv7, %conv8
+ %add10 = add i64 %mul, %mac1.026
+
+ %add11 = add i64 %mul9, %add10
+
+ %exitcond = icmp ne i64 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad11.ll b/llvm/test/CodeGen/ARM/smlad11.ll
new file mode 100644
index 0000000..a503818
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad11.ll
@@ -0,0 +1,74 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; A more complicated chain: 4 mul operations, so we expect 2 smlad calls.
+;
+; CHECK: %mac1{{\.}}054 = phi i32 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK: [[V8:%[0-9]+]] = bitcast i16* %arrayidx8 to i32*
+; CHECK: [[V9:%[0-9]+]] = load i32, i32* [[V8]], align 2
+; CHECK: [[V10:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK: [[V11:%[0-9]+]] = load i32, i32* [[V10]], align 2
+; CHECK: [[V12:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V9]], i32 [[V11]], i32 %mac1{{\.}}054)
+; CHECK: [[V13:%[0-9]+]] = bitcast i16* %arrayidx17 to i32*
+; CHECK: [[V14:%[0-9]+]] = load i32, i32* [[V13]], align 2
+; CHECK: [[V15:%[0-9]+]] = bitcast i16* %arrayidx4 to i32*
+; CHECK: [[V16:%[0-9]+]] = load i32, i32* [[V15]], align 2
+; CHECK: [[V17:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V14]], i32 [[V16]], i32 [[V12]])
+;
+; And we don't want to see a 3rd smlad:
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp52 = icmp sgt i32 %arg, 0
+ br i1 %cmp52, label %for.body.preheader, label %for.cond.cleanup
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add28, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body.preheader:
+ br label %for.body
+
+for.body:
+ %mac1.054 = phi i32 [ %add28, %for.body ], [ 0, %for.body.preheader ]
+ %i.053 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.053
+ %0 = load i16, i16* %arrayidx, align 2
+ %add1 = or i32 %i.053, 1
+ %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
+ %1 = load i16, i16* %arrayidx2, align 2
+ %add3 = or i32 %i.053, 2
+ %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
+ %2 = load i16, i16* %arrayidx4, align 2
+ %add5 = or i32 %i.053, 3
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
+ %3 = load i16, i16* %arrayidx6, align 2
+ %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.053
+ %4 = load i16, i16* %arrayidx8, align 2
+ %conv = sext i16 %4 to i32
+ %conv9 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv9
+ %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
+ %5 = load i16, i16* %arrayidx11, align 2
+ %conv12 = sext i16 %5 to i32
+ %conv13 = sext i16 %1 to i32
+ %mul14 = mul nsw i32 %conv12, %conv13
+ %arrayidx17 = getelementptr inbounds i16, i16* %arg2, i32 %add3
+ %6 = load i16, i16* %arrayidx17, align 2
+ %conv18 = sext i16 %6 to i32
+ %conv19 = sext i16 %2 to i32
+ %mul20 = mul nsw i32 %conv18, %conv19
+ %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+ %7 = load i16, i16* %arrayidx23, align 2
+ %conv24 = sext i16 %7 to i32
+ %conv25 = sext i16 %3 to i32
+ %mul26 = mul nsw i32 %conv24, %conv25
+ %add15 = add i32 %mul, %mac1.054
+ %add21 = add i32 %add15, %mul14
+ %add27 = add i32 %add21, %mul20
+ %add28 = add i32 %add27, %mul26
+ %add29 = add nuw nsw i32 %i.053, 4
+ %cmp = icmp slt i32 %add29, %arg
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
diff --git a/llvm/test/CodeGen/ARM/smlad12.ll b/llvm/test/CodeGen/ARM/smlad12.ll
new file mode 100644
index 0000000..391ce5c
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad12.ll
@@ -0,0 +1,48 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; The loop header is not the loop latch.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+; This is the loop header:
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body2 ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body2 ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %mul9, %add10
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body2, label %for.cond.cleanup
+
+; And this is the loop latch:
+for.body2:
+ br label %for.body
+}
diff --git a/llvm/test/CodeGen/ARM/smlad2.ll b/llvm/test/CodeGen/ARM/smlad2.ll
new file mode 100644
index 0000000..219b55c
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad2.ll
@@ -0,0 +1,52 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Operands of both muls are not symmetrical (see also comments inlined below), check
+; that the rewrite isn't triggered.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+
+; This zero-extends the 2nd operand of %mul:
+ %conv4 = zext i16 %0 to i32
+
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+
+; And here we only have sign-extensions. Thus, the operands of
+; %mul and %mul9 are not symmetrical:
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %add10, %mul9
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad3.ll b/llvm/test/CodeGen/ARM/smlad3.ll
new file mode 100644
index 0000000..5584c83
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad3.ll
@@ -0,0 +1,50 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; The loads are not consecutive: check that the rewrite isn't triggered.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+
+; Here we add another constants offset of 2, to make sure the
+; loads to %3 and %2 are not consecutive:
+
+ %add5 = add nuw nsw i32 %i.025, 2
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+ %3 = load i16, i16* %arrayidx6, align 2
+
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %add10, %mul9
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad4.ll b/llvm/test/CodeGen/ARM/smlad4.ll
new file mode 100644
index 0000000..defce8a
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad4.ll
@@ -0,0 +1,48 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; The loads are not narrow loads: check that the rewrite isn't triggered.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+; Arg2 is now an i32, while Arg3 is still and i16:
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i32* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp22 = icmp sgt i32 %arg, 0
+ br i1 %cmp22, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add9, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %0 = phi i16 [ %1, %for.body ], [ %.pre, %for.body.preheader ]
+ %mac1.024 = phi i32 [ %add9, %for.body ], [ 0, %for.body.preheader ]
+ %i.023 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %add = add nuw nsw i32 %i.023, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %conv = sext i16 %0 to i32
+
+; This is a 'normal' i32 load to %2:
+ %arrayidx3 = getelementptr inbounds i32, i32* %arg2, i32 %i.023
+ %2 = load i32, i32* %arrayidx3, align 4
+
+; This mul has now 1 operand which is a narrow load, and the other a normal
+; i32 load:
+ %mul = mul nsw i32 %2, %conv
+
+ %add4 = add nuw nsw i32 %i.023, 2
+ %arrayidx5 = getelementptr inbounds i32, i32* %arg2, i32 %add4
+ %3 = load i32, i32* %arrayidx5, align 4
+ %conv6 = sext i16 %1 to i32
+ %mul7 = mul nsw i32 %3, %conv6
+ %add8 = add i32 %mul, %mac1.024
+ %add9 = add i32 %add8, %mul7
+ %exitcond = icmp eq i32 %add, %arg
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/ARM/smlad5.ll b/llvm/test/CodeGen/ARM/smlad5.ll
new file mode 100644
index 0000000..f1e5e72
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad5.ll
@@ -0,0 +1,44 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; The loads are volatile loads: check that the rewrite isn't triggered.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load volatile i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load volatile i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load volatile i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load volatile i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %add10, %mul9
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad6.ll b/llvm/test/CodeGen/ARM/smlad6.ll
new file mode 100644
index 0000000..831cbfd
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad6.ll
@@ -0,0 +1,50 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Alias check: check that the rewrite isn't triggered when there's a store
+; instruction possibly aliasing any mul load operands; arguments are passed
+; without 'restrict' enabled.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+
+; Store inserted here, aliasing with arrayidx, arrayidx1, arrayidx3
+ store i16 42, i16* %arrayidx, align 2
+
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %mul9, %add10
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad7.ll b/llvm/test/CodeGen/ARM/smlad7.ll
new file mode 100644
index 0000000..b8e54e6
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad7.ll
@@ -0,0 +1,53 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Alias check: check that the rewrite isn't triggered when there's a store
+; aliasing one of the mul load operands. Arguments are now annotated with
+; 'noalias'.
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* noalias %arg1, i16* noalias readonly %arg2, i16* noalias readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+
+; Store inserted here, aliasing only with loads from 'arrayidx'.
+ store i16 42, i16* %arrayidx, align 2
+
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %mul = mul nsw i32 %conv, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %mul9 = mul nsw i32 %conv7, %conv8
+ %add10 = add i32 %mul, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+ %add11 = add i32 %mul9, %add10
+
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/ARM/smlad8.ll b/llvm/test/CodeGen/ARM/smlad8.ll
new file mode 100644
index 0000000..0fd75e6
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad8.ll
@@ -0,0 +1,59 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Mul with operands that are not simple load and sext/zext chains: this is not
+; yet supported so the rewrite shouldn't trigger (but we do want to support this
+; soon).
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3, i16* %arg4) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ %gep0 = getelementptr inbounds i16, i16* %arg4, i32 0
+ %gep1 = getelementptr inbounds i16, i16* %arg4, i32 1
+ %.add4 = load i16, i16* %gep0, align 2
+ %.add5 = load i16, i16* %gep1, align 2
+ %.zext4 = zext i16 %.add4 to i32
+ %.zext5 = zext i16 %.add5 to i32
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+ %0 = load i16, i16* %arrayidx, align 2
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+ %1 = load i16, i16* %arrayidx1, align 2
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %2 to i32
+ %conv4 = sext i16 %0 to i32
+ %add1 = add i32 %conv, %.zext4
+
+; This mul has a more complicated pattern as an operand, %add1
+; is another add and load, which we don't support for now.
+ %mul = mul nsw i32 %add1, %conv4
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %3 to i32
+ %conv8 = sext i16 %1 to i32
+ %add2 = add i32 %conv7, %.zext5
+
+; Same here
+ %mul9 = mul nsw i32 %add2, %conv8
+ %add10 = add i32 %mul, %mac1.026
+
+ %add11 = add i32 %mul9, %add10
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
diff --git a/llvm/test/CodeGen/ARM/smlad9.ll b/llvm/test/CodeGen/ARM/smlad9.ll
new file mode 100644
index 0000000..7ae5965
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/smlad9.ll
@@ -0,0 +1,45 @@
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -parallel-dsp -S | FileCheck %s
+;
+; Muls with operands that are constants: not yet supported, so the rewrite
+; should not trigger (but we do want to add this soon).
+;
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+ %cmp24 = icmp sgt i32 %arg, 0
+ br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+ %.pre = load i16, i16* %arg3, align 2
+ %.pre27 = load i16, i16* %arg2, align 2
+ br label %for.body
+
+for.cond.cleanup:
+ %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+ ret i32 %mac1.0.lcssa
+
+for.body:
+ %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+ %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+ %add = add nuw nsw i32 %i.025, 1
+ %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+ %v2 = load i16, i16* %arrayidx3, align 2
+ %conv = sext i16 %v2 to i32
+
+; RHS operand of this mul is a constant
+ %mul = mul nsw i32 %conv, 43
+
+ %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+ %v3 = load i16, i16* %arrayidx6, align 2
+ %conv7 = sext i16 %v3 to i32
+
+; And this RHS operand is a constant too.
+ %mul9 = mul nsw i32 %conv7, 42
+
+ %add10 = add i32 %mul, %mac1.026
+ %add11 = add i32 %mul9, %add10
+ %exitcond = icmp ne i32 %add, %arg
+ br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+