Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 322d0afd875df66b36e4810a2b95c20a8f22ab9b / . / llvm / test / CodeGen / Thumb2 / mve-vecreduce-loops.ll
blob: 2544474932c931b6fd0a4b764506b1ec8b62c1bd [file] [log] [blame]
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -tail-predication=enabled -verify-machineinstrs %s -o - | FileCheck %s
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]3
4define i32 @add_i32(i32* nocapture readonly %x, i32 %n) {
5; CHECK-LABEL: add_i32:
6; CHECK: @ %bb.0: @ %entry
7; CHECK-NEXT: .save {r7, lr}
8; CHECK-NEXT: push {r7, lr}
9; CHECK-NEXT: cmp r1, #1
10; CHECK-NEXT: blt .LBB0_3
11; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
12; CHECK-NEXT: mov r12, r0
13; CHECK-NEXT: cmp r1, #4
14; CHECK-NEXT: bhs .LBB0_4
15; CHECK-NEXT: @ %bb.2:
16; CHECK-NEXT: movs r3, #0
17; CHECK-NEXT: movs r0, #0
18; CHECK-NEXT: b .LBB0_7
19; CHECK-NEXT: .LBB0_3:
20; CHECK-NEXT: movs r0, #0
21; CHECK-NEXT: b .LBB0_9
22; CHECK-NEXT: .LBB0_4: @ %vector.ph
23; CHECK-NEXT: bic r3, r1, #3
24; CHECK-NEXT: movs r2, #1
25; CHECK-NEXT: subs r0, r3, #4
26; CHECK-NEXT: add.w lr, r2, r0, lsr #2
27; CHECK-NEXT: movs r0, #0
28; CHECK-NEXT: mov r2, r12
29; CHECK-NEXT: dls lr, lr
30; CHECK-NEXT: .LBB0_5: @ %vector.body
31; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
32; CHECK-NEXT: vldrw.u32 q0, [r2], #16
33; CHECK-NEXT: vaddva.u32 r0, q0
34; CHECK-NEXT: le lr, .LBB0_5
35; CHECK-NEXT: @ %bb.6: @ %middle.block
36; CHECK-NEXT: cmp r3, r1
37; CHECK-NEXT: it eq
38; CHECK-NEXT: popeq {r7, pc}
39; CHECK-NEXT: .LBB0_7: @ %for.body.preheader1
40; CHECK-NEXT: sub.w lr, r1, r3
41; CHECK-NEXT: add.w r1, r12, r3, lsl #2
42; CHECK-NEXT: dls lr, lr
43; CHECK-NEXT: .LBB0_8: @ %for.body
44; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
45; CHECK-NEXT: ldr r2, [r1], #4
46; CHECK-NEXT: add r0, r2
47; CHECK-NEXT: le lr, .LBB0_8
48; CHECK-NEXT: .LBB0_9: @ %for.cond.cleanup
49; CHECK-NEXT: pop {r7, pc}
50entry:
51 %cmp6 = icmp sgt i32 %n, 0
52 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
53
54for.body.preheader: ; preds = %entry
55 %min.iters.check = icmp ult i32 %n, 4
56 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
57
58vector.ph: ; preds = %for.body.preheader
59 %n.vec = and i32 %n, -4
60 br label %vector.body
61
62vector.body: ; preds = %vector.body, %vector.ph
63 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
64 %vec.phi = phi i32 [ 0, %vector.ph ], [ %3, %vector.body ]
65 %0 = getelementptr inbounds i32, i32* %x, i32 %index
66 %1 = bitcast i32* %0 to <4 x i32>*
67 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]68 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]69 %3 = add i32 %2, %vec.phi
70 %index.next = add i32 %index, 4
71 %4 = icmp eq i32 %index.next, %n.vec
72 br i1 %4, label %middle.block, label %vector.body
73
74middle.block: ; preds = %vector.body
75 %cmp.n = icmp eq i32 %n.vec, %n
76 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
77
78for.body.preheader1: ; preds = %middle.block, %for.body.preheader
79 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
80 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %3, %middle.block ]
81 br label %for.body
82
83for.body: ; preds = %for.body.preheader1, %for.body
84 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
85 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
86 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
87 %5 = load i32, i32* %arrayidx, align 4
88 %add = add nsw i32 %5, %r.07
89 %inc = add nuw nsw i32 %i.08, 1
90 %exitcond = icmp eq i32 %inc, %n
91 br i1 %exitcond, label %for.cond.cleanup, label %for.body
92
93for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
94 %r.0.lcssa = phi i32 [ 0, %entry ], [ %3, %middle.block ], [ %add, %for.body ]
95 ret i32 %r.0.lcssa
96}
97
98define i32 @mul_i32(i32* nocapture readonly %x, i32 %n) {
99; CHECK-LABEL: mul_i32:
100; CHECK: @ %bb.0: @ %entry
101; CHECK-NEXT: .save {r7, lr}
102; CHECK-NEXT: push {r7, lr}
103; CHECK-NEXT: movs r2, #1
104; CHECK-NEXT: cmp r1, #1
105; CHECK-NEXT: blt .LBB1_8
106; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
107; CHECK-NEXT: cmp r1, #4
108; CHECK-NEXT: bhs .LBB1_3
109; CHECK-NEXT: @ %bb.2:
110; CHECK-NEXT: mov.w r12, #0
111; CHECK-NEXT: b .LBB1_6
112; CHECK-NEXT: .LBB1_3: @ %vector.ph
113; CHECK-NEXT: bic r12, r1, #3
114; CHECK-NEXT: vmov.i32 q0, #0x1
115; CHECK-NEXT: sub.w r3, r12, #4
116; CHECK-NEXT: add.w lr, r2, r3, lsr #2
117; CHECK-NEXT: mov r2, r0
118; CHECK-NEXT: dls lr, lr
119; CHECK-NEXT: .LBB1_4: @ %vector.body
120; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
121; CHECK-NEXT: vldrw.u32 q1, [r2], #16
122; CHECK-NEXT: vmul.i32 q0, q1, q0
123; CHECK-NEXT: le lr, .LBB1_4
124; CHECK-NEXT: @ %bb.5: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]125; CHECK-NEXT: vmov r2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]126; CHECK-NEXT: cmp r12, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]127; CHECK-NEXT: vmov r3, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]128; CHECK-NEXT: mul lr, r3, r2
129; CHECK-NEXT: vmov r3, s1
130; CHECK-NEXT: vmov r2, s0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]131; CHECK-NEXT: mul r2, r3, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]132; CHECK-NEXT: mul r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]133; CHECK-NEXT: beq .LBB1_8
134; CHECK-NEXT: .LBB1_6: @ %for.body.preheader1
135; CHECK-NEXT: sub.w lr, r1, r12
136; CHECK-NEXT: add.w r0, r0, r12, lsl #2
137; CHECK-NEXT: dls lr, lr
138; CHECK-NEXT: .LBB1_7: @ %for.body
139; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
140; CHECK-NEXT: ldr r1, [r0], #4
141; CHECK-NEXT: muls r2, r1, r2
142; CHECK-NEXT: le lr, .LBB1_7
143; CHECK-NEXT: .LBB1_8: @ %for.cond.cleanup
144; CHECK-NEXT: mov r0, r2
145; CHECK-NEXT: pop {r7, pc}
146entry:
147 %cmp6 = icmp sgt i32 %n, 0
148 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
149
150for.body.preheader: ; preds = %entry
151 %min.iters.check = icmp ult i32 %n, 4
152 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
153
154vector.ph: ; preds = %for.body.preheader
155 %n.vec = and i32 %n, -4
156 br label %vector.body
157
158vector.body: ; preds = %vector.body, %vector.ph
159 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
160 %vec.phi = phi <4 x i32> [ <i32 1, i32 1, i32 1, i32 1>, %vector.ph ], [ %2, %vector.body ]
161 %0 = getelementptr inbounds i32, i32* %x, i32 %index
162 %1 = bitcast i32* %0 to <4 x i32>*
163 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
164 %2 = mul <4 x i32> %wide.load, %vec.phi
165 %index.next = add i32 %index, 4
166 %3 = icmp eq i32 %index.next, %n.vec
167 br i1 %3, label %middle.block, label %vector.body
168
169middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]170 %4 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]171 %cmp.n = icmp eq i32 %n.vec, %n
172 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
173
174for.body.preheader1: ; preds = %middle.block, %for.body.preheader
175 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
176 %r.07.ph = phi i32 [ 1, %for.body.preheader ], [ %4, %middle.block ]
177 br label %for.body
178
179for.body: ; preds = %for.body.preheader1, %for.body
180 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
181 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
182 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
183 %5 = load i32, i32* %arrayidx, align 4
184 %add = mul nsw i32 %5, %r.07
185 %inc = add nuw nsw i32 %i.08, 1
186 %exitcond = icmp eq i32 %inc, %n
187 br i1 %exitcond, label %for.cond.cleanup, label %for.body
188
189for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
190 %r.0.lcssa = phi i32 [ 1, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
191 ret i32 %r.0.lcssa
192}
193
194define i32 @and_i32(i32* nocapture readonly %x, i32 %n) {
195; CHECK-LABEL: and_i32:
196; CHECK: @ %bb.0: @ %entry
197; CHECK-NEXT: .save {r7, lr}
198; CHECK-NEXT: push {r7, lr}
199; CHECK-NEXT: cmp r1, #1
200; CHECK-NEXT: blt .LBB2_3
201; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
202; CHECK-NEXT: cmp r1, #4
203; CHECK-NEXT: bhs .LBB2_4
204; CHECK-NEXT: @ %bb.2:
205; CHECK-NEXT: mov.w r2, #-1
206; CHECK-NEXT: movs r3, #0
207; CHECK-NEXT: b .LBB2_7
208; CHECK-NEXT: .LBB2_3:
209; CHECK-NEXT: mov.w r2, #-1
210; CHECK-NEXT: b .LBB2_9
211; CHECK-NEXT: .LBB2_4: @ %vector.ph
212; CHECK-NEXT: bic r3, r1, #3
213; CHECK-NEXT: movs r2, #1
214; CHECK-NEXT: sub.w r12, r3, #4
215; CHECK-NEXT: vmov.i8 q0, #0xff
216; CHECK-NEXT: add.w lr, r2, r12, lsr #2
217; CHECK-NEXT: mov r2, r0
218; CHECK-NEXT: dls lr, lr
219; CHECK-NEXT: .LBB2_5: @ %vector.body
220; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
221; CHECK-NEXT: vldrw.u32 q1, [r2], #16
222; CHECK-NEXT: vand q0, q1, q0
223; CHECK-NEXT: le lr, .LBB2_5
224; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]225; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]226; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]227; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]228; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]229; CHECK-NEXT: and.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]230; CHECK-NEXT: vmov r2, s0
231; CHECK-NEXT: and.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]232; CHECK-NEXT: and.w r2, r2, r12
233; CHECK-NEXT: beq .LBB2_9
234; CHECK-NEXT: .LBB2_7: @ %for.body.preheader1
235; CHECK-NEXT: sub.w lr, r1, r3
236; CHECK-NEXT: add.w r0, r0, r3, lsl #2
237; CHECK-NEXT: dls lr, lr
238; CHECK-NEXT: .LBB2_8: @ %for.body
239; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
240; CHECK-NEXT: ldr r1, [r0], #4
241; CHECK-NEXT: ands r2, r1
242; CHECK-NEXT: le lr, .LBB2_8
243; CHECK-NEXT: .LBB2_9: @ %for.cond.cleanup
244; CHECK-NEXT: mov r0, r2
245; CHECK-NEXT: pop {r7, pc}
246entry:
247 %cmp6 = icmp sgt i32 %n, 0
248 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
249
250for.body.preheader: ; preds = %entry
251 %min.iters.check = icmp ult i32 %n, 4
252 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
253
254vector.ph: ; preds = %for.body.preheader
255 %n.vec = and i32 %n, -4
256 br label %vector.body
257
258vector.body: ; preds = %vector.body, %vector.ph
259 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
260 %vec.phi = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %vector.ph ], [ %2, %vector.body ]
261 %0 = getelementptr inbounds i32, i32* %x, i32 %index
262 %1 = bitcast i32* %0 to <4 x i32>*
263 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
264 %2 = and <4 x i32> %wide.load, %vec.phi
265 %index.next = add i32 %index, 4
266 %3 = icmp eq i32 %index.next, %n.vec
267 br i1 %3, label %middle.block, label %vector.body
268
269middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]270 %4 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]271 %cmp.n = icmp eq i32 %n.vec, %n
272 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
273
274for.body.preheader1: ; preds = %middle.block, %for.body.preheader
275 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
276 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %4, %middle.block ]
277 br label %for.body
278
279for.body: ; preds = %for.body.preheader1, %for.body
280 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
281 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
282 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
283 %5 = load i32, i32* %arrayidx, align 4
284 %add = and i32 %5, %r.07
285 %inc = add nuw nsw i32 %i.08, 1
286 %exitcond = icmp eq i32 %inc, %n
287 br i1 %exitcond, label %for.cond.cleanup, label %for.body
288
289for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
290 %r.0.lcssa = phi i32 [ -1, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
291 ret i32 %r.0.lcssa
292}
293
294define i32 @or_i32(i32* nocapture readonly %x, i32 %n) {
295; CHECK-LABEL: or_i32:
296; CHECK: @ %bb.0: @ %entry
297; CHECK-NEXT: .save {r7, lr}
298; CHECK-NEXT: push {r7, lr}
299; CHECK-NEXT: cmp r1, #1
300; CHECK-NEXT: blt .LBB3_3
301; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
302; CHECK-NEXT: cmp r1, #4
303; CHECK-NEXT: bhs .LBB3_4
304; CHECK-NEXT: @ %bb.2:
305; CHECK-NEXT: movs r3, #0
306; CHECK-NEXT: movs r2, #0
307; CHECK-NEXT: b .LBB3_7
308; CHECK-NEXT: .LBB3_3:
309; CHECK-NEXT: movs r2, #0
310; CHECK-NEXT: b .LBB3_9
311; CHECK-NEXT: .LBB3_4: @ %vector.ph
312; CHECK-NEXT: bic r3, r1, #3
313; CHECK-NEXT: movs r2, #1
314; CHECK-NEXT: sub.w r12, r3, #4
315; CHECK-NEXT: vmov.i32 q0, #0x0
316; CHECK-NEXT: add.w lr, r2, r12, lsr #2
317; CHECK-NEXT: mov r2, r0
318; CHECK-NEXT: dls lr, lr
319; CHECK-NEXT: .LBB3_5: @ %vector.body
320; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
321; CHECK-NEXT: vldrw.u32 q1, [r2], #16
322; CHECK-NEXT: vorr q0, q1, q0
323; CHECK-NEXT: le lr, .LBB3_5
324; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]325; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]326; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]327; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]328; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]329; CHECK-NEXT: orr.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]330; CHECK-NEXT: vmov r2, s0
331; CHECK-NEXT: orr.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]332; CHECK-NEXT: orr.w r2, r2, r12
333; CHECK-NEXT: beq .LBB3_9
334; CHECK-NEXT: .LBB3_7: @ %for.body.preheader1
335; CHECK-NEXT: sub.w lr, r1, r3
336; CHECK-NEXT: add.w r0, r0, r3, lsl #2
337; CHECK-NEXT: dls lr, lr
338; CHECK-NEXT: .LBB3_8: @ %for.body
339; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
340; CHECK-NEXT: ldr r1, [r0], #4
341; CHECK-NEXT: orrs r2, r1
342; CHECK-NEXT: le lr, .LBB3_8
343; CHECK-NEXT: .LBB3_9: @ %for.cond.cleanup
344; CHECK-NEXT: mov r0, r2
345; CHECK-NEXT: pop {r7, pc}
346entry:
347 %cmp6 = icmp sgt i32 %n, 0
348 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
349
350for.body.preheader: ; preds = %entry
351 %min.iters.check = icmp ult i32 %n, 4
352 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
353
354vector.ph: ; preds = %for.body.preheader
355 %n.vec = and i32 %n, -4
356 br label %vector.body
357
358vector.body: ; preds = %vector.body, %vector.ph
359 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
360 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
361 %0 = getelementptr inbounds i32, i32* %x, i32 %index
362 %1 = bitcast i32* %0 to <4 x i32>*
363 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
364 %2 = or <4 x i32> %wide.load, %vec.phi
365 %index.next = add i32 %index, 4
366 %3 = icmp eq i32 %index.next, %n.vec
367 br i1 %3, label %middle.block, label %vector.body
368
369middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]370 %4 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]371 %cmp.n = icmp eq i32 %n.vec, %n
372 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
373
374for.body.preheader1: ; preds = %middle.block, %for.body.preheader
375 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
376 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %4, %middle.block ]
377 br label %for.body
378
379for.body: ; preds = %for.body.preheader1, %for.body
380 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
381 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
382 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
383 %5 = load i32, i32* %arrayidx, align 4
384 %add = or i32 %5, %r.07
385 %inc = add nuw nsw i32 %i.08, 1
386 %exitcond = icmp eq i32 %inc, %n
387 br i1 %exitcond, label %for.cond.cleanup, label %for.body
388
389for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
390 %r.0.lcssa = phi i32 [ 0, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
391 ret i32 %r.0.lcssa
392}
393
394define i32 @xor_i32(i32* nocapture readonly %x, i32 %n) {
395; CHECK-LABEL: xor_i32:
396; CHECK: @ %bb.0: @ %entry
397; CHECK-NEXT: .save {r7, lr}
398; CHECK-NEXT: push {r7, lr}
399; CHECK-NEXT: cmp r1, #1
400; CHECK-NEXT: blt .LBB4_3
401; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
402; CHECK-NEXT: cmp r1, #4
403; CHECK-NEXT: bhs .LBB4_4
404; CHECK-NEXT: @ %bb.2:
405; CHECK-NEXT: movs r3, #0
406; CHECK-NEXT: movs r2, #0
407; CHECK-NEXT: b .LBB4_7
408; CHECK-NEXT: .LBB4_3:
409; CHECK-NEXT: movs r2, #0
410; CHECK-NEXT: b .LBB4_9
411; CHECK-NEXT: .LBB4_4: @ %vector.ph
412; CHECK-NEXT: bic r3, r1, #3
413; CHECK-NEXT: movs r2, #1
414; CHECK-NEXT: sub.w r12, r3, #4
415; CHECK-NEXT: vmov.i32 q0, #0x0
416; CHECK-NEXT: add.w lr, r2, r12, lsr #2
417; CHECK-NEXT: mov r2, r0
418; CHECK-NEXT: dls lr, lr
419; CHECK-NEXT: .LBB4_5: @ %vector.body
420; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
421; CHECK-NEXT: vldrw.u32 q1, [r2], #16
422; CHECK-NEXT: veor q0, q1, q0
423; CHECK-NEXT: le lr, .LBB4_5
424; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]425; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]426; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]427; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]428; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]429; CHECK-NEXT: eor.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]430; CHECK-NEXT: vmov r2, s0
431; CHECK-NEXT: eor.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]432; CHECK-NEXT: eor.w r2, r2, r12
433; CHECK-NEXT: beq .LBB4_9
434; CHECK-NEXT: .LBB4_7: @ %for.body.preheader1
435; CHECK-NEXT: sub.w lr, r1, r3
436; CHECK-NEXT: add.w r0, r0, r3, lsl #2
437; CHECK-NEXT: dls lr, lr
438; CHECK-NEXT: .LBB4_8: @ %for.body
439; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
440; CHECK-NEXT: ldr r1, [r0], #4
441; CHECK-NEXT: eors r2, r1
442; CHECK-NEXT: le lr, .LBB4_8
443; CHECK-NEXT: .LBB4_9: @ %for.cond.cleanup
444; CHECK-NEXT: mov r0, r2
445; CHECK-NEXT: pop {r7, pc}
446entry:
447 %cmp6 = icmp sgt i32 %n, 0
448 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
449
450for.body.preheader: ; preds = %entry
451 %min.iters.check = icmp ult i32 %n, 4
452 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
453
454vector.ph: ; preds = %for.body.preheader
455 %n.vec = and i32 %n, -4
456 br label %vector.body
457
458vector.body: ; preds = %vector.body, %vector.ph
459 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
460 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
461 %0 = getelementptr inbounds i32, i32* %x, i32 %index
462 %1 = bitcast i32* %0 to <4 x i32>*
463 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
464 %2 = xor <4 x i32> %wide.load, %vec.phi
465 %index.next = add i32 %index, 4
466 %3 = icmp eq i32 %index.next, %n.vec
467 br i1 %3, label %middle.block, label %vector.body
468
469middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]470 %4 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]471 %cmp.n = icmp eq i32 %n.vec, %n
472 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
473
474for.body.preheader1: ; preds = %middle.block, %for.body.preheader
475 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
476 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %4, %middle.block ]
477 br label %for.body
478
479for.body: ; preds = %for.body.preheader1, %for.body
480 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
481 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
482 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
483 %5 = load i32, i32* %arrayidx, align 4
484 %add = xor i32 %5, %r.07
485 %inc = add nuw nsw i32 %i.08, 1
486 %exitcond = icmp eq i32 %inc, %n
487 br i1 %exitcond, label %for.cond.cleanup, label %for.body
488
489for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
490 %r.0.lcssa = phi i32 [ 0, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
491 ret i32 %r.0.lcssa
492}
493
494define float @fadd_f32(float* nocapture readonly %x, i32 %n) {
495; CHECK-LABEL: fadd_f32:
496; CHECK: @ %bb.0: @ %entry
497; CHECK-NEXT: .save {r7, lr}
498; CHECK-NEXT: push {r7, lr}
499; CHECK-NEXT: cmp r1, #1
500; CHECK-NEXT: blt .LBB5_3
501; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
502; CHECK-NEXT: cmp r1, #4
503; CHECK-NEXT: bhs .LBB5_4
504; CHECK-NEXT: @ %bb.2:
505; CHECK-NEXT: vldr s0, .LCPI5_0
506; CHECK-NEXT: movs r2, #0
507; CHECK-NEXT: b .LBB5_7
508; CHECK-NEXT: .LBB5_3:
509; CHECK-NEXT: vldr s0, .LCPI5_0
510; CHECK-NEXT: b .LBB5_9
511; CHECK-NEXT: .LBB5_4: @ %vector.ph
512; CHECK-NEXT: bic r2, r1, #3
513; CHECK-NEXT: movs r3, #1
514; CHECK-NEXT: sub.w r12, r2, #4
515; CHECK-NEXT: vmov.i32 q0, #0x0
516; CHECK-NEXT: add.w lr, r3, r12, lsr #2
517; CHECK-NEXT: mov r3, r0
518; CHECK-NEXT: dls lr, lr
519; CHECK-NEXT: .LBB5_5: @ %vector.body
520; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
521; CHECK-NEXT: vldrw.u32 q1, [r3], #16
522; CHECK-NEXT: vadd.f32 q0, q1, q0
523; CHECK-NEXT: le lr, .LBB5_5
524; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]525; CHECK-NEXT: vadd.f32 s4, s2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]526; CHECK-NEXT: cmp r2, r1
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]527; CHECK-NEXT: vadd.f32 s0, s0, s1
528; CHECK-NEXT: vadd.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]529; CHECK-NEXT: beq .LBB5_9
530; CHECK-NEXT: .LBB5_7: @ %for.body.preheader1
531; CHECK-NEXT: sub.w lr, r1, r2
532; CHECK-NEXT: add.w r0, r0, r2, lsl #2
533; CHECK-NEXT: dls lr, lr
534; CHECK-NEXT: .LBB5_8: @ %for.body
535; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
536; CHECK-NEXT: vldr s2, [r0]
537; CHECK-NEXT: adds r0, #4
538; CHECK-NEXT: vadd.f32 s0, s2, s0
539; CHECK-NEXT: le lr, .LBB5_8
540; CHECK-NEXT: .LBB5_9: @ %for.cond.cleanup
541; CHECK-NEXT: vmov r0, s0
542; CHECK-NEXT: pop {r7, pc}
543; CHECK-NEXT: .p2align 2
544; CHECK-NEXT: @ %bb.10:
545; CHECK-NEXT: .LCPI5_0:
546; CHECK-NEXT: .long 0x00000000 @ float 0
547entry:
548 %cmp6 = icmp sgt i32 %n, 0
549 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
550
551for.body.preheader: ; preds = %entry
552 %min.iters.check = icmp ult i32 %n, 4
553 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
554
555vector.ph: ; preds = %for.body.preheader
556 %n.vec = and i32 %n, -4
557 br label %vector.body
558
559vector.body: ; preds = %vector.body, %vector.ph
560 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
561 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
562 %0 = getelementptr inbounds float, float* %x, i32 %index
563 %1 = bitcast float* %0 to <4 x float>*
564 %wide.load = load <4 x float>, <4 x float>* %1, align 4
565 %2 = fadd fast <4 x float> %wide.load, %vec.phi
566 %index.next = add i32 %index, 4
567 %3 = icmp eq i32 %index.next, %n.vec
568 br i1 %3, label %middle.block, label %vector.body
569
570middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]571 %4 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float 0.000000e+00, <4 x float> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]572 %cmp.n = icmp eq i32 %n.vec, %n
573 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
574
575for.body.preheader1: ; preds = %middle.block, %for.body.preheader
576 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
577 %r.07.ph = phi float [ 0.000000e+00, %for.body.preheader ], [ %4, %middle.block ]
578 br label %for.body
579
580for.body: ; preds = %for.body.preheader1, %for.body
581 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
582 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
583 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
584 %5 = load float, float* %arrayidx, align 4
585 %add = fadd fast float %5, %r.07
586 %inc = add nuw nsw i32 %i.08, 1
587 %exitcond = icmp eq i32 %inc, %n
588 br i1 %exitcond, label %for.cond.cleanup, label %for.body
589
590for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
591 %r.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
592 ret float %r.0.lcssa
593}
594
595define float @fmul_f32(float* nocapture readonly %x, i32 %n) {
596; CHECK-LABEL: fmul_f32:
597; CHECK: @ %bb.0: @ %entry
598; CHECK-NEXT: .save {r7, lr}
599; CHECK-NEXT: push {r7, lr}
600; CHECK-NEXT: cmp r1, #1
601; CHECK-NEXT: blt .LBB6_3
602; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
603; CHECK-NEXT: cmp r1, #4
604; CHECK-NEXT: bhs .LBB6_4
605; CHECK-NEXT: @ %bb.2:
606; CHECK-NEXT: vmov.f32 s0, #1.000000e+00
607; CHECK-NEXT: movs r2, #0
608; CHECK-NEXT: b .LBB6_7
609; CHECK-NEXT: .LBB6_3:
610; CHECK-NEXT: vmov.f32 s0, #1.000000e+00
611; CHECK-NEXT: b .LBB6_9
612; CHECK-NEXT: .LBB6_4: @ %vector.ph
613; CHECK-NEXT: bic r2, r1, #3
614; CHECK-NEXT: movs r3, #1
615; CHECK-NEXT: sub.w r12, r2, #4
616; CHECK-NEXT: vmov.f32 q0, #1.000000e+00
617; CHECK-NEXT: add.w lr, r3, r12, lsr #2
618; CHECK-NEXT: mov r3, r0
619; CHECK-NEXT: dls lr, lr
620; CHECK-NEXT: .LBB6_5: @ %vector.body
621; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
622; CHECK-NEXT: vldrw.u32 q1, [r3], #16
623; CHECK-NEXT: vmul.f32 q0, q1, q0
624; CHECK-NEXT: le lr, .LBB6_5
625; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]626; CHECK-NEXT: vmul.f32 s4, s2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]627; CHECK-NEXT: cmp r2, r1
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]628; CHECK-NEXT: vmul.f32 s0, s0, s1
629; CHECK-NEXT: vmul.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]630; CHECK-NEXT: beq .LBB6_9
631; CHECK-NEXT: .LBB6_7: @ %for.body.preheader1
632; CHECK-NEXT: sub.w lr, r1, r2
633; CHECK-NEXT: add.w r0, r0, r2, lsl #2
634; CHECK-NEXT: dls lr, lr
635; CHECK-NEXT: .LBB6_8: @ %for.body
636; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
637; CHECK-NEXT: vldr s2, [r0]
638; CHECK-NEXT: adds r0, #4
639; CHECK-NEXT: vmul.f32 s0, s2, s0
640; CHECK-NEXT: le lr, .LBB6_8
641; CHECK-NEXT: .LBB6_9: @ %for.cond.cleanup
642; CHECK-NEXT: vmov r0, s0
643; CHECK-NEXT: pop {r7, pc}
644entry:
645 %cmp6 = icmp sgt i32 %n, 0
646 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
647
648for.body.preheader: ; preds = %entry
649 %min.iters.check = icmp ult i32 %n, 4
650 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
651
652vector.ph: ; preds = %for.body.preheader
653 %n.vec = and i32 %n, -4
654 br label %vector.body
655
656vector.body: ; preds = %vector.body, %vector.ph
657 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
658 %vec.phi = phi <4 x float> [ <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %vector.ph ], [ %2, %vector.body ]
659 %0 = getelementptr inbounds float, float* %x, i32 %index
660 %1 = bitcast float* %0 to <4 x float>*
661 %wide.load = load <4 x float>, <4 x float>* %1, align 4
662 %2 = fmul fast <4 x float> %wide.load, %vec.phi
663 %index.next = add i32 %index, 4
664 %3 = icmp eq i32 %index.next, %n.vec
665 br i1 %3, label %middle.block, label %vector.body
666
667middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]668 %4 = call fast float @llvm.vector.reduce.fmul.f32.v4f32(float 1.000000e+00, <4 x float> %2)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]669 %cmp.n = icmp eq i32 %n.vec, %n
670 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
671
672for.body.preheader1: ; preds = %middle.block, %for.body.preheader
673 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
674 %r.07.ph = phi float [ 1.000000e+00, %for.body.preheader ], [ %4, %middle.block ]
675 br label %for.body
676
677for.body: ; preds = %for.body.preheader1, %for.body
678 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
679 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
680 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
681 %5 = load float, float* %arrayidx, align 4
682 %add = fmul fast float %5, %r.07
683 %inc = add nuw nsw i32 %i.08, 1
684 %exitcond = icmp eq i32 %inc, %n
685 br i1 %exitcond, label %for.cond.cleanup, label %for.body
686
687for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
688 %r.0.lcssa = phi float [ 1.000000e+00, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
689 ret float %r.0.lcssa
690}
691
692define i32 @smin_i32(i32* nocapture readonly %x, i32 %n) {
693; CHECK-LABEL: smin_i32:
694; CHECK: @ %bb.0: @ %entry
695; CHECK-NEXT: .save {r7, lr}
696; CHECK-NEXT: push {r7, lr}
697; CHECK-NEXT: cmp r1, #1
698; CHECK-NEXT: blt .LBB7_3
699; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
700; CHECK-NEXT: cmp r1, #4
701; CHECK-NEXT: bhs .LBB7_4
702; CHECK-NEXT: @ %bb.2:
703; CHECK-NEXT: mvn r2, #-2147483648
704; CHECK-NEXT: movs r3, #0
705; CHECK-NEXT: b .LBB7_7
706; CHECK-NEXT: .LBB7_3:
707; CHECK-NEXT: mvn r2, #-2147483648
708; CHECK-NEXT: b .LBB7_9
709; CHECK-NEXT: .LBB7_4: @ %vector.ph
710; CHECK-NEXT: bic r3, r1, #3
711; CHECK-NEXT: movs r2, #1
712; CHECK-NEXT: sub.w r12, r3, #4
713; CHECK-NEXT: vmvn.i32 q0, #0x80000000
714; CHECK-NEXT: add.w lr, r2, r12, lsr #2
715; CHECK-NEXT: mov r2, r0
716; CHECK-NEXT: dls lr, lr
717; CHECK-NEXT: .LBB7_5: @ %vector.body
718; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
719; CHECK-NEXT: vldrw.u32 q1, [r2], #16
720; CHECK-NEXT: vmin.s32 q0, q0, q1
721; CHECK-NEXT: le lr, .LBB7_5
722; CHECK-NEXT: @ %bb.6: @ %middle.block
723; CHECK-NEXT: mvn r2, #-2147483648
724; CHECK-NEXT: cmp r3, r1
725; CHECK-NEXT: vminv.s32 r2, q0
726; CHECK-NEXT: beq .LBB7_9
727; CHECK-NEXT: .LBB7_7: @ %for.body.preheader1
728; CHECK-NEXT: sub.w lr, r1, r3
729; CHECK-NEXT: add.w r0, r0, r3, lsl #2
730; CHECK-NEXT: dls lr, lr
731; CHECK-NEXT: .LBB7_8: @ %for.body
732; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
733; CHECK-NEXT: ldr r1, [r0], #4
734; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]735; CHECK-NEXT: csel r2, r2, r1, lt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]736; CHECK-NEXT: le lr, .LBB7_8
737; CHECK-NEXT: .LBB7_9: @ %for.cond.cleanup
738; CHECK-NEXT: mov r0, r2
739; CHECK-NEXT: pop {r7, pc}
740entry:
741 %cmp6 = icmp sgt i32 %n, 0
742 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
743
744for.body.preheader: ; preds = %entry
745 %min.iters.check = icmp ult i32 %n, 4
746 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
747
748vector.ph: ; preds = %for.body.preheader
749 %n.vec = and i32 %n, -4
750 br label %vector.body
751
752vector.body: ; preds = %vector.body, %vector.ph
753 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
754 %vec.phi = phi <4 x i32> [ <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, %vector.ph ], [ %3, %vector.body ]
755 %0 = getelementptr inbounds i32, i32* %x, i32 %index
756 %1 = bitcast i32* %0 to <4 x i32>*
757 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
758 %2 = icmp slt <4 x i32> %vec.phi, %wide.load
759 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
760 %index.next = add i32 %index, 4
761 %4 = icmp eq i32 %index.next, %n.vec
762 br i1 %4, label %middle.block, label %vector.body
763
764middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]765 %5 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]766 %cmp.n = icmp eq i32 %n.vec, %n
767 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
768
769for.body.preheader1: ; preds = %middle.block, %for.body.preheader
770 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
771 %r.07.ph = phi i32 [ 2147483647, %for.body.preheader ], [ %5, %middle.block ]
772 br label %for.body
773
774for.body: ; preds = %for.body.preheader1, %for.body
775 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
776 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
777 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
778 %6 = load i32, i32* %arrayidx, align 4
779 %c = icmp slt i32 %r.07, %6
780 %add = select i1 %c, i32 %r.07, i32 %6
781 %inc = add nuw nsw i32 %i.08, 1
782 %exitcond = icmp eq i32 %inc, %n
783 br i1 %exitcond, label %for.cond.cleanup, label %for.body
784
785for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
786 %r.0.lcssa = phi i32 [ 2147483647, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
787 ret i32 %r.0.lcssa
788}
789
790define i32 @smin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
791; CHECK-LABEL: smin_i32_inloop:
792; CHECK: @ %bb.0: @ %entry
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]793; CHECK-NEXT: .save {r7, lr}
794; CHECK-NEXT: push {r7, lr}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]795; CHECK-NEXT: cmp r1, #1
796; CHECK-NEXT: blt .LBB8_3
797; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
798; CHECK-NEXT: mov r12, r0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]799; CHECK-NEXT: cmp r1, #4
800; CHECK-NEXT: bhs .LBB8_4
801; CHECK-NEXT: @ %bb.2:
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]802; CHECK-NEXT: mvn r0, #-2147483648
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]803; CHECK-NEXT: movs r3, #0
804; CHECK-NEXT: b .LBB8_7
805; CHECK-NEXT: .LBB8_3:
806; CHECK-NEXT: mvn r0, #-2147483648
807; CHECK-NEXT: b .LBB8_9
808; CHECK-NEXT: .LBB8_4: @ %vector.ph
809; CHECK-NEXT: bic r3, r1, #3
810; CHECK-NEXT: movs r2, #1
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]811; CHECK-NEXT: subs r0, r3, #4
812; CHECK-NEXT: add.w lr, r2, r0, lsr #2
813; CHECK-NEXT: mvn r0, #-2147483648
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]814; CHECK-NEXT: mov r2, r12
815; CHECK-NEXT: dls lr, lr
816; CHECK-NEXT: .LBB8_5: @ %vector.body
817; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
818; CHECK-NEXT: vldrw.u32 q0, [r2], #16
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]819; CHECK-NEXT: vminv.s32 r0, q0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]820; CHECK-NEXT: le lr, .LBB8_5
821; CHECK-NEXT: @ %bb.6: @ %middle.block
822; CHECK-NEXT: cmp r3, r1
823; CHECK-NEXT: it eq
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]824; CHECK-NEXT: popeq {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]825; CHECK-NEXT: .LBB8_7: @ %for.body.preheader1
826; CHECK-NEXT: sub.w lr, r1, r3
827; CHECK-NEXT: add.w r1, r12, r3, lsl #2
828; CHECK-NEXT: dls lr, lr
829; CHECK-NEXT: .LBB8_8: @ %for.body
830; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
831; CHECK-NEXT: ldr r2, [r1], #4
832; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]833; CHECK-NEXT: csel r0, r0, r2, lt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]834; CHECK-NEXT: le lr, .LBB8_8
835; CHECK-NEXT: .LBB8_9: @ %for.cond.cleanup
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]836; CHECK-NEXT: pop {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]837entry:
838 %cmp6 = icmp sgt i32 %n, 0
839 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
840
841for.body.preheader: ; preds = %entry
842 %min.iters.check = icmp ult i32 %n, 4
843 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
844
845vector.ph: ; preds = %for.body.preheader
846 %n.vec = and i32 %n, -4
847 br label %vector.body
848
849vector.body: ; preds = %vector.body, %vector.ph
850 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
851 %vec.phi = phi i32 [ 2147483647, %vector.ph ], [ %3, %vector.body ]
852 %0 = getelementptr inbounds i32, i32* %x, i32 %index
853 %1 = bitcast i32* %0 to <4 x i32>*
854 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]855 %l5 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %wide.load)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]856 %2 = icmp slt i32 %vec.phi, %l5
857 %3 = select i1 %2, i32 %vec.phi, i32 %l5
858 %index.next = add i32 %index, 4
859 %4 = icmp eq i32 %index.next, %n.vec
860 br i1 %4, label %middle.block, label %vector.body
861
862middle.block: ; preds = %vector.body
863 %5 = phi i32 [ %3, %vector.body ]
864 %cmp.n = icmp eq i32 %n.vec, %n
865 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
866
867for.body.preheader1: ; preds = %middle.block, %for.body.preheader
868 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
869 %r.07.ph = phi i32 [ 2147483647, %for.body.preheader ], [ %5, %middle.block ]
870 br label %for.body
871
872for.body: ; preds = %for.body.preheader1, %for.body
873 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
874 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
875 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
876 %6 = load i32, i32* %arrayidx, align 4
877 %c = icmp slt i32 %r.07, %6
878 %add = select i1 %c, i32 %r.07, i32 %6
879 %inc = add nuw nsw i32 %i.08, 1
880 %exitcond = icmp eq i32 %inc, %n
881 br i1 %exitcond, label %for.cond.cleanup, label %for.body
882
883for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
884 %r.0.lcssa = phi i32 [ 2147483647, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
885 ret i32 %r.0.lcssa
886}
887
888define i32 @smax_i32(i32* nocapture readonly %x, i32 %n) {
889; CHECK-LABEL: smax_i32:
890; CHECK: @ %bb.0: @ %entry
891; CHECK-NEXT: .save {r7, lr}
892; CHECK-NEXT: push {r7, lr}
893; CHECK-NEXT: cmp r1, #1
894; CHECK-NEXT: blt .LBB9_3
895; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
896; CHECK-NEXT: cmp r1, #4
897; CHECK-NEXT: bhs .LBB9_4
898; CHECK-NEXT: @ %bb.2:
899; CHECK-NEXT: mov.w r2, #-2147483648
900; CHECK-NEXT: movs r3, #0
901; CHECK-NEXT: b .LBB9_7
902; CHECK-NEXT: .LBB9_3:
903; CHECK-NEXT: mov.w r2, #-2147483648
904; CHECK-NEXT: b .LBB9_9
905; CHECK-NEXT: .LBB9_4: @ %vector.ph
906; CHECK-NEXT: bic r3, r1, #3
907; CHECK-NEXT: movs r2, #1
908; CHECK-NEXT: sub.w r12, r3, #4
909; CHECK-NEXT: vmov.i32 q0, #0x80000000
910; CHECK-NEXT: add.w lr, r2, r12, lsr #2
911; CHECK-NEXT: mov r2, r0
912; CHECK-NEXT: dls lr, lr
913; CHECK-NEXT: .LBB9_5: @ %vector.body
914; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
915; CHECK-NEXT: vldrw.u32 q1, [r2], #16
916; CHECK-NEXT: vmax.s32 q0, q0, q1
917; CHECK-NEXT: le lr, .LBB9_5
918; CHECK-NEXT: @ %bb.6: @ %middle.block
919; CHECK-NEXT: mov.w r2, #-2147483648
920; CHECK-NEXT: cmp r3, r1
921; CHECK-NEXT: vmaxv.s32 r2, q0
922; CHECK-NEXT: beq .LBB9_9
923; CHECK-NEXT: .LBB9_7: @ %for.body.preheader1
924; CHECK-NEXT: sub.w lr, r1, r3
925; CHECK-NEXT: add.w r0, r0, r3, lsl #2
926; CHECK-NEXT: dls lr, lr
927; CHECK-NEXT: .LBB9_8: @ %for.body
928; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
929; CHECK-NEXT: ldr r1, [r0], #4
930; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]931; CHECK-NEXT: csel r2, r2, r1, gt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]932; CHECK-NEXT: le lr, .LBB9_8
933; CHECK-NEXT: .LBB9_9: @ %for.cond.cleanup
934; CHECK-NEXT: mov r0, r2
935; CHECK-NEXT: pop {r7, pc}
936entry:
937 %cmp6 = icmp sgt i32 %n, 0
938 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
939
940for.body.preheader: ; preds = %entry
941 %min.iters.check = icmp ult i32 %n, 4
942 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
943
944vector.ph: ; preds = %for.body.preheader
945 %n.vec = and i32 %n, -4
946 br label %vector.body
947
948vector.body: ; preds = %vector.body, %vector.ph
949 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
950 %vec.phi = phi <4 x i32> [ <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %vector.ph ], [ %3, %vector.body ]
951 %0 = getelementptr inbounds i32, i32* %x, i32 %index
952 %1 = bitcast i32* %0 to <4 x i32>*
953 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
954 %2 = icmp sgt <4 x i32> %vec.phi, %wide.load
955 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
956 %index.next = add i32 %index, 4
957 %4 = icmp eq i32 %index.next, %n.vec
958 br i1 %4, label %middle.block, label %vector.body
959
960middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]961 %5 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]962 %cmp.n = icmp eq i32 %n.vec, %n
963 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
964
965for.body.preheader1: ; preds = %middle.block, %for.body.preheader
966 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
967 %r.07.ph = phi i32 [ -2147483648, %for.body.preheader ], [ %5, %middle.block ]
968 br label %for.body
969
970for.body: ; preds = %for.body.preheader1, %for.body
971 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
972 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
973 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
974 %6 = load i32, i32* %arrayidx, align 4
975 %c = icmp sgt i32 %r.07, %6
976 %add = select i1 %c, i32 %r.07, i32 %6
977 %inc = add nuw nsw i32 %i.08, 1
978 %exitcond = icmp eq i32 %inc, %n
979 br i1 %exitcond, label %for.cond.cleanup, label %for.body
980
981for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
982 %r.0.lcssa = phi i32 [ -2147483648, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
983 ret i32 %r.0.lcssa
984}
985
986define i32 @smax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
987; CHECK-LABEL: smax_i32_inloop:
988; CHECK: @ %bb.0: @ %entry
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]989; CHECK-NEXT: .save {r7, lr}
990; CHECK-NEXT: push {r7, lr}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]991; CHECK-NEXT: cmp r1, #1
992; CHECK-NEXT: blt .LBB10_3
993; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
994; CHECK-NEXT: mov r12, r0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]995; CHECK-NEXT: cmp r1, #4
996; CHECK-NEXT: bhs .LBB10_4
997; CHECK-NEXT: @ %bb.2:
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]998; CHECK-NEXT: mov.w r0, #-2147483648
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]999; CHECK-NEXT: movs r3, #0
1000; CHECK-NEXT: b .LBB10_7
1001; CHECK-NEXT: .LBB10_3:
1002; CHECK-NEXT: mov.w r0, #-2147483648
1003; CHECK-NEXT: b .LBB10_9
1004; CHECK-NEXT: .LBB10_4: @ %vector.ph
1005; CHECK-NEXT: bic r3, r1, #3
1006; CHECK-NEXT: movs r2, #1
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1007; CHECK-NEXT: subs r0, r3, #4
1008; CHECK-NEXT: add.w lr, r2, r0, lsr #2
1009; CHECK-NEXT: mov.w r0, #-2147483648
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1010; CHECK-NEXT: mov r2, r12
1011; CHECK-NEXT: dls lr, lr
1012; CHECK-NEXT: .LBB10_5: @ %vector.body
1013; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1014; CHECK-NEXT: vldrw.u32 q0, [r2], #16
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1015; CHECK-NEXT: vmaxv.s32 r0, q0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1016; CHECK-NEXT: le lr, .LBB10_5
1017; CHECK-NEXT: @ %bb.6: @ %middle.block
1018; CHECK-NEXT: cmp r3, r1
1019; CHECK-NEXT: it eq
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1020; CHECK-NEXT: popeq {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1021; CHECK-NEXT: .LBB10_7: @ %for.body.preheader1
1022; CHECK-NEXT: sub.w lr, r1, r3
1023; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1024; CHECK-NEXT: dls lr, lr
1025; CHECK-NEXT: .LBB10_8: @ %for.body
1026; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1027; CHECK-NEXT: ldr r2, [r1], #4
1028; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1029; CHECK-NEXT: csel r0, r0, r2, gt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1030; CHECK-NEXT: le lr, .LBB10_8
1031; CHECK-NEXT: .LBB10_9: @ %for.cond.cleanup
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1032; CHECK-NEXT: pop {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1033entry:
1034 %cmp6 = icmp sgt i32 %n, 0
1035 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1036
1037for.body.preheader: ; preds = %entry
1038 %min.iters.check = icmp ult i32 %n, 4
1039 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1040
1041vector.ph: ; preds = %for.body.preheader
1042 %n.vec = and i32 %n, -4
1043 br label %vector.body
1044
1045vector.body: ; preds = %vector.body, %vector.ph
1046 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1047 %vec.phi = phi i32 [ -2147483648, %vector.ph ], [ %3, %vector.body ]
1048 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1049 %1 = bitcast i32* %0 to <4 x i32>*
1050 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1051 %l5 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %wide.load)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1052 %2 = icmp sgt i32 %vec.phi, %l5
1053 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1054 %index.next = add i32 %index, 4
1055 %4 = icmp eq i32 %index.next, %n.vec
1056 br i1 %4, label %middle.block, label %vector.body
1057
1058middle.block: ; preds = %vector.body
1059 %5 = phi i32 [ %3, %vector.body ]
1060 %cmp.n = icmp eq i32 %n.vec, %n
1061 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1062
1063for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1064 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1065 %r.07.ph = phi i32 [ -2147483648, %for.body.preheader ], [ %5, %middle.block ]
1066 br label %for.body
1067
1068for.body: ; preds = %for.body.preheader1, %for.body
1069 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1070 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1071 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1072 %6 = load i32, i32* %arrayidx, align 4
1073 %c = icmp sgt i32 %r.07, %6
1074 %add = select i1 %c, i32 %r.07, i32 %6
1075 %inc = add nuw nsw i32 %i.08, 1
1076 %exitcond = icmp eq i32 %inc, %n
1077 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1078
1079for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1080 %r.0.lcssa = phi i32 [ -2147483648, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1081 ret i32 %r.0.lcssa
1082}
1083
1084define i32 @umin_i32(i32* nocapture readonly %x, i32 %n) {
1085; CHECK-LABEL: umin_i32:
1086; CHECK: @ %bb.0: @ %entry
1087; CHECK-NEXT: .save {r7, lr}
1088; CHECK-NEXT: push {r7, lr}
1089; CHECK-NEXT: cmp r1, #1
1090; CHECK-NEXT: blt .LBB11_3
1091; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1092; CHECK-NEXT: cmp r1, #4
1093; CHECK-NEXT: bhs .LBB11_4
1094; CHECK-NEXT: @ %bb.2:
1095; CHECK-NEXT: mov.w r2, #-1
1096; CHECK-NEXT: movs r3, #0
1097; CHECK-NEXT: b .LBB11_7
1098; CHECK-NEXT: .LBB11_3:
1099; CHECK-NEXT: mov.w r2, #-1
1100; CHECK-NEXT: b .LBB11_9
1101; CHECK-NEXT: .LBB11_4: @ %vector.ph
1102; CHECK-NEXT: bic r3, r1, #3
1103; CHECK-NEXT: movs r2, #1
1104; CHECK-NEXT: sub.w r12, r3, #4
1105; CHECK-NEXT: vmov.i8 q0, #0xff
1106; CHECK-NEXT: add.w lr, r2, r12, lsr #2
1107; CHECK-NEXT: mov r2, r0
1108; CHECK-NEXT: dls lr, lr
1109; CHECK-NEXT: .LBB11_5: @ %vector.body
1110; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1111; CHECK-NEXT: vldrw.u32 q1, [r2], #16
1112; CHECK-NEXT: vmin.u32 q0, q0, q1
1113; CHECK-NEXT: le lr, .LBB11_5
1114; CHECK-NEXT: @ %bb.6: @ %middle.block
1115; CHECK-NEXT: mov.w r2, #-1
1116; CHECK-NEXT: cmp r3, r1
1117; CHECK-NEXT: vminv.u32 r2, q0
1118; CHECK-NEXT: beq .LBB11_9
1119; CHECK-NEXT: .LBB11_7: @ %for.body.preheader1
1120; CHECK-NEXT: sub.w lr, r1, r3
1121; CHECK-NEXT: add.w r0, r0, r3, lsl #2
1122; CHECK-NEXT: dls lr, lr
1123; CHECK-NEXT: .LBB11_8: @ %for.body
1124; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1125; CHECK-NEXT: ldr r1, [r0], #4
1126; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1127; CHECK-NEXT: csel r2, r2, r1, lo
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1128; CHECK-NEXT: le lr, .LBB11_8
1129; CHECK-NEXT: .LBB11_9: @ %for.cond.cleanup
1130; CHECK-NEXT: mov r0, r2
1131; CHECK-NEXT: pop {r7, pc}
1132entry:
1133 %cmp6 = icmp sgt i32 %n, 0
1134 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1135
1136for.body.preheader: ; preds = %entry
1137 %min.iters.check = icmp ult i32 %n, 4
1138 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1139
1140vector.ph: ; preds = %for.body.preheader
1141 %n.vec = and i32 %n, -4
1142 br label %vector.body
1143
1144vector.body: ; preds = %vector.body, %vector.ph
1145 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1146 %vec.phi = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %vector.ph ], [ %3, %vector.body ]
1147 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1148 %1 = bitcast i32* %0 to <4 x i32>*
1149 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1150 %2 = icmp ult <4 x i32> %vec.phi, %wide.load
1151 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
1152 %index.next = add i32 %index, 4
1153 %4 = icmp eq i32 %index.next, %n.vec
1154 br i1 %4, label %middle.block, label %vector.body
1155
1156middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1157 %5 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1158 %cmp.n = icmp eq i32 %n.vec, %n
1159 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1160
1161for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1162 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1163 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %5, %middle.block ]
1164 br label %for.body
1165
1166for.body: ; preds = %for.body.preheader1, %for.body
1167 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1168 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1169 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1170 %6 = load i32, i32* %arrayidx, align 4
1171 %c = icmp ult i32 %r.07, %6
1172 %add = select i1 %c, i32 %r.07, i32 %6
1173 %inc = add nuw nsw i32 %i.08, 1
1174 %exitcond = icmp eq i32 %inc, %n
1175 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1176
1177for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1178 %r.0.lcssa = phi i32 [ -1, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1179 ret i32 %r.0.lcssa
1180}
1181
1182define i32 @umin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
1183; CHECK-LABEL: umin_i32_inloop:
1184; CHECK: @ %bb.0: @ %entry
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1185; CHECK-NEXT: .save {r7, lr}
1186; CHECK-NEXT: push {r7, lr}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1187; CHECK-NEXT: cmp r1, #1
1188; CHECK-NEXT: blt .LBB12_3
1189; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1190; CHECK-NEXT: mov r12, r0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1191; CHECK-NEXT: cmp r1, #4
1192; CHECK-NEXT: bhs .LBB12_4
1193; CHECK-NEXT: @ %bb.2:
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1194; CHECK-NEXT: mov.w r0, #-1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1195; CHECK-NEXT: movs r3, #0
1196; CHECK-NEXT: b .LBB12_7
1197; CHECK-NEXT: .LBB12_3:
1198; CHECK-NEXT: mov.w r0, #-1
1199; CHECK-NEXT: b .LBB12_9
1200; CHECK-NEXT: .LBB12_4: @ %vector.ph
1201; CHECK-NEXT: bic r3, r1, #3
1202; CHECK-NEXT: movs r2, #1
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1203; CHECK-NEXT: subs r0, r3, #4
1204; CHECK-NEXT: add.w lr, r2, r0, lsr #2
1205; CHECK-NEXT: mov.w r0, #-1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1206; CHECK-NEXT: mov r2, r12
1207; CHECK-NEXT: dls lr, lr
1208; CHECK-NEXT: .LBB12_5: @ %vector.body
1209; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1210; CHECK-NEXT: vldrw.u32 q0, [r2], #16
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1211; CHECK-NEXT: vminv.u32 r0, q0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1212; CHECK-NEXT: le lr, .LBB12_5
1213; CHECK-NEXT: @ %bb.6: @ %middle.block
1214; CHECK-NEXT: cmp r3, r1
1215; CHECK-NEXT: it eq
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1216; CHECK-NEXT: popeq {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1217; CHECK-NEXT: .LBB12_7: @ %for.body.preheader1
1218; CHECK-NEXT: sub.w lr, r1, r3
1219; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1220; CHECK-NEXT: dls lr, lr
1221; CHECK-NEXT: .LBB12_8: @ %for.body
1222; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1223; CHECK-NEXT: ldr r2, [r1], #4
1224; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1225; CHECK-NEXT: csel r0, r0, r2, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1226; CHECK-NEXT: le lr, .LBB12_8
1227; CHECK-NEXT: .LBB12_9: @ %for.cond.cleanup
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1228; CHECK-NEXT: pop {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1229entry:
1230 %cmp6 = icmp sgt i32 %n, 0
1231 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1232
1233for.body.preheader: ; preds = %entry
1234 %min.iters.check = icmp ult i32 %n, 4
1235 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1236
1237vector.ph: ; preds = %for.body.preheader
1238 %n.vec = and i32 %n, -4
1239 br label %vector.body
1240
1241vector.body: ; preds = %vector.body, %vector.ph
1242 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1243 %vec.phi = phi i32 [ -1, %vector.ph ], [ %3, %vector.body ]
1244 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1245 %1 = bitcast i32* %0 to <4 x i32>*
1246 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1247 %l5 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %wide.load)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1248 %2 = icmp ult i32 %vec.phi, %l5
1249 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1250 %index.next = add i32 %index, 4
1251 %4 = icmp eq i32 %index.next, %n.vec
1252 br i1 %4, label %middle.block, label %vector.body
1253
1254middle.block: ; preds = %vector.body
1255 %5 = phi i32 [ %3, %vector.body ]
1256 %cmp.n = icmp eq i32 %n.vec, %n
1257 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1258
1259for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1260 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1261 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %5, %middle.block ]
1262 br label %for.body
1263
1264for.body: ; preds = %for.body.preheader1, %for.body
1265 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1266 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1267 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1268 %6 = load i32, i32* %arrayidx, align 4
1269 %c = icmp ugt i32 %r.07, %6
1270 %add = select i1 %c, i32 %r.07, i32 %6
1271 %inc = add nuw nsw i32 %i.08, 1
1272 %exitcond = icmp eq i32 %inc, %n
1273 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1274
1275for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1276 %r.0.lcssa = phi i32 [ -1, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1277 ret i32 %r.0.lcssa
1278}
1279
1280define i32 @umax_i32(i32* nocapture readonly %x, i32 %n) {
1281; CHECK-LABEL: umax_i32:
1282; CHECK: @ %bb.0: @ %entry
1283; CHECK-NEXT: .save {r7, lr}
1284; CHECK-NEXT: push {r7, lr}
1285; CHECK-NEXT: cmp r1, #1
1286; CHECK-NEXT: blt .LBB13_3
1287; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1288; CHECK-NEXT: cmp r1, #4
1289; CHECK-NEXT: bhs .LBB13_4
1290; CHECK-NEXT: @ %bb.2:
1291; CHECK-NEXT: movs r3, #0
1292; CHECK-NEXT: movs r2, #0
1293; CHECK-NEXT: b .LBB13_7
1294; CHECK-NEXT: .LBB13_3:
1295; CHECK-NEXT: movs r2, #0
1296; CHECK-NEXT: b .LBB13_9
1297; CHECK-NEXT: .LBB13_4: @ %vector.ph
1298; CHECK-NEXT: bic r3, r1, #3
1299; CHECK-NEXT: movs r2, #1
1300; CHECK-NEXT: sub.w r12, r3, #4
1301; CHECK-NEXT: vmov.i32 q0, #0x0
1302; CHECK-NEXT: add.w lr, r2, r12, lsr #2
1303; CHECK-NEXT: mov r2, r0
1304; CHECK-NEXT: dls lr, lr
1305; CHECK-NEXT: .LBB13_5: @ %vector.body
1306; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1307; CHECK-NEXT: vldrw.u32 q1, [r2], #16
1308; CHECK-NEXT: vmax.u32 q0, q0, q1
1309; CHECK-NEXT: le lr, .LBB13_5
1310; CHECK-NEXT: @ %bb.6: @ %middle.block
1311; CHECK-NEXT: movs r2, #0
1312; CHECK-NEXT: cmp r3, r1
1313; CHECK-NEXT: vmaxv.u32 r2, q0
1314; CHECK-NEXT: beq .LBB13_9
1315; CHECK-NEXT: .LBB13_7: @ %for.body.preheader1
1316; CHECK-NEXT: sub.w lr, r1, r3
1317; CHECK-NEXT: add.w r0, r0, r3, lsl #2
1318; CHECK-NEXT: dls lr, lr
1319; CHECK-NEXT: .LBB13_8: @ %for.body
1320; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1321; CHECK-NEXT: ldr r1, [r0], #4
1322; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1323; CHECK-NEXT: csel r2, r2, r1, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1324; CHECK-NEXT: le lr, .LBB13_8
1325; CHECK-NEXT: .LBB13_9: @ %for.cond.cleanup
1326; CHECK-NEXT: mov r0, r2
1327; CHECK-NEXT: pop {r7, pc}
1328entry:
1329 %cmp6 = icmp sgt i32 %n, 0
1330 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1331
1332for.body.preheader: ; preds = %entry
1333 %min.iters.check = icmp ult i32 %n, 4
1334 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1335
1336vector.ph: ; preds = %for.body.preheader
1337 %n.vec = and i32 %n, -4
1338 br label %vector.body
1339
1340vector.body: ; preds = %vector.body, %vector.ph
1341 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1342 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1343 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1344 %1 = bitcast i32* %0 to <4 x i32>*
1345 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1346 %2 = icmp ugt <4 x i32> %vec.phi, %wide.load
1347 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
1348 %index.next = add i32 %index, 4
1349 %4 = icmp eq i32 %index.next, %n.vec
1350 br i1 %4, label %middle.block, label %vector.body
1351
1352middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1353 %5 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1354 %cmp.n = icmp eq i32 %n.vec, %n
1355 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1356
1357for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1358 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1359 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %5, %middle.block ]
1360 br label %for.body
1361
1362for.body: ; preds = %for.body.preheader1, %for.body
1363 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1364 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1365 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1366 %6 = load i32, i32* %arrayidx, align 4
1367 %c = icmp ugt i32 %r.07, %6
1368 %add = select i1 %c, i32 %r.07, i32 %6
1369 %inc = add nuw nsw i32 %i.08, 1
1370 %exitcond = icmp eq i32 %inc, %n
1371 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1372
1373for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1374 %r.0.lcssa = phi i32 [ 0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1375 ret i32 %r.0.lcssa
1376}
1377
1378define i32 @umax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
1379; CHECK-LABEL: umax_i32_inloop:
1380; CHECK: @ %bb.0: @ %entry
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1381; CHECK-NEXT: .save {r7, lr}
1382; CHECK-NEXT: push {r7, lr}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1383; CHECK-NEXT: cmp r1, #1
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1384; CHECK-NEXT: blt .LBB14_3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1385; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1386; CHECK-NEXT: mov r12, r0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1387; CHECK-NEXT: cmp r1, #4
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1388; CHECK-NEXT: bhs .LBB14_4
1389; CHECK-NEXT: @ %bb.2:
1390; CHECK-NEXT: movs r3, #0
1391; CHECK-NEXT: movs r0, #0
1392; CHECK-NEXT: b .LBB14_7
1393; CHECK-NEXT: .LBB14_3:
1394; CHECK-NEXT: movs r0, #0
1395; CHECK-NEXT: b .LBB14_9
1396; CHECK-NEXT: .LBB14_4: @ %vector.ph
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1397; CHECK-NEXT: bic r3, r1, #3
1398; CHECK-NEXT: movs r2, #1
1399; CHECK-NEXT: subs r0, r3, #4
1400; CHECK-NEXT: add.w lr, r2, r0, lsr #2
1401; CHECK-NEXT: movs r0, #0
1402; CHECK-NEXT: mov r2, r12
1403; CHECK-NEXT: dls lr, lr
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1404; CHECK-NEXT: .LBB14_5: @ %vector.body
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1405; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1406; CHECK-NEXT: vldrw.u32 q0, [r2], #16
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1407; CHECK-NEXT: vmaxv.u32 r0, q0
1408; CHECK-NEXT: le lr, .LBB14_5
1409; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1410; CHECK-NEXT: cmp r3, r1
1411; CHECK-NEXT: it eq
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1412; CHECK-NEXT: popeq {r7, pc}
1413; CHECK-NEXT: .LBB14_7: @ %for.body.preheader1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1414; CHECK-NEXT: sub.w lr, r1, r3
1415; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1416; CHECK-NEXT: dls lr, lr
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1417; CHECK-NEXT: .LBB14_8: @ %for.body
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1418; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1419; CHECK-NEXT: ldr r2, [r1], #4
1420; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1421; CHECK-NEXT: csel r0, r0, r2, hi
Sam Tebbs68e002e2020-09-23 11:43:27 +0100[diff] [blame]1422; CHECK-NEXT: le lr, .LBB14_8
1423; CHECK-NEXT: .LBB14_9: @ %for.cond.cleanup
1424; CHECK-NEXT: pop {r7, pc}
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1425entry:
1426 %cmp6 = icmp sgt i32 %n, 0
1427 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1428
1429for.body.preheader: ; preds = %entry
1430 %min.iters.check = icmp ult i32 %n, 4
1431 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1432
1433vector.ph: ; preds = %for.body.preheader
1434 %n.vec = and i32 %n, -4
1435 br label %vector.body
1436
1437vector.body: ; preds = %vector.body, %vector.ph
1438 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1439 %vec.phi = phi i32 [ 0, %vector.ph ], [ %3, %vector.body ]
1440 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1441 %1 = bitcast i32* %0 to <4 x i32>*
1442 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1443 %l5 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %wide.load)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1444 %2 = icmp ugt i32 %vec.phi, %l5
1445 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1446 %index.next = add i32 %index, 4
1447 %4 = icmp eq i32 %index.next, %n.vec
1448 br i1 %4, label %middle.block, label %vector.body
1449
1450middle.block: ; preds = %vector.body
1451 %5 = phi i32 [ %3, %vector.body ]
1452 %cmp.n = icmp eq i32 %n.vec, %n
1453 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1454
1455for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1456 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1457 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %5, %middle.block ]
1458 br label %for.body
1459
1460for.body: ; preds = %for.body.preheader1, %for.body
1461 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1462 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1463 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1464 %6 = load i32, i32* %arrayidx, align 4
1465 %c = icmp ugt i32 %r.07, %6
1466 %add = select i1 %c, i32 %r.07, i32 %6
1467 %inc = add nuw nsw i32 %i.08, 1
1468 %exitcond = icmp eq i32 %inc, %n
1469 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1470
1471for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1472 %r.0.lcssa = phi i32 [ 0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1473 ret i32 %r.0.lcssa
1474}
1475
1476define float @fmin_f32(float* nocapture readonly %x, i32 %n) {
1477; CHECK-LABEL: fmin_f32:
1478; CHECK: @ %bb.0: @ %entry
1479; CHECK-NEXT: .save {r7, lr}
1480; CHECK-NEXT: push {r7, lr}
1481; CHECK-NEXT: cmp r1, #1
1482; CHECK-NEXT: blt .LBB15_3
1483; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1484; CHECK-NEXT: cmp r1, #4
1485; CHECK-NEXT: bhs .LBB15_4
1486; CHECK-NEXT: @ %bb.2:
1487; CHECK-NEXT: vldr s0, .LCPI15_0
1488; CHECK-NEXT: movs r2, #0
1489; CHECK-NEXT: b .LBB15_7
1490; CHECK-NEXT: .LBB15_3:
1491; CHECK-NEXT: vldr s0, .LCPI15_0
1492; CHECK-NEXT: b .LBB15_9
1493; CHECK-NEXT: .LBB15_4: @ %vector.ph
1494; CHECK-NEXT: bic r2, r1, #3
1495; CHECK-NEXT: movs r3, #1
1496; CHECK-NEXT: sub.w r12, r2, #4
1497; CHECK-NEXT: vmov.i32 q0, #0x0
1498; CHECK-NEXT: add.w lr, r3, r12, lsr #2
1499; CHECK-NEXT: mov r3, r0
1500; CHECK-NEXT: dls lr, lr
1501; CHECK-NEXT: .LBB15_5: @ %vector.body
1502; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1503; CHECK-NEXT: vldrw.u32 q1, [r3], #16
1504; CHECK-NEXT: vcmp.f32 lt, q0, q1
1505; CHECK-NEXT: vpsel q0, q0, q1
1506; CHECK-NEXT: le lr, .LBB15_5
1507; CHECK-NEXT: @ %bb.6: @ %middle.block
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1508; CHECK-NEXT: vminnm.f32 s4, s2, s3
1509; CHECK-NEXT: vminnm.f32 s0, s0, s1
1510; CHECK-NEXT: vminnm.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1511; CHECK-NEXT: cmp r2, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1512; CHECK-NEXT: beq .LBB15_9
1513; CHECK-NEXT: .LBB15_7: @ %for.body.preheader1
1514; CHECK-NEXT: sub.w lr, r1, r2
1515; CHECK-NEXT: add.w r0, r0, r2, lsl #2
1516; CHECK-NEXT: dls lr, lr
1517; CHECK-NEXT: .LBB15_8: @ %for.body
1518; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1519; CHECK-NEXT: vldmia r0!, {s2}
1520; CHECK-NEXT: vcmp.f32 s0, s2
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1521; CHECK-NEXT: vmrs APSR_nzcv, fpscr
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1522; CHECK-NEXT: vselge.f32 s0, s2, s0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1523; CHECK-NEXT: le lr, .LBB15_8
1524; CHECK-NEXT: .LBB15_9: @ %for.cond.cleanup
1525; CHECK-NEXT: vmov r0, s0
1526; CHECK-NEXT: pop {r7, pc}
1527; CHECK-NEXT: .p2align 2
1528; CHECK-NEXT: @ %bb.10:
1529; CHECK-NEXT: .LCPI15_0:
1530; CHECK-NEXT: .long 0x00000000 @ float 0
1531entry:
1532 %cmp6 = icmp sgt i32 %n, 0
1533 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1534
1535for.body.preheader: ; preds = %entry
1536 %min.iters.check = icmp ult i32 %n, 4
1537 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1538
1539vector.ph: ; preds = %for.body.preheader
1540 %n.vec = and i32 %n, -4
1541 br label %vector.body
1542
1543vector.body: ; preds = %vector.body, %vector.ph
1544 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1545 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1546 %0 = getelementptr inbounds float, float* %x, i32 %index
1547 %1 = bitcast float* %0 to <4 x float>*
1548 %wide.load = load <4 x float>, <4 x float>* %1, align 4
1549 %2 = fcmp ult <4 x float> %vec.phi, %wide.load
1550 %3 = select <4 x i1> %2, <4 x float> %vec.phi, <4 x float> %wide.load
1551 %index.next = add i32 %index, 4
1552 %4 = icmp eq i32 %index.next, %n.vec
1553 br i1 %4, label %middle.block, label %vector.body
1554
1555middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1556 %5 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1557 %cmp.n = icmp eq i32 %n.vec, %n
1558 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1559
1560for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1561 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1562 %r.07.ph = phi float [ 0.0, %for.body.preheader ], [ %5, %middle.block ]
1563 br label %for.body
1564
1565for.body: ; preds = %for.body.preheader1, %for.body
1566 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1567 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1568 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
1569 %6 = load float, float* %arrayidx, align 4
1570 %c = fcmp ult float %r.07, %6
1571 %add = select i1 %c, float %r.07, float %6
1572 %inc = add nuw nsw i32 %i.08, 1
1573 %exitcond = icmp eq i32 %inc, %n
1574 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1575
1576for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1577 %r.0.lcssa = phi float [ 0.0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1578 ret float %r.0.lcssa
1579}
1580
1581define float @fmax_f32(float* nocapture readonly %x, i32 %n) {
1582; CHECK-LABEL: fmax_f32:
1583; CHECK: @ %bb.0: @ %entry
1584; CHECK-NEXT: .save {r7, lr}
1585; CHECK-NEXT: push {r7, lr}
1586; CHECK-NEXT: cmp r1, #1
1587; CHECK-NEXT: blt .LBB16_3
1588; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1589; CHECK-NEXT: cmp r1, #4
1590; CHECK-NEXT: bhs .LBB16_4
1591; CHECK-NEXT: @ %bb.2:
1592; CHECK-NEXT: vldr s0, .LCPI16_0
1593; CHECK-NEXT: movs r2, #0
1594; CHECK-NEXT: b .LBB16_7
1595; CHECK-NEXT: .LBB16_3:
1596; CHECK-NEXT: vldr s0, .LCPI16_0
1597; CHECK-NEXT: b .LBB16_9
1598; CHECK-NEXT: .LBB16_4: @ %vector.ph
1599; CHECK-NEXT: bic r2, r1, #3
1600; CHECK-NEXT: movs r3, #1
1601; CHECK-NEXT: sub.w r12, r2, #4
1602; CHECK-NEXT: vmov.i32 q0, #0x0
1603; CHECK-NEXT: add.w lr, r3, r12, lsr #2
1604; CHECK-NEXT: mov r3, r0
1605; CHECK-NEXT: dls lr, lr
1606; CHECK-NEXT: .LBB16_5: @ %vector.body
1607; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1608; CHECK-NEXT: vldrw.u32 q1, [r3], #16
1609; CHECK-NEXT: vcmp.f32 lt, q1, q0
1610; CHECK-NEXT: vpsel q0, q0, q1
1611; CHECK-NEXT: le lr, .LBB16_5
1612; CHECK-NEXT: @ %bb.6: @ %middle.block
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1613; CHECK-NEXT: vmaxnm.f32 s4, s2, s3
1614; CHECK-NEXT: vmaxnm.f32 s0, s0, s1
1615; CHECK-NEXT: vmaxnm.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1616; CHECK-NEXT: cmp r2, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1617; CHECK-NEXT: beq .LBB16_9
1618; CHECK-NEXT: .LBB16_7: @ %for.body.preheader1
1619; CHECK-NEXT: sub.w lr, r1, r2
1620; CHECK-NEXT: add.w r0, r0, r2, lsl #2
1621; CHECK-NEXT: dls lr, lr
1622; CHECK-NEXT: .LBB16_8: @ %for.body
1623; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1624; CHECK-NEXT: vldmia r0!, {s2}
1625; CHECK-NEXT: vcmp.f32 s2, s0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1626; CHECK-NEXT: vmrs APSR_nzcv, fpscr
Sanjay Patel3a8ea862020-09-12 09:08:07 -0400[diff] [blame]1627; CHECK-NEXT: vselge.f32 s0, s2, s0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1628; CHECK-NEXT: le lr, .LBB16_8
1629; CHECK-NEXT: .LBB16_9: @ %for.cond.cleanup
1630; CHECK-NEXT: vmov r0, s0
1631; CHECK-NEXT: pop {r7, pc}
1632; CHECK-NEXT: .p2align 2
1633; CHECK-NEXT: @ %bb.10:
1634; CHECK-NEXT: .LCPI16_0:
1635; CHECK-NEXT: .long 0x00000000 @ float 0
1636entry:
1637 %cmp6 = icmp sgt i32 %n, 0
1638 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1639
1640for.body.preheader: ; preds = %entry
1641 %min.iters.check = icmp ult i32 %n, 4
1642 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1643
1644vector.ph: ; preds = %for.body.preheader
1645 %n.vec = and i32 %n, -4
1646 br label %vector.body
1647
1648vector.body: ; preds = %vector.body, %vector.ph
1649 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1650 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1651 %0 = getelementptr inbounds float, float* %x, i32 %index
1652 %1 = bitcast float* %0 to <4 x float>*
1653 %wide.load = load <4 x float>, <4 x float>* %1, align 4
1654 %2 = fcmp ugt <4 x float> %vec.phi, %wide.load
1655 %3 = select <4 x i1> %2, <4 x float> %vec.phi, <4 x float> %wide.load
1656 %index.next = add i32 %index, 4
1657 %4 = icmp eq i32 %index.next, %n.vec
1658 br i1 %4, label %middle.block, label %vector.body
1659
1660middle.block: ; preds = %vector.body
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1661 %5 = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %3)
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1662 %cmp.n = icmp eq i32 %n.vec, %n
1663 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1664
1665for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1666 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1667 %r.07.ph = phi float [ 0.0, %for.body.preheader ], [ %5, %middle.block ]
1668 br label %for.body
1669
1670for.body: ; preds = %for.body.preheader1, %for.body
1671 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1672 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1673 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
1674 %6 = load float, float* %arrayidx, align 4
1675 %c = fcmp ugt float %r.07, %6
1676 %add = select i1 %c, float %r.07, float %6
1677 %inc = add nuw nsw i32 %i.08, 1
1678 %exitcond = icmp eq i32 %inc, %n
1679 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1680
1681for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1682 %r.0.lcssa = phi float [ 0.0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1683 ret float %r.0.lcssa
1684}
1685
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1686define i32 @add4i32(i32* noalias nocapture readonly %x, i32 %n) {
1687; CHECK-LABEL: add4i32:
1688; CHECK: @ %bb.0: @ %entry
1689; CHECK-NEXT: .save {r7, lr}
1690; CHECK-NEXT: push {r7, lr}
1691; CHECK-NEXT: cbz r1, .LBB17_4
1692; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1693; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1694; CHECK-NEXT: dlstp.32 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1695; CHECK-NEXT: .LBB17_2: @ %vector.body
1696; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1697; CHECK-NEXT: vldrw.u32 q0, [r0], #16
1698; CHECK-NEXT: vaddva.u32 r2, q0
1699; CHECK-NEXT: letp lr, .LBB17_2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1700; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
1701; CHECK-NEXT: mov r0, r2
1702; CHECK-NEXT: pop {r7, pc}
1703; CHECK-NEXT: .LBB17_4:
1704; CHECK-NEXT: movs r2, #0
1705; CHECK-NEXT: mov r0, r2
1706; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1707entry:
1708 %cmp6.not = icmp eq i32 %n, 0
1709 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
1710
1711vector.ph: ; preds = %entry
1712 %n.rnd.up = add i32 %n, 3
1713 %n.vec = and i32 %n.rnd.up, -4
1714 %trip.count.minus.1 = add i32 %n, -1
1715 br label %vector.body
1716
1717vector.body: ; preds = %vector.body, %vector.ph
1718 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1719 %vec.phi = phi i32 [ 0, %vector.ph ], [ %4, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1720 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1721 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1722 %1 = bitcast i32* %0 to <4 x i32>*
1723 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1724 %2 = select <4 x i1> %active.lane.mask, <4 x i32> %wide.masked.load, <4 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1725 %3 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %2)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1726 %4 = add i32 %3, %vec.phi
1727 %index.next = add i32 %index, 4
1728 %5 = icmp eq i32 %index.next, %n.vec
1729 br i1 %5, label %for.cond.cleanup, label %vector.body
1730
1731for.cond.cleanup: ; preds = %vector.body, %entry
1732 %s.0.lcssa = phi i32 [ 0, %entry ], [ %4, %vector.body ]
1733 ret i32 %s.0.lcssa
1734}
1735
1736define i32 @mla4i32(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32 %n) {
1737; CHECK-LABEL: mla4i32:
1738; CHECK: @ %bb.0: @ %entry
1739; CHECK-NEXT: .save {r7, lr}
1740; CHECK-NEXT: push {r7, lr}
1741; CHECK-NEXT: cbz r2, .LBB18_4
1742; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1743; CHECK-NEXT: mov.w r12, #0
1744; CHECK-NEXT: dlstp.32 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1745; CHECK-NEXT: .LBB18_2: @ %vector.body
1746; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1747; CHECK-NEXT: vldrw.u32 q0, [r0], #16
1748; CHECK-NEXT: vldrw.u32 q1, [r1], #16
1749; CHECK-NEXT: vmlava.u32 r12, q1, q0
1750; CHECK-NEXT: letp lr, .LBB18_2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1751; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1752; CHECK-NEXT: mov r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1753; CHECK-NEXT: pop {r7, pc}
1754; CHECK-NEXT: .LBB18_4:
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1755; CHECK-NEXT: mov.w r12, #0
1756; CHECK-NEXT: mov r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1757; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1758entry:
1759 %cmp8.not = icmp eq i32 %n, 0
1760 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
1761
1762vector.ph: ; preds = %entry
1763 %n.rnd.up = add i32 %n, 3
1764 %n.vec = and i32 %n.rnd.up, -4
1765 %trip.count.minus.1 = add i32 %n, -1
1766 br label %vector.body
1767
1768vector.body: ; preds = %vector.body, %vector.ph
1769 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1770 %vec.phi = phi i32 [ 0, %vector.ph ], [ %7, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1771 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1772 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1773 %1 = bitcast i32* %0 to <4 x i32>*
1774 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1775 %2 = getelementptr inbounds i32, i32* %y, i32 %index
1776 %3 = bitcast i32* %2 to <4 x i32>*
1777 %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1778 %4 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
1779 %5 = select <4 x i1> %active.lane.mask, <4 x i32> %4, <4 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1780 %6 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %5)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1781 %7 = add i32 %6, %vec.phi
1782 %index.next = add i32 %index, 4
1783 %8 = icmp eq i32 %index.next, %n.vec
1784 br i1 %8, label %for.cond.cleanup, label %vector.body
1785
1786for.cond.cleanup: ; preds = %vector.body, %entry
1787 %s.0.lcssa = phi i32 [ 0, %entry ], [ %7, %vector.body ]
1788 ret i32 %s.0.lcssa
1789}
1790
1791define i32 @add8i32(i16* noalias nocapture readonly %x, i32 %n) {
1792; CHECK-LABEL: add8i32:
1793; CHECK: @ %bb.0: @ %entry
1794; CHECK-NEXT: .save {r7, lr}
1795; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1796; CHECK-NEXT: cbz r1, .LBB19_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1797; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1798; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1799; CHECK-NEXT: dlstp.16 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1800; CHECK-NEXT: .LBB19_2: @ %vector.body
1801; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1802; CHECK-NEXT: vldrh.u16 q0, [r0], #16
1803; CHECK-NEXT: vaddva.s16 r2, q0
1804; CHECK-NEXT: letp lr, .LBB19_2
1805; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1806; CHECK-NEXT: mov r0, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1807; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1808; CHECK-NEXT: .LBB19_4:
1809; CHECK-NEXT: movs r2, #0
1810; CHECK-NEXT: mov r0, r2
1811; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1812entry:
1813 %cmp6.not = icmp eq i32 %n, 0
1814 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
1815
1816vector.ph: ; preds = %entry
1817 %n.rnd.up = add i32 %n, 7
1818 %n.vec = and i32 %n.rnd.up, -8
1819 %trip.count.minus.1 = add i32 %n, -1
1820 br label %vector.body
1821
1822vector.body: ; preds = %vector.body, %vector.ph
1823 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1824 %vec.phi = phi i32 [ 0, %vector.ph ], [ %5, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1825 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1826 %0 = getelementptr inbounds i16, i16* %x, i32 %index
1827 %1 = bitcast i16* %0 to <8 x i16>*
1828 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
1829 %2 = sext <8 x i16> %wide.masked.load to <8 x i32>
1830 %3 = select <8 x i1> %active.lane.mask, <8 x i32> %2, <8 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1831 %4 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %3)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1832 %5 = add i32 %4, %vec.phi
1833 %index.next = add i32 %index, 8
1834 %6 = icmp eq i32 %index.next, %n.vec
1835 br i1 %6, label %for.cond.cleanup, label %vector.body
1836
1837for.cond.cleanup: ; preds = %vector.body, %entry
1838 %s.0.lcssa = phi i32 [ 0, %entry ], [ %5, %vector.body ]
1839 ret i32 %s.0.lcssa
1840}
1841
1842define i32 @mla8i32(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
1843; CHECK-LABEL: mla8i32:
1844; CHECK: @ %bb.0: @ %entry
1845; CHECK-NEXT: .save {r7, lr}
1846; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1847; CHECK-NEXT: cbz r2, .LBB20_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1848; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1849; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1850; CHECK-NEXT: dlstp.16 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1851; CHECK-NEXT: .LBB20_2: @ %vector.body
1852; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1853; CHECK-NEXT: vldrh.u16 q0, [r0], #16
1854; CHECK-NEXT: vldrh.u16 q1, [r1], #16
1855; CHECK-NEXT: vmlava.s16 r12, q1, q0
1856; CHECK-NEXT: letp lr, .LBB20_2
1857; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1858; CHECK-NEXT: mov r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1859; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1860; CHECK-NEXT: .LBB20_4:
1861; CHECK-NEXT: mov.w r12, #0
1862; CHECK-NEXT: mov r0, r12
1863; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1864entry:
1865 %cmp9.not = icmp eq i32 %n, 0
1866 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
1867
1868vector.ph: ; preds = %entry
1869 %n.rnd.up = add i32 %n, 7
1870 %n.vec = and i32 %n.rnd.up, -8
1871 %trip.count.minus.1 = add i32 %n, -1
1872 br label %vector.body
1873
1874vector.body: ; preds = %vector.body, %vector.ph
1875 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1876 %vec.phi = phi i32 [ 0, %vector.ph ], [ %9, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1877 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1878 %0 = getelementptr inbounds i16, i16* %x, i32 %index
1879 %1 = bitcast i16* %0 to <8 x i16>*
1880 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
1881 %2 = sext <8 x i16> %wide.masked.load to <8 x i32>
1882 %3 = getelementptr inbounds i16, i16* %y, i32 %index
1883 %4 = bitcast i16* %3 to <8 x i16>*
1884 %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
1885 %5 = sext <8 x i16> %wide.masked.load14 to <8 x i32>
1886 %6 = mul nsw <8 x i32> %5, %2
1887 %7 = select <8 x i1> %active.lane.mask, <8 x i32> %6, <8 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1888 %8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %7)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1889 %9 = add i32 %8, %vec.phi
1890 %index.next = add i32 %index, 8
1891 %10 = icmp eq i32 %index.next, %n.vec
1892 br i1 %10, label %for.cond.cleanup, label %vector.body
1893
1894for.cond.cleanup: ; preds = %vector.body, %entry
1895 %s.0.lcssa = phi i32 [ 0, %entry ], [ %9, %vector.body ]
1896 ret i32 %s.0.lcssa
1897}
1898
1899define i32 @add16i32(i8* noalias nocapture readonly %x, i32 %n) {
1900; CHECK-LABEL: add16i32:
1901; CHECK: @ %bb.0: @ %entry
1902; CHECK-NEXT: .save {r7, lr}
1903; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1904; CHECK-NEXT: cbz r1, .LBB21_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1905; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1906; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1907; CHECK-NEXT: dlstp.8 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1908; CHECK-NEXT: .LBB21_2: @ %vector.body
1909; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1910; CHECK-NEXT: vldrb.u8 q0, [r0], #16
1911; CHECK-NEXT: vaddva.u8 r2, q0
1912; CHECK-NEXT: letp lr, .LBB21_2
1913; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1914; CHECK-NEXT: mov r0, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1915; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1916; CHECK-NEXT: .LBB21_4:
1917; CHECK-NEXT: movs r2, #0
1918; CHECK-NEXT: mov r0, r2
1919; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1920entry:
1921 %cmp6.not = icmp eq i32 %n, 0
1922 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
1923
1924vector.ph: ; preds = %entry
1925 %n.rnd.up = add i32 %n, 15
1926 %n.vec = and i32 %n.rnd.up, -16
1927 %trip.count.minus.1 = add i32 %n, -1
1928 br label %vector.body
1929
1930vector.body: ; preds = %vector.body, %vector.ph
1931 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1932 %vec.phi = phi i32 [ 0, %vector.ph ], [ %5, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1933 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1934 %0 = getelementptr inbounds i8, i8* %x, i32 %index
1935 %1 = bitcast i8* %0 to <16 x i8>*
1936 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
1937 %2 = zext <16 x i8> %wide.masked.load to <16 x i32>
1938 %3 = select <16 x i1> %active.lane.mask, <16 x i32> %2, <16 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1939 %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1940 %5 = add i32 %4, %vec.phi
1941 %index.next = add i32 %index, 16
1942 %6 = icmp eq i32 %index.next, %n.vec
1943 br i1 %6, label %for.cond.cleanup, label %vector.body
1944
1945for.cond.cleanup: ; preds = %vector.body, %entry
1946 %s.0.lcssa = phi i32 [ 0, %entry ], [ %5, %vector.body ]
1947 ret i32 %s.0.lcssa
1948}
1949
1950define i32 @mla16i32(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
1951; CHECK-LABEL: mla16i32:
1952; CHECK: @ %bb.0: @ %entry
1953; CHECK-NEXT: .save {r7, lr}
1954; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1955; CHECK-NEXT: cbz r2, .LBB22_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1956; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1957; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1958; CHECK-NEXT: dlstp.8 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1959; CHECK-NEXT: .LBB22_2: @ %vector.body
1960; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1961; CHECK-NEXT: vldrb.u8 q0, [r0], #16
1962; CHECK-NEXT: vldrb.u8 q1, [r1], #16
1963; CHECK-NEXT: vmlava.u8 r12, q1, q0
1964; CHECK-NEXT: letp lr, .LBB22_2
1965; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1966; CHECK-NEXT: mov r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1967; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]1968; CHECK-NEXT: .LBB22_4:
1969; CHECK-NEXT: mov.w r12, #0
1970; CHECK-NEXT: mov r0, r12
1971; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1972entry:
1973 %cmp9.not = icmp eq i32 %n, 0
1974 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
1975
1976vector.ph: ; preds = %entry
1977 %n.rnd.up = add i32 %n, 15
1978 %n.vec = and i32 %n.rnd.up, -16
1979 %trip.count.minus.1 = add i32 %n, -1
1980 br label %vector.body
1981
1982vector.body: ; preds = %vector.body, %vector.ph
1983 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1984 %vec.phi = phi i32 [ 0, %vector.ph ], [ %9, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]1985 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1986 %0 = getelementptr inbounds i8, i8* %x, i32 %index
1987 %1 = bitcast i8* %0 to <16 x i8>*
1988 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
1989 %2 = zext <16 x i8> %wide.masked.load to <16 x i32>
1990 %3 = getelementptr inbounds i8, i8* %y, i32 %index
1991 %4 = bitcast i8* %3 to <16 x i8>*
1992 %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
1993 %5 = zext <16 x i8> %wide.masked.load14 to <16 x i32>
1994 %6 = mul nuw nsw <16 x i32> %5, %2
1995 %7 = select <16 x i1> %active.lane.mask, <16 x i32> %6, <16 x i32> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]1996 %8 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %7)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]1997 %9 = add i32 %8, %vec.phi
1998 %index.next = add i32 %index, 16
1999 %10 = icmp eq i32 %index.next, %n.vec
2000 br i1 %10, label %for.cond.cleanup, label %vector.body
2001
2002for.cond.cleanup: ; preds = %vector.body, %entry
2003 %s.0.lcssa = phi i32 [ 0, %entry ], [ %9, %vector.body ]
2004 ret i32 %s.0.lcssa
2005}
2006
2007define signext i16 @add8i16(i16* noalias nocapture readonly %x, i32 %n) {
2008; CHECK-LABEL: add8i16:
2009; CHECK: @ %bb.0: @ %entry
2010; CHECK-NEXT: .save {r7, lr}
2011; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2012; CHECK-NEXT: cbz r1, .LBB23_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2013; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2014; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2015; CHECK-NEXT: dlstp.16 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2016; CHECK-NEXT: .LBB23_2: @ %vector.body
2017; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2018; CHECK-NEXT: vldrh.u16 q0, [r0], #16
2019; CHECK-NEXT: vaddva.u16 r2, q0
2020; CHECK-NEXT: letp lr, .LBB23_2
2021; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2022; CHECK-NEXT: sxth r0, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2023; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2024; CHECK-NEXT: .LBB23_4:
2025; CHECK-NEXT: movs r2, #0
2026; CHECK-NEXT: sxth r0, r2
2027; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2028entry:
2029 %cmp8.not = icmp eq i32 %n, 0
2030 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
2031
2032vector.ph: ; preds = %entry
2033 %n.rnd.up = add i32 %n, 7
2034 %n.vec = and i32 %n.rnd.up, -8
2035 %trip.count.minus.1 = add i32 %n, -1
2036 br label %vector.body
2037
2038vector.body: ; preds = %vector.body, %vector.ph
2039 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2040 %vec.phi = phi i16 [ 0, %vector.ph ], [ %4, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2041 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2042 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2043 %1 = bitcast i16* %0 to <8 x i16>*
2044 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2045 %2 = select <8 x i1> %active.lane.mask, <8 x i16> %wide.masked.load, <8 x i16> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2046 %3 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %2)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2047 %4 = add i16 %3, %vec.phi
2048 %index.next = add i32 %index, 8
2049 %5 = icmp eq i32 %index.next, %n.vec
2050 br i1 %5, label %for.cond.cleanup, label %vector.body
2051
2052for.cond.cleanup: ; preds = %vector.body, %entry
2053 %s.0.lcssa = phi i16 [ 0, %entry ], [ %4, %vector.body ]
2054 ret i16 %s.0.lcssa
2055}
2056
2057define signext i16 @mla8i16(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
2058; CHECK-LABEL: mla8i16:
2059; CHECK: @ %bb.0: @ %entry
2060; CHECK-NEXT: .save {r7, lr}
2061; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2062; CHECK-NEXT: cbz r2, .LBB24_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2063; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2064; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2065; CHECK-NEXT: dlstp.16 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2066; CHECK-NEXT: .LBB24_2: @ %vector.body
2067; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2068; CHECK-NEXT: vldrh.u16 q0, [r0], #16
2069; CHECK-NEXT: vldrh.u16 q1, [r1], #16
2070; CHECK-NEXT: vmlava.u16 r12, q1, q0
2071; CHECK-NEXT: letp lr, .LBB24_2
2072; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2073; CHECK-NEXT: sxth.w r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2074; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2075; CHECK-NEXT: .LBB24_4:
2076; CHECK-NEXT: mov.w r12, #0
2077; CHECK-NEXT: sxth.w r0, r12
2078; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2079entry:
2080 %cmp11.not = icmp eq i32 %n, 0
2081 br i1 %cmp11.not, label %for.cond.cleanup, label %vector.ph
2082
2083vector.ph: ; preds = %entry
2084 %n.rnd.up = add i32 %n, 7
2085 %n.vec = and i32 %n.rnd.up, -8
2086 %trip.count.minus.1 = add i32 %n, -1
2087 br label %vector.body
2088
2089vector.body: ; preds = %vector.body, %vector.ph
2090 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2091 %vec.phi = phi i16 [ 0, %vector.ph ], [ %7, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2092 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2093 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2094 %1 = bitcast i16* %0 to <8 x i16>*
2095 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2096 %2 = getelementptr inbounds i16, i16* %y, i32 %index
2097 %3 = bitcast i16* %2 to <8 x i16>*
2098 %wide.masked.load16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2099 %4 = mul <8 x i16> %wide.masked.load16, %wide.masked.load
2100 %5 = select <8 x i1> %active.lane.mask, <8 x i16> %4, <8 x i16> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2101 %6 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %5)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2102 %7 = add i16 %6, %vec.phi
2103 %index.next = add i32 %index, 8
2104 %8 = icmp eq i32 %index.next, %n.vec
2105 br i1 %8, label %for.cond.cleanup, label %vector.body
2106
2107for.cond.cleanup: ; preds = %vector.body, %entry
2108 %s.0.lcssa = phi i16 [ 0, %entry ], [ %7, %vector.body ]
2109 ret i16 %s.0.lcssa
2110}
2111
2112define signext i16 @add16i16(i8* noalias nocapture readonly %x, i32 %n) {
2113; CHECK-LABEL: add16i16:
2114; CHECK: @ %bb.0: @ %entry
2115; CHECK-NEXT: .save {r7, lr}
2116; CHECK-NEXT: push {r7, lr}
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2117; CHECK-NEXT: cbz r1, .LBB25_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2118; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2119; CHECK-NEXT: movs r2, #0
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2120; CHECK-NEXT: dlstp.8 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2121; CHECK-NEXT: .LBB25_2: @ %vector.body
2122; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2123; CHECK-NEXT: vldrb.u8 q0, [r0], #16
2124; CHECK-NEXT: vaddva.u8 r2, q0
2125; CHECK-NEXT: letp lr, .LBB25_2
2126; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2127; CHECK-NEXT: sxth r0, r2
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2128; CHECK-NEXT: pop {r7, pc}
2129; CHECK-NEXT: .LBB25_4:
2130; CHECK-NEXT: movs r2, #0
2131; CHECK-NEXT: sxth r0, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2132; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2133entry:
2134 %cmp8.not = icmp eq i32 %n, 0
2135 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
2136
2137vector.ph: ; preds = %entry
2138 %n.rnd.up = add i32 %n, 15
2139 %n.vec = and i32 %n.rnd.up, -16
2140 %trip.count.minus.1 = add i32 %n, -1
2141 br label %vector.body
2142
2143vector.body: ; preds = %vector.body, %vector.ph
2144 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2145 %vec.phi = phi i16 [ 0, %vector.ph ], [ %5, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2146 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2147 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2148 %1 = bitcast i8* %0 to <16 x i8>*
2149 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2150 %2 = zext <16 x i8> %wide.masked.load to <16 x i16>
2151 %3 = select <16 x i1> %active.lane.mask, <16 x i16> %2, <16 x i16> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2152 %4 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %3)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2153 %5 = add i16 %4, %vec.phi
2154 %index.next = add i32 %index, 16
2155 %6 = icmp eq i32 %index.next, %n.vec
2156 br i1 %6, label %for.cond.cleanup, label %vector.body
2157
2158for.cond.cleanup: ; preds = %vector.body, %entry
2159 %s.0.lcssa = phi i16 [ 0, %entry ], [ %5, %vector.body ]
2160 ret i16 %s.0.lcssa
2161}
2162
2163define signext i16 @mla16i16(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
2164; CHECK-LABEL: mla16i16:
2165; CHECK: @ %bb.0: @ %entry
2166; CHECK-NEXT: .save {r7, lr}
2167; CHECK-NEXT: push {r7, lr}
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2168; CHECK-NEXT: cbz r2, .LBB26_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2169; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2170; CHECK-NEXT: mov.w r12, #0
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2171; CHECK-NEXT: dlstp.8 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2172; CHECK-NEXT: .LBB26_2: @ %vector.body
2173; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2174; CHECK-NEXT: vldrb.u8 q0, [r0], #16
2175; CHECK-NEXT: vldrb.u8 q1, [r1], #16
2176; CHECK-NEXT: vmlava.u8 r12, q1, q0
2177; CHECK-NEXT: letp lr, .LBB26_2
2178; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2179; CHECK-NEXT: sxth.w r0, r12
David Green186a7f82020-08-09 11:09:49 +0100[diff] [blame]2180; CHECK-NEXT: pop {r7, pc}
2181; CHECK-NEXT: .LBB26_4:
2182; CHECK-NEXT: mov.w r12, #0
2183; CHECK-NEXT: sxth.w r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2184; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2185entry:
2186 %cmp13.not = icmp eq i32 %n, 0
2187 br i1 %cmp13.not, label %for.cond.cleanup, label %vector.ph
2188
2189vector.ph: ; preds = %entry
2190 %n.rnd.up = add i32 %n, 15
2191 %n.vec = and i32 %n.rnd.up, -16
2192 %trip.count.minus.1 = add i32 %n, -1
2193 br label %vector.body
2194
2195vector.body: ; preds = %vector.body, %vector.ph
2196 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2197 %vec.phi = phi i16 [ 0, %vector.ph ], [ %9, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2198 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2199 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2200 %1 = bitcast i8* %0 to <16 x i8>*
2201 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2202 %2 = zext <16 x i8> %wide.masked.load to <16 x i16>
2203 %3 = getelementptr inbounds i8, i8* %y, i32 %index
2204 %4 = bitcast i8* %3 to <16 x i8>*
2205 %wide.masked.load18 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2206 %5 = zext <16 x i8> %wide.masked.load18 to <16 x i16>
2207 %6 = mul nuw <16 x i16> %5, %2
2208 %7 = select <16 x i1> %active.lane.mask, <16 x i16> %6, <16 x i16> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2209 %8 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %7)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2210 %9 = add i16 %8, %vec.phi
2211 %index.next = add i32 %index, 16
2212 %10 = icmp eq i32 %index.next, %n.vec
2213 br i1 %10, label %for.cond.cleanup, label %vector.body
2214
2215for.cond.cleanup: ; preds = %vector.body, %entry
2216 %s.0.lcssa = phi i16 [ 0, %entry ], [ %9, %vector.body ]
2217 ret i16 %s.0.lcssa
2218}
2219
2220define zeroext i8 @add16i8(i8* noalias nocapture readonly %x, i32 %n) {
2221; CHECK-LABEL: add16i8:
2222; CHECK: @ %bb.0: @ %entry
2223; CHECK-NEXT: .save {r7, lr}
2224; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2225; CHECK-NEXT: cbz r1, .LBB27_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2226; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2227; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2228; CHECK-NEXT: dlstp.8 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2229; CHECK-NEXT: .LBB27_2: @ %vector.body
2230; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2231; CHECK-NEXT: vldrb.u8 q0, [r0], #16
2232; CHECK-NEXT: vaddva.u8 r2, q0
2233; CHECK-NEXT: letp lr, .LBB27_2
2234; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2235; CHECK-NEXT: uxtb r0, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2236; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2237; CHECK-NEXT: .LBB27_4:
2238; CHECK-NEXT: movs r2, #0
2239; CHECK-NEXT: uxtb r0, r2
2240; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2241entry:
2242 %cmp7.not = icmp eq i32 %n, 0
2243 br i1 %cmp7.not, label %for.cond.cleanup, label %vector.ph
2244
2245vector.ph: ; preds = %entry
2246 %n.rnd.up = add i32 %n, 15
2247 %n.vec = and i32 %n.rnd.up, -16
2248 %trip.count.minus.1 = add i32 %n, -1
2249 br label %vector.body
2250
2251vector.body: ; preds = %vector.body, %vector.ph
2252 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2253 %vec.phi = phi i8 [ 0, %vector.ph ], [ %4, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2254 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2255 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2256 %1 = bitcast i8* %0 to <16 x i8>*
2257 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2258 %2 = select <16 x i1> %active.lane.mask, <16 x i8> %wide.masked.load, <16 x i8> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2259 %3 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %2)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2260 %4 = add i8 %3, %vec.phi
2261 %index.next = add i32 %index, 16
2262 %5 = icmp eq i32 %index.next, %n.vec
2263 br i1 %5, label %for.cond.cleanup, label %vector.body
2264
2265for.cond.cleanup: ; preds = %vector.body, %entry
2266 %s.0.lcssa = phi i8 [ 0, %entry ], [ %4, %vector.body ]
2267 ret i8 %s.0.lcssa
2268}
2269
2270define zeroext i8 @mla16i8(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
2271; CHECK-LABEL: mla16i8:
2272; CHECK: @ %bb.0: @ %entry
2273; CHECK-NEXT: .save {r7, lr}
2274; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2275; CHECK-NEXT: cbz r2, .LBB28_4
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2276; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2277; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2278; CHECK-NEXT: dlstp.8 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2279; CHECK-NEXT: .LBB28_2: @ %vector.body
2280; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2281; CHECK-NEXT: vldrb.u8 q0, [r0], #16
2282; CHECK-NEXT: vldrb.u8 q1, [r1], #16
2283; CHECK-NEXT: vmlava.u8 r12, q1, q0
2284; CHECK-NEXT: letp lr, .LBB28_2
2285; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2286; CHECK-NEXT: uxtb.w r0, r12
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2287; CHECK-NEXT: pop {r7, pc}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2288; CHECK-NEXT: .LBB28_4:
2289; CHECK-NEXT: mov.w r12, #0
2290; CHECK-NEXT: uxtb.w r0, r12
2291; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2292entry:
2293 %cmp10.not = icmp eq i32 %n, 0
2294 br i1 %cmp10.not, label %for.cond.cleanup, label %vector.ph
2295
2296vector.ph: ; preds = %entry
2297 %n.rnd.up = add i32 %n, 15
2298 %n.vec = and i32 %n.rnd.up, -16
2299 %trip.count.minus.1 = add i32 %n, -1
2300 br label %vector.body
2301
2302vector.body: ; preds = %vector.body, %vector.ph
2303 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2304 %vec.phi = phi i8 [ 0, %vector.ph ], [ %7, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2305 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2306 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2307 %1 = bitcast i8* %0 to <16 x i8>*
2308 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2309 %2 = getelementptr inbounds i8, i8* %y, i32 %index
2310 %3 = bitcast i8* %2 to <16 x i8>*
2311 %wide.masked.load15 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %3, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2312 %4 = mul <16 x i8> %wide.masked.load15, %wide.masked.load
2313 %5 = select <16 x i1> %active.lane.mask, <16 x i8> %4, <16 x i8> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2314 %6 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %5)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2315 %7 = add i8 %6, %vec.phi
2316 %index.next = add i32 %index, 16
2317 %8 = icmp eq i32 %index.next, %n.vec
2318 br i1 %8, label %for.cond.cleanup, label %vector.body
2319
2320for.cond.cleanup: ; preds = %vector.body, %entry
2321 %s.0.lcssa = phi i8 [ 0, %entry ], [ %7, %vector.body ]
2322 ret i8 %s.0.lcssa
2323}
2324
2325define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
2326; CHECK-LABEL: add4i64:
2327; CHECK: @ %bb.0: @ %entry
2328; CHECK-NEXT: .save {r7, lr}
2329; CHECK-NEXT: push {r7, lr}
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2330; CHECK-NEXT: cbz r1, .LBB29_3
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2331; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2332; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2333; CHECK-NEXT: mov r3, r2
2334; CHECK-NEXT: dlstp.32 lr, r1
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2335; CHECK-NEXT: .LBB29_2: @ %vector.body
2336; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2337; CHECK-NEXT: vldrw.u32 q0, [r0], #16
2338; CHECK-NEXT: vaddlva.s32 r2, r3, q0
2339; CHECK-NEXT: letp lr, .LBB29_2
2340; CHECK-NEXT: b .LBB29_4
2341; CHECK-NEXT: .LBB29_3:
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2342; CHECK-NEXT: movs r2, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2343; CHECK-NEXT: mov r3, r2
2344; CHECK-NEXT: .LBB29_4: @ %for.cond.cleanup
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2345; CHECK-NEXT: mov r0, r2
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2346; CHECK-NEXT: mov r1, r3
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2347; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2348entry:
2349 %cmp6.not = icmp eq i32 %n, 0
2350 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
2351
2352vector.ph: ; preds = %entry
2353 %n.rnd.up = add i32 %n, 3
2354 %n.vec = and i32 %n.rnd.up, -4
2355 %trip.count.minus.1 = add i32 %n, -1
2356 br label %vector.body
2357
2358vector.body: ; preds = %vector.body, %vector.ph
2359 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2360 %vec.phi = phi i64 [ 0, %vector.ph ], [ %5, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2361 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2362 %0 = getelementptr inbounds i32, i32* %x, i32 %index
2363 %1 = bitcast i32* %0 to <4 x i32>*
2364 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
2365 %2 = sext <4 x i32> %wide.masked.load to <4 x i64>
2366 %3 = select <4 x i1> %active.lane.mask, <4 x i64> %2, <4 x i64> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2367 %4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %3)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2368 %5 = add i64 %4, %vec.phi
2369 %index.next = add i32 %index, 4
2370 %6 = icmp eq i32 %index.next, %n.vec
2371 br i1 %6, label %for.cond.cleanup, label %vector.body
2372
2373for.cond.cleanup: ; preds = %vector.body, %entry
2374 %s.0.lcssa = phi i64 [ 0, %entry ], [ %5, %vector.body ]
2375 ret i64 %s.0.lcssa
2376}
2377
2378define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32 %n) {
2379; CHECK-LABEL: mla4i64:
2380; CHECK: @ %bb.0: @ %entry
2381; CHECK-NEXT: .save {r7, lr}
2382; CHECK-NEXT: push {r7, lr}
2383; CHECK-NEXT: cbz r2, .LBB30_3
2384; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2385; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2386; CHECK-NEXT: mov r3, r12
2387; CHECK-NEXT: dlstp.32 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2388; CHECK-NEXT: .LBB30_2: @ %vector.body
2389; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2390; CHECK-NEXT: vldrw.u32 q0, [r0], #16
2391; CHECK-NEXT: vldrw.u32 q1, [r1], #16
2392; CHECK-NEXT: vmlalva.s32 r12, r3, q1, q0
2393; CHECK-NEXT: letp lr, .LBB30_2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2394; CHECK-NEXT: b .LBB30_4
2395; CHECK-NEXT: .LBB30_3:
2396; CHECK-NEXT: mov.w r12, #0
2397; CHECK-NEXT: mov r3, r12
2398; CHECK-NEXT: .LBB30_4: @ %for.cond.cleanup
2399; CHECK-NEXT: mov r0, r12
2400; CHECK-NEXT: mov r1, r3
2401; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2402entry:
2403 %cmp9.not = icmp eq i32 %n, 0
2404 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
2405
2406vector.ph: ; preds = %entry
2407 %n.rnd.up = add i32 %n, 3
2408 %n.vec = and i32 %n.rnd.up, -4
2409 %trip.count.minus.1 = add i32 %n, -1
2410 br label %vector.body
2411
2412vector.body: ; preds = %vector.body, %vector.ph
2413 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2414 %vec.phi = phi i64 [ 0, %vector.ph ], [ %9, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2415 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2416 %0 = getelementptr inbounds i32, i32* %x, i32 %index
2417 %1 = bitcast i32* %0 to <4 x i32>*
2418 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
2419 %2 = sext <4 x i32> %wide.masked.load to <4 x i64>
2420 %3 = getelementptr inbounds i32, i32* %y, i32 %index
2421 %4 = bitcast i32* %3 to <4 x i32>*
2422 %wide.masked.load14 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
2423 %5 = sext <4 x i32> %wide.masked.load14 to <4 x i64>
2424 %6 = mul nsw <4 x i64> %5, %2
2425 %7 = select <4 x i1> %active.lane.mask, <4 x i64> %6, <4 x i64> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2426 %8 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %7)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2427 %9 = add i64 %8, %vec.phi
2428 %index.next = add i32 %index, 4
2429 %10 = icmp eq i32 %index.next, %n.vec
2430 br i1 %10, label %for.cond.cleanup, label %vector.body
2431
2432for.cond.cleanup: ; preds = %vector.body, %entry
2433 %s.0.lcssa = phi i64 [ 0, %entry ], [ %9, %vector.body ]
2434 ret i64 %s.0.lcssa
2435}
2436
2437define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
2438; CHECK-LABEL: mla8i64:
2439; CHECK: @ %bb.0: @ %entry
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2440; CHECK-NEXT: .save {r7, lr}
2441; CHECK-NEXT: push {r7, lr}
2442; CHECK-NEXT: cbz r2, .LBB31_3
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2443; CHECK-NEXT: @ %bb.1: @ %vector.ph
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2444; CHECK-NEXT: mov.w r12, #0
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2445; CHECK-NEXT: mov r3, r12
2446; CHECK-NEXT: dlstp.16 lr, r2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2447; CHECK-NEXT: .LBB31_2: @ %vector.body
2448; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2449; CHECK-NEXT: vldrh.u16 q0, [r0], #16
2450; CHECK-NEXT: vldrh.u16 q1, [r1], #16
2451; CHECK-NEXT: vmlalva.s16 r12, r3, q1, q0
2452; CHECK-NEXT: letp lr, .LBB31_2
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2453; CHECK-NEXT: b .LBB31_4
2454; CHECK-NEXT: .LBB31_3:
2455; CHECK-NEXT: mov.w r12, #0
2456; CHECK-NEXT: mov r3, r12
2457; CHECK-NEXT: .LBB31_4: @ %for.cond.cleanup
2458; CHECK-NEXT: mov r0, r12
2459; CHECK-NEXT: mov r1, r3
David Green8590e5a2020-08-09 10:57:17 +0100[diff] [blame]2460; CHECK-NEXT: pop {r7, pc}
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2461entry:
2462 %cmp9.not = icmp eq i32 %n, 0
2463 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
2464
2465vector.ph: ; preds = %entry
2466 %n.rnd.up = add i32 %n, 7
2467 %n.vec = and i32 %n.rnd.up, -8
2468 %trip.count.minus.1 = add i32 %n, -1
2469 br label %vector.body
2470
2471vector.body: ; preds = %vector.body, %vector.ph
2472 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2473 %vec.phi = phi i64 [ 0, %vector.ph ], [ %9, %vector.body ]
Sjoerd Meijerc352e7f2020-08-25 13:53:26 +0100[diff] [blame]2474 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2475 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2476 %1 = bitcast i16* %0 to <8 x i16>*
2477 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2478 %2 = sext <8 x i16> %wide.masked.load to <8 x i64>
2479 %3 = getelementptr inbounds i16, i16* %y, i32 %index
2480 %4 = bitcast i16* %3 to <8 x i16>*
2481 %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2482 %5 = sext <8 x i16> %wide.masked.load14 to <8 x i64>
2483 %6 = mul nsw <8 x i64> %5, %2
2484 %7 = select <8 x i1> %active.lane.mask, <8 x i64> %6, <8 x i64> zeroinitializer
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2485 %8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %7)
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2486 %9 = add i64 %8, %vec.phi
2487 %index.next = add i32 %index, 8
2488 %10 = icmp eq i32 %index.next, %n.vec
2489 br i1 %10, label %for.cond.cleanup, label %vector.body
2490
2491for.cond.cleanup: ; preds = %vector.body, %entry
2492 %s.0.lcssa = phi i64 [ 0, %entry ], [ %9, %vector.body ]
2493 ret i64 %s.0.lcssa
2494}
2495
2496declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #1
2497declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #2
2498declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32) #1
2499declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>) #2
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2500declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>) #3
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2501declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32) #1
2502declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) #2
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2503declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>) #3
2504declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>) #3
2505declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>) #3
2506declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>) #3
2507declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>) #3
2508declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>) #3
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame]2509
Amara Emerson322d0af2020-10-02 18:30:53 -0700[diff] [blame^]2510declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
2511declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
2512declare i32 @llvm.vector.reduce.and.v4i32(<4 x i32>)
2513declare i32 @llvm.vector.reduce.or.v4i32(<4 x i32>)
2514declare i32 @llvm.vector.reduce.xor.v4i32(<4 x i32>)
2515declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)
2516declare float @llvm.vector.reduce.fmul.f32.v4f32(float, <4 x float>)
2517declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
2518declare i32 @llvm.vector.reduce.smax.v4i32(<4 x i32>)
2519declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)
2520declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
2521declare float @llvm.vector.reduce.fmin.v4f32(<4 x float>)
2522declare float @llvm.vector.reduce.fmax.v4f32(<4 x float>)