Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 25e38c3f3c2a814797f53d0f08a687b3b4a028e4 / . / llvm / test / CodeGen / Thumb2 / mve-vecreduce-loops.ll
blob: f3252ac1c4a84d2a3415909238f55a48e6e2983c [file] [log] [blame]
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame^]2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -tail-predication=enabled -verify-machineinstrs %s -o - | FileCheck %s
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]3
4define i32 @add_i32(i32* nocapture readonly %x, i32 %n) {
5; CHECK-LABEL: add_i32:
6; CHECK: @ %bb.0: @ %entry
7; CHECK-NEXT: .save {r7, lr}
8; CHECK-NEXT: push {r7, lr}
9; CHECK-NEXT: cmp r1, #1
10; CHECK-NEXT: blt .LBB0_3
11; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
12; CHECK-NEXT: mov r12, r0
13; CHECK-NEXT: cmp r1, #4
14; CHECK-NEXT: bhs .LBB0_4
15; CHECK-NEXT: @ %bb.2:
16; CHECK-NEXT: movs r3, #0
17; CHECK-NEXT: movs r0, #0
18; CHECK-NEXT: b .LBB0_7
19; CHECK-NEXT: .LBB0_3:
20; CHECK-NEXT: movs r0, #0
21; CHECK-NEXT: b .LBB0_9
22; CHECK-NEXT: .LBB0_4: @ %vector.ph
23; CHECK-NEXT: bic r3, r1, #3
24; CHECK-NEXT: movs r2, #1
25; CHECK-NEXT: subs r0, r3, #4
26; CHECK-NEXT: add.w lr, r2, r0, lsr #2
27; CHECK-NEXT: movs r0, #0
28; CHECK-NEXT: mov r2, r12
29; CHECK-NEXT: dls lr, lr
30; CHECK-NEXT: .LBB0_5: @ %vector.body
31; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
32; CHECK-NEXT: vldrw.u32 q0, [r2], #16
33; CHECK-NEXT: vaddva.u32 r0, q0
34; CHECK-NEXT: le lr, .LBB0_5
35; CHECK-NEXT: @ %bb.6: @ %middle.block
36; CHECK-NEXT: cmp r3, r1
37; CHECK-NEXT: it eq
38; CHECK-NEXT: popeq {r7, pc}
39; CHECK-NEXT: .LBB0_7: @ %for.body.preheader1
40; CHECK-NEXT: sub.w lr, r1, r3
41; CHECK-NEXT: add.w r1, r12, r3, lsl #2
42; CHECK-NEXT: dls lr, lr
43; CHECK-NEXT: .LBB0_8: @ %for.body
44; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
45; CHECK-NEXT: ldr r2, [r1], #4
46; CHECK-NEXT: add r0, r2
47; CHECK-NEXT: le lr, .LBB0_8
48; CHECK-NEXT: .LBB0_9: @ %for.cond.cleanup
49; CHECK-NEXT: pop {r7, pc}
50entry:
51 %cmp6 = icmp sgt i32 %n, 0
52 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
53
54for.body.preheader: ; preds = %entry
55 %min.iters.check = icmp ult i32 %n, 4
56 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
57
58vector.ph: ; preds = %for.body.preheader
59 %n.vec = and i32 %n, -4
60 br label %vector.body
61
62vector.body: ; preds = %vector.body, %vector.ph
63 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
64 %vec.phi = phi i32 [ 0, %vector.ph ], [ %3, %vector.body ]
65 %0 = getelementptr inbounds i32, i32* %x, i32 %index
66 %1 = bitcast i32* %0 to <4 x i32>*
67 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
68 %2 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %wide.load)
69 %3 = add i32 %2, %vec.phi
70 %index.next = add i32 %index, 4
71 %4 = icmp eq i32 %index.next, %n.vec
72 br i1 %4, label %middle.block, label %vector.body
73
74middle.block: ; preds = %vector.body
75 %cmp.n = icmp eq i32 %n.vec, %n
76 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
77
78for.body.preheader1: ; preds = %middle.block, %for.body.preheader
79 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
80 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %3, %middle.block ]
81 br label %for.body
82
83for.body: ; preds = %for.body.preheader1, %for.body
84 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
85 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
86 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
87 %5 = load i32, i32* %arrayidx, align 4
88 %add = add nsw i32 %5, %r.07
89 %inc = add nuw nsw i32 %i.08, 1
90 %exitcond = icmp eq i32 %inc, %n
91 br i1 %exitcond, label %for.cond.cleanup, label %for.body
92
93for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
94 %r.0.lcssa = phi i32 [ 0, %entry ], [ %3, %middle.block ], [ %add, %for.body ]
95 ret i32 %r.0.lcssa
96}
97
98define i32 @mul_i32(i32* nocapture readonly %x, i32 %n) {
99; CHECK-LABEL: mul_i32:
100; CHECK: @ %bb.0: @ %entry
101; CHECK-NEXT: .save {r7, lr}
102; CHECK-NEXT: push {r7, lr}
103; CHECK-NEXT: movs r2, #1
104; CHECK-NEXT: cmp r1, #1
105; CHECK-NEXT: blt .LBB1_8
106; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
107; CHECK-NEXT: cmp r1, #4
108; CHECK-NEXT: bhs .LBB1_3
109; CHECK-NEXT: @ %bb.2:
110; CHECK-NEXT: mov.w r12, #0
111; CHECK-NEXT: b .LBB1_6
112; CHECK-NEXT: .LBB1_3: @ %vector.ph
113; CHECK-NEXT: bic r12, r1, #3
114; CHECK-NEXT: vmov.i32 q0, #0x1
115; CHECK-NEXT: sub.w r3, r12, #4
116; CHECK-NEXT: add.w lr, r2, r3, lsr #2
117; CHECK-NEXT: mov r2, r0
118; CHECK-NEXT: dls lr, lr
119; CHECK-NEXT: .LBB1_4: @ %vector.body
120; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
121; CHECK-NEXT: vldrw.u32 q1, [r2], #16
122; CHECK-NEXT: vmul.i32 q0, q1, q0
123; CHECK-NEXT: le lr, .LBB1_4
124; CHECK-NEXT: @ %bb.5: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]125; CHECK-NEXT: vmov r2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]126; CHECK-NEXT: cmp r12, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]127; CHECK-NEXT: vmov r3, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]128; CHECK-NEXT: mul lr, r3, r2
129; CHECK-NEXT: vmov r3, s1
130; CHECK-NEXT: vmov r2, s0
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]131; CHECK-NEXT: mul r2, r3, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]132; CHECK-NEXT: mul r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]133; CHECK-NEXT: beq .LBB1_8
134; CHECK-NEXT: .LBB1_6: @ %for.body.preheader1
135; CHECK-NEXT: sub.w lr, r1, r12
136; CHECK-NEXT: add.w r0, r0, r12, lsl #2
137; CHECK-NEXT: dls lr, lr
138; CHECK-NEXT: .LBB1_7: @ %for.body
139; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
140; CHECK-NEXT: ldr r1, [r0], #4
141; CHECK-NEXT: muls r2, r1, r2
142; CHECK-NEXT: le lr, .LBB1_7
143; CHECK-NEXT: .LBB1_8: @ %for.cond.cleanup
144; CHECK-NEXT: mov r0, r2
145; CHECK-NEXT: pop {r7, pc}
146entry:
147 %cmp6 = icmp sgt i32 %n, 0
148 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
149
150for.body.preheader: ; preds = %entry
151 %min.iters.check = icmp ult i32 %n, 4
152 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
153
154vector.ph: ; preds = %for.body.preheader
155 %n.vec = and i32 %n, -4
156 br label %vector.body
157
158vector.body: ; preds = %vector.body, %vector.ph
159 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
160 %vec.phi = phi <4 x i32> [ <i32 1, i32 1, i32 1, i32 1>, %vector.ph ], [ %2, %vector.body ]
161 %0 = getelementptr inbounds i32, i32* %x, i32 %index
162 %1 = bitcast i32* %0 to <4 x i32>*
163 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
164 %2 = mul <4 x i32> %wide.load, %vec.phi
165 %index.next = add i32 %index, 4
166 %3 = icmp eq i32 %index.next, %n.vec
167 br i1 %3, label %middle.block, label %vector.body
168
169middle.block: ; preds = %vector.body
170 %4 = call i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32> %2)
171 %cmp.n = icmp eq i32 %n.vec, %n
172 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
173
174for.body.preheader1: ; preds = %middle.block, %for.body.preheader
175 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
176 %r.07.ph = phi i32 [ 1, %for.body.preheader ], [ %4, %middle.block ]
177 br label %for.body
178
179for.body: ; preds = %for.body.preheader1, %for.body
180 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
181 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
182 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
183 %5 = load i32, i32* %arrayidx, align 4
184 %add = mul nsw i32 %5, %r.07
185 %inc = add nuw nsw i32 %i.08, 1
186 %exitcond = icmp eq i32 %inc, %n
187 br i1 %exitcond, label %for.cond.cleanup, label %for.body
188
189for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
190 %r.0.lcssa = phi i32 [ 1, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
191 ret i32 %r.0.lcssa
192}
193
194define i32 @and_i32(i32* nocapture readonly %x, i32 %n) {
195; CHECK-LABEL: and_i32:
196; CHECK: @ %bb.0: @ %entry
197; CHECK-NEXT: .save {r7, lr}
198; CHECK-NEXT: push {r7, lr}
199; CHECK-NEXT: cmp r1, #1
200; CHECK-NEXT: blt .LBB2_3
201; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
202; CHECK-NEXT: cmp r1, #4
203; CHECK-NEXT: bhs .LBB2_4
204; CHECK-NEXT: @ %bb.2:
205; CHECK-NEXT: mov.w r2, #-1
206; CHECK-NEXT: movs r3, #0
207; CHECK-NEXT: b .LBB2_7
208; CHECK-NEXT: .LBB2_3:
209; CHECK-NEXT: mov.w r2, #-1
210; CHECK-NEXT: b .LBB2_9
211; CHECK-NEXT: .LBB2_4: @ %vector.ph
212; CHECK-NEXT: bic r3, r1, #3
213; CHECK-NEXT: movs r2, #1
214; CHECK-NEXT: sub.w r12, r3, #4
215; CHECK-NEXT: vmov.i8 q0, #0xff
216; CHECK-NEXT: add.w lr, r2, r12, lsr #2
217; CHECK-NEXT: mov r2, r0
218; CHECK-NEXT: dls lr, lr
219; CHECK-NEXT: .LBB2_5: @ %vector.body
220; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
221; CHECK-NEXT: vldrw.u32 q1, [r2], #16
222; CHECK-NEXT: vand q0, q1, q0
223; CHECK-NEXT: le lr, .LBB2_5
224; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]225; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]226; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]227; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]228; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]229; CHECK-NEXT: and.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]230; CHECK-NEXT: vmov r2, s0
231; CHECK-NEXT: and.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]232; CHECK-NEXT: and.w r2, r2, r12
233; CHECK-NEXT: beq .LBB2_9
234; CHECK-NEXT: .LBB2_7: @ %for.body.preheader1
235; CHECK-NEXT: sub.w lr, r1, r3
236; CHECK-NEXT: add.w r0, r0, r3, lsl #2
237; CHECK-NEXT: dls lr, lr
238; CHECK-NEXT: .LBB2_8: @ %for.body
239; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
240; CHECK-NEXT: ldr r1, [r0], #4
241; CHECK-NEXT: ands r2, r1
242; CHECK-NEXT: le lr, .LBB2_8
243; CHECK-NEXT: .LBB2_9: @ %for.cond.cleanup
244; CHECK-NEXT: mov r0, r2
245; CHECK-NEXT: pop {r7, pc}
246entry:
247 %cmp6 = icmp sgt i32 %n, 0
248 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
249
250for.body.preheader: ; preds = %entry
251 %min.iters.check = icmp ult i32 %n, 4
252 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
253
254vector.ph: ; preds = %for.body.preheader
255 %n.vec = and i32 %n, -4
256 br label %vector.body
257
258vector.body: ; preds = %vector.body, %vector.ph
259 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
260 %vec.phi = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %vector.ph ], [ %2, %vector.body ]
261 %0 = getelementptr inbounds i32, i32* %x, i32 %index
262 %1 = bitcast i32* %0 to <4 x i32>*
263 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
264 %2 = and <4 x i32> %wide.load, %vec.phi
265 %index.next = add i32 %index, 4
266 %3 = icmp eq i32 %index.next, %n.vec
267 br i1 %3, label %middle.block, label %vector.body
268
269middle.block: ; preds = %vector.body
270 %4 = call i32 @llvm.experimental.vector.reduce.and.v4i32(<4 x i32> %2)
271 %cmp.n = icmp eq i32 %n.vec, %n
272 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
273
274for.body.preheader1: ; preds = %middle.block, %for.body.preheader
275 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
276 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %4, %middle.block ]
277 br label %for.body
278
279for.body: ; preds = %for.body.preheader1, %for.body
280 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
281 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
282 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
283 %5 = load i32, i32* %arrayidx, align 4
284 %add = and i32 %5, %r.07
285 %inc = add nuw nsw i32 %i.08, 1
286 %exitcond = icmp eq i32 %inc, %n
287 br i1 %exitcond, label %for.cond.cleanup, label %for.body
288
289for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
290 %r.0.lcssa = phi i32 [ -1, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
291 ret i32 %r.0.lcssa
292}
293
294define i32 @or_i32(i32* nocapture readonly %x, i32 %n) {
295; CHECK-LABEL: or_i32:
296; CHECK: @ %bb.0: @ %entry
297; CHECK-NEXT: .save {r7, lr}
298; CHECK-NEXT: push {r7, lr}
299; CHECK-NEXT: cmp r1, #1
300; CHECK-NEXT: blt .LBB3_3
301; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
302; CHECK-NEXT: cmp r1, #4
303; CHECK-NEXT: bhs .LBB3_4
304; CHECK-NEXT: @ %bb.2:
305; CHECK-NEXT: movs r3, #0
306; CHECK-NEXT: movs r2, #0
307; CHECK-NEXT: b .LBB3_7
308; CHECK-NEXT: .LBB3_3:
309; CHECK-NEXT: movs r2, #0
310; CHECK-NEXT: b .LBB3_9
311; CHECK-NEXT: .LBB3_4: @ %vector.ph
312; CHECK-NEXT: bic r3, r1, #3
313; CHECK-NEXT: movs r2, #1
314; CHECK-NEXT: sub.w r12, r3, #4
315; CHECK-NEXT: vmov.i32 q0, #0x0
316; CHECK-NEXT: add.w lr, r2, r12, lsr #2
317; CHECK-NEXT: mov r2, r0
318; CHECK-NEXT: dls lr, lr
319; CHECK-NEXT: .LBB3_5: @ %vector.body
320; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
321; CHECK-NEXT: vldrw.u32 q1, [r2], #16
322; CHECK-NEXT: vorr q0, q1, q0
323; CHECK-NEXT: le lr, .LBB3_5
324; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]325; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]326; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]327; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]328; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]329; CHECK-NEXT: orr.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]330; CHECK-NEXT: vmov r2, s0
331; CHECK-NEXT: orr.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]332; CHECK-NEXT: orr.w r2, r2, r12
333; CHECK-NEXT: beq .LBB3_9
334; CHECK-NEXT: .LBB3_7: @ %for.body.preheader1
335; CHECK-NEXT: sub.w lr, r1, r3
336; CHECK-NEXT: add.w r0, r0, r3, lsl #2
337; CHECK-NEXT: dls lr, lr
338; CHECK-NEXT: .LBB3_8: @ %for.body
339; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
340; CHECK-NEXT: ldr r1, [r0], #4
341; CHECK-NEXT: orrs r2, r1
342; CHECK-NEXT: le lr, .LBB3_8
343; CHECK-NEXT: .LBB3_9: @ %for.cond.cleanup
344; CHECK-NEXT: mov r0, r2
345; CHECK-NEXT: pop {r7, pc}
346entry:
347 %cmp6 = icmp sgt i32 %n, 0
348 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
349
350for.body.preheader: ; preds = %entry
351 %min.iters.check = icmp ult i32 %n, 4
352 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
353
354vector.ph: ; preds = %for.body.preheader
355 %n.vec = and i32 %n, -4
356 br label %vector.body
357
358vector.body: ; preds = %vector.body, %vector.ph
359 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
360 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
361 %0 = getelementptr inbounds i32, i32* %x, i32 %index
362 %1 = bitcast i32* %0 to <4 x i32>*
363 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
364 %2 = or <4 x i32> %wide.load, %vec.phi
365 %index.next = add i32 %index, 4
366 %3 = icmp eq i32 %index.next, %n.vec
367 br i1 %3, label %middle.block, label %vector.body
368
369middle.block: ; preds = %vector.body
370 %4 = call i32 @llvm.experimental.vector.reduce.or.v4i32(<4 x i32> %2)
371 %cmp.n = icmp eq i32 %n.vec, %n
372 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
373
374for.body.preheader1: ; preds = %middle.block, %for.body.preheader
375 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
376 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %4, %middle.block ]
377 br label %for.body
378
379for.body: ; preds = %for.body.preheader1, %for.body
380 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
381 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
382 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
383 %5 = load i32, i32* %arrayidx, align 4
384 %add = or i32 %5, %r.07
385 %inc = add nuw nsw i32 %i.08, 1
386 %exitcond = icmp eq i32 %inc, %n
387 br i1 %exitcond, label %for.cond.cleanup, label %for.body
388
389for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
390 %r.0.lcssa = phi i32 [ 0, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
391 ret i32 %r.0.lcssa
392}
393
394define i32 @xor_i32(i32* nocapture readonly %x, i32 %n) {
395; CHECK-LABEL: xor_i32:
396; CHECK: @ %bb.0: @ %entry
397; CHECK-NEXT: .save {r7, lr}
398; CHECK-NEXT: push {r7, lr}
399; CHECK-NEXT: cmp r1, #1
400; CHECK-NEXT: blt .LBB4_3
401; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
402; CHECK-NEXT: cmp r1, #4
403; CHECK-NEXT: bhs .LBB4_4
404; CHECK-NEXT: @ %bb.2:
405; CHECK-NEXT: movs r3, #0
406; CHECK-NEXT: movs r2, #0
407; CHECK-NEXT: b .LBB4_7
408; CHECK-NEXT: .LBB4_3:
409; CHECK-NEXT: movs r2, #0
410; CHECK-NEXT: b .LBB4_9
411; CHECK-NEXT: .LBB4_4: @ %vector.ph
412; CHECK-NEXT: bic r3, r1, #3
413; CHECK-NEXT: movs r2, #1
414; CHECK-NEXT: sub.w r12, r3, #4
415; CHECK-NEXT: vmov.i32 q0, #0x0
416; CHECK-NEXT: add.w lr, r2, r12, lsr #2
417; CHECK-NEXT: mov r2, r0
418; CHECK-NEXT: dls lr, lr
419; CHECK-NEXT: .LBB4_5: @ %vector.body
420; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
421; CHECK-NEXT: vldrw.u32 q1, [r2], #16
422; CHECK-NEXT: veor q0, q1, q0
423; CHECK-NEXT: le lr, .LBB4_5
424; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]425; CHECK-NEXT: vmov r12, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]426; CHECK-NEXT: cmp r3, r1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]427; CHECK-NEXT: vmov r2, s2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]428; CHECK-NEXT: vmov lr, s1
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]429; CHECK-NEXT: eor.w r12, r12, r2
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]430; CHECK-NEXT: vmov r2, s0
431; CHECK-NEXT: eor.w r2, r2, lr
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]432; CHECK-NEXT: eor.w r2, r2, r12
433; CHECK-NEXT: beq .LBB4_9
434; CHECK-NEXT: .LBB4_7: @ %for.body.preheader1
435; CHECK-NEXT: sub.w lr, r1, r3
436; CHECK-NEXT: add.w r0, r0, r3, lsl #2
437; CHECK-NEXT: dls lr, lr
438; CHECK-NEXT: .LBB4_8: @ %for.body
439; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
440; CHECK-NEXT: ldr r1, [r0], #4
441; CHECK-NEXT: eors r2, r1
442; CHECK-NEXT: le lr, .LBB4_8
443; CHECK-NEXT: .LBB4_9: @ %for.cond.cleanup
444; CHECK-NEXT: mov r0, r2
445; CHECK-NEXT: pop {r7, pc}
446entry:
447 %cmp6 = icmp sgt i32 %n, 0
448 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
449
450for.body.preheader: ; preds = %entry
451 %min.iters.check = icmp ult i32 %n, 4
452 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
453
454vector.ph: ; preds = %for.body.preheader
455 %n.vec = and i32 %n, -4
456 br label %vector.body
457
458vector.body: ; preds = %vector.body, %vector.ph
459 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
460 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
461 %0 = getelementptr inbounds i32, i32* %x, i32 %index
462 %1 = bitcast i32* %0 to <4 x i32>*
463 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
464 %2 = xor <4 x i32> %wide.load, %vec.phi
465 %index.next = add i32 %index, 4
466 %3 = icmp eq i32 %index.next, %n.vec
467 br i1 %3, label %middle.block, label %vector.body
468
469middle.block: ; preds = %vector.body
470 %4 = call i32 @llvm.experimental.vector.reduce.xor.v4i32(<4 x i32> %2)
471 %cmp.n = icmp eq i32 %n.vec, %n
472 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
473
474for.body.preheader1: ; preds = %middle.block, %for.body.preheader
475 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
476 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %4, %middle.block ]
477 br label %for.body
478
479for.body: ; preds = %for.body.preheader1, %for.body
480 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
481 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
482 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
483 %5 = load i32, i32* %arrayidx, align 4
484 %add = xor i32 %5, %r.07
485 %inc = add nuw nsw i32 %i.08, 1
486 %exitcond = icmp eq i32 %inc, %n
487 br i1 %exitcond, label %for.cond.cleanup, label %for.body
488
489for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
490 %r.0.lcssa = phi i32 [ 0, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
491 ret i32 %r.0.lcssa
492}
493
494define float @fadd_f32(float* nocapture readonly %x, i32 %n) {
495; CHECK-LABEL: fadd_f32:
496; CHECK: @ %bb.0: @ %entry
497; CHECK-NEXT: .save {r7, lr}
498; CHECK-NEXT: push {r7, lr}
499; CHECK-NEXT: cmp r1, #1
500; CHECK-NEXT: blt .LBB5_3
501; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
502; CHECK-NEXT: cmp r1, #4
503; CHECK-NEXT: bhs .LBB5_4
504; CHECK-NEXT: @ %bb.2:
505; CHECK-NEXT: vldr s0, .LCPI5_0
506; CHECK-NEXT: movs r2, #0
507; CHECK-NEXT: b .LBB5_7
508; CHECK-NEXT: .LBB5_3:
509; CHECK-NEXT: vldr s0, .LCPI5_0
510; CHECK-NEXT: b .LBB5_9
511; CHECK-NEXT: .LBB5_4: @ %vector.ph
512; CHECK-NEXT: bic r2, r1, #3
513; CHECK-NEXT: movs r3, #1
514; CHECK-NEXT: sub.w r12, r2, #4
515; CHECK-NEXT: vmov.i32 q0, #0x0
516; CHECK-NEXT: add.w lr, r3, r12, lsr #2
517; CHECK-NEXT: mov r3, r0
518; CHECK-NEXT: dls lr, lr
519; CHECK-NEXT: .LBB5_5: @ %vector.body
520; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
521; CHECK-NEXT: vldrw.u32 q1, [r3], #16
522; CHECK-NEXT: vadd.f32 q0, q1, q0
523; CHECK-NEXT: le lr, .LBB5_5
524; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]525; CHECK-NEXT: vadd.f32 s4, s2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]526; CHECK-NEXT: cmp r2, r1
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]527; CHECK-NEXT: vadd.f32 s0, s0, s1
528; CHECK-NEXT: vadd.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]529; CHECK-NEXT: beq .LBB5_9
530; CHECK-NEXT: .LBB5_7: @ %for.body.preheader1
531; CHECK-NEXT: sub.w lr, r1, r2
532; CHECK-NEXT: add.w r0, r0, r2, lsl #2
533; CHECK-NEXT: dls lr, lr
534; CHECK-NEXT: .LBB5_8: @ %for.body
535; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
536; CHECK-NEXT: vldr s2, [r0]
537; CHECK-NEXT: adds r0, #4
538; CHECK-NEXT: vadd.f32 s0, s2, s0
539; CHECK-NEXT: le lr, .LBB5_8
540; CHECK-NEXT: .LBB5_9: @ %for.cond.cleanup
541; CHECK-NEXT: vmov r0, s0
542; CHECK-NEXT: pop {r7, pc}
543; CHECK-NEXT: .p2align 2
544; CHECK-NEXT: @ %bb.10:
545; CHECK-NEXT: .LCPI5_0:
546; CHECK-NEXT: .long 0x00000000 @ float 0
547entry:
548 %cmp6 = icmp sgt i32 %n, 0
549 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
550
551for.body.preheader: ; preds = %entry
552 %min.iters.check = icmp ult i32 %n, 4
553 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
554
555vector.ph: ; preds = %for.body.preheader
556 %n.vec = and i32 %n, -4
557 br label %vector.body
558
559vector.body: ; preds = %vector.body, %vector.ph
560 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
561 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %2, %vector.body ]
562 %0 = getelementptr inbounds float, float* %x, i32 %index
563 %1 = bitcast float* %0 to <4 x float>*
564 %wide.load = load <4 x float>, <4 x float>* %1, align 4
565 %2 = fadd fast <4 x float> %wide.load, %vec.phi
566 %index.next = add i32 %index, 4
567 %3 = icmp eq i32 %index.next, %n.vec
568 br i1 %3, label %middle.block, label %vector.body
569
570middle.block: ; preds = %vector.body
571 %4 = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float 0.000000e+00, <4 x float> %2)
572 %cmp.n = icmp eq i32 %n.vec, %n
573 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
574
575for.body.preheader1: ; preds = %middle.block, %for.body.preheader
576 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
577 %r.07.ph = phi float [ 0.000000e+00, %for.body.preheader ], [ %4, %middle.block ]
578 br label %for.body
579
580for.body: ; preds = %for.body.preheader1, %for.body
581 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
582 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
583 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
584 %5 = load float, float* %arrayidx, align 4
585 %add = fadd fast float %5, %r.07
586 %inc = add nuw nsw i32 %i.08, 1
587 %exitcond = icmp eq i32 %inc, %n
588 br i1 %exitcond, label %for.cond.cleanup, label %for.body
589
590for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
591 %r.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
592 ret float %r.0.lcssa
593}
594
595define float @fmul_f32(float* nocapture readonly %x, i32 %n) {
596; CHECK-LABEL: fmul_f32:
597; CHECK: @ %bb.0: @ %entry
598; CHECK-NEXT: .save {r7, lr}
599; CHECK-NEXT: push {r7, lr}
600; CHECK-NEXT: cmp r1, #1
601; CHECK-NEXT: blt .LBB6_3
602; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
603; CHECK-NEXT: cmp r1, #4
604; CHECK-NEXT: bhs .LBB6_4
605; CHECK-NEXT: @ %bb.2:
606; CHECK-NEXT: vmov.f32 s0, #1.000000e+00
607; CHECK-NEXT: movs r2, #0
608; CHECK-NEXT: b .LBB6_7
609; CHECK-NEXT: .LBB6_3:
610; CHECK-NEXT: vmov.f32 s0, #1.000000e+00
611; CHECK-NEXT: b .LBB6_9
612; CHECK-NEXT: .LBB6_4: @ %vector.ph
613; CHECK-NEXT: bic r2, r1, #3
614; CHECK-NEXT: movs r3, #1
615; CHECK-NEXT: sub.w r12, r2, #4
616; CHECK-NEXT: vmov.f32 q0, #1.000000e+00
617; CHECK-NEXT: add.w lr, r3, r12, lsr #2
618; CHECK-NEXT: mov r3, r0
619; CHECK-NEXT: dls lr, lr
620; CHECK-NEXT: .LBB6_5: @ %vector.body
621; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
622; CHECK-NEXT: vldrw.u32 q1, [r3], #16
623; CHECK-NEXT: vmul.f32 q0, q1, q0
624; CHECK-NEXT: le lr, .LBB6_5
625; CHECK-NEXT: @ %bb.6: @ %middle.block
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]626; CHECK-NEXT: vmul.f32 s4, s2, s3
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]627; CHECK-NEXT: cmp r2, r1
David Greendeb72ce2020-06-29 13:53:19 +0100[diff] [blame]628; CHECK-NEXT: vmul.f32 s0, s0, s1
629; CHECK-NEXT: vmul.f32 s0, s0, s4
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]630; CHECK-NEXT: beq .LBB6_9
631; CHECK-NEXT: .LBB6_7: @ %for.body.preheader1
632; CHECK-NEXT: sub.w lr, r1, r2
633; CHECK-NEXT: add.w r0, r0, r2, lsl #2
634; CHECK-NEXT: dls lr, lr
635; CHECK-NEXT: .LBB6_8: @ %for.body
636; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
637; CHECK-NEXT: vldr s2, [r0]
638; CHECK-NEXT: adds r0, #4
639; CHECK-NEXT: vmul.f32 s0, s2, s0
640; CHECK-NEXT: le lr, .LBB6_8
641; CHECK-NEXT: .LBB6_9: @ %for.cond.cleanup
642; CHECK-NEXT: vmov r0, s0
643; CHECK-NEXT: pop {r7, pc}
644entry:
645 %cmp6 = icmp sgt i32 %n, 0
646 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
647
648for.body.preheader: ; preds = %entry
649 %min.iters.check = icmp ult i32 %n, 4
650 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
651
652vector.ph: ; preds = %for.body.preheader
653 %n.vec = and i32 %n, -4
654 br label %vector.body
655
656vector.body: ; preds = %vector.body, %vector.ph
657 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
658 %vec.phi = phi <4 x float> [ <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %vector.ph ], [ %2, %vector.body ]
659 %0 = getelementptr inbounds float, float* %x, i32 %index
660 %1 = bitcast float* %0 to <4 x float>*
661 %wide.load = load <4 x float>, <4 x float>* %1, align 4
662 %2 = fmul fast <4 x float> %wide.load, %vec.phi
663 %index.next = add i32 %index, 4
664 %3 = icmp eq i32 %index.next, %n.vec
665 br i1 %3, label %middle.block, label %vector.body
666
667middle.block: ; preds = %vector.body
668 %4 = call fast float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float 1.000000e+00, <4 x float> %2)
669 %cmp.n = icmp eq i32 %n.vec, %n
670 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
671
672for.body.preheader1: ; preds = %middle.block, %for.body.preheader
673 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
674 %r.07.ph = phi float [ 1.000000e+00, %for.body.preheader ], [ %4, %middle.block ]
675 br label %for.body
676
677for.body: ; preds = %for.body.preheader1, %for.body
678 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
679 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
680 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
681 %5 = load float, float* %arrayidx, align 4
682 %add = fmul fast float %5, %r.07
683 %inc = add nuw nsw i32 %i.08, 1
684 %exitcond = icmp eq i32 %inc, %n
685 br i1 %exitcond, label %for.cond.cleanup, label %for.body
686
687for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
688 %r.0.lcssa = phi float [ 1.000000e+00, %entry ], [ %4, %middle.block ], [ %add, %for.body ]
689 ret float %r.0.lcssa
690}
691
692define i32 @smin_i32(i32* nocapture readonly %x, i32 %n) {
693; CHECK-LABEL: smin_i32:
694; CHECK: @ %bb.0: @ %entry
695; CHECK-NEXT: .save {r7, lr}
696; CHECK-NEXT: push {r7, lr}
697; CHECK-NEXT: cmp r1, #1
698; CHECK-NEXT: blt .LBB7_3
699; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
700; CHECK-NEXT: cmp r1, #4
701; CHECK-NEXT: bhs .LBB7_4
702; CHECK-NEXT: @ %bb.2:
703; CHECK-NEXT: mvn r2, #-2147483648
704; CHECK-NEXT: movs r3, #0
705; CHECK-NEXT: b .LBB7_7
706; CHECK-NEXT: .LBB7_3:
707; CHECK-NEXT: mvn r2, #-2147483648
708; CHECK-NEXT: b .LBB7_9
709; CHECK-NEXT: .LBB7_4: @ %vector.ph
710; CHECK-NEXT: bic r3, r1, #3
711; CHECK-NEXT: movs r2, #1
712; CHECK-NEXT: sub.w r12, r3, #4
713; CHECK-NEXT: vmvn.i32 q0, #0x80000000
714; CHECK-NEXT: add.w lr, r2, r12, lsr #2
715; CHECK-NEXT: mov r2, r0
716; CHECK-NEXT: dls lr, lr
717; CHECK-NEXT: .LBB7_5: @ %vector.body
718; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
719; CHECK-NEXT: vldrw.u32 q1, [r2], #16
720; CHECK-NEXT: vmin.s32 q0, q0, q1
721; CHECK-NEXT: le lr, .LBB7_5
722; CHECK-NEXT: @ %bb.6: @ %middle.block
723; CHECK-NEXT: mvn r2, #-2147483648
724; CHECK-NEXT: cmp r3, r1
725; CHECK-NEXT: vminv.s32 r2, q0
726; CHECK-NEXT: beq .LBB7_9
727; CHECK-NEXT: .LBB7_7: @ %for.body.preheader1
728; CHECK-NEXT: sub.w lr, r1, r3
729; CHECK-NEXT: add.w r0, r0, r3, lsl #2
730; CHECK-NEXT: dls lr, lr
731; CHECK-NEXT: .LBB7_8: @ %for.body
732; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
733; CHECK-NEXT: ldr r1, [r0], #4
734; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]735; CHECK-NEXT: csel r2, r2, r1, lt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]736; CHECK-NEXT: le lr, .LBB7_8
737; CHECK-NEXT: .LBB7_9: @ %for.cond.cleanup
738; CHECK-NEXT: mov r0, r2
739; CHECK-NEXT: pop {r7, pc}
740entry:
741 %cmp6 = icmp sgt i32 %n, 0
742 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
743
744for.body.preheader: ; preds = %entry
745 %min.iters.check = icmp ult i32 %n, 4
746 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
747
748vector.ph: ; preds = %for.body.preheader
749 %n.vec = and i32 %n, -4
750 br label %vector.body
751
752vector.body: ; preds = %vector.body, %vector.ph
753 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
754 %vec.phi = phi <4 x i32> [ <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, %vector.ph ], [ %3, %vector.body ]
755 %0 = getelementptr inbounds i32, i32* %x, i32 %index
756 %1 = bitcast i32* %0 to <4 x i32>*
757 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
758 %2 = icmp slt <4 x i32> %vec.phi, %wide.load
759 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
760 %index.next = add i32 %index, 4
761 %4 = icmp eq i32 %index.next, %n.vec
762 br i1 %4, label %middle.block, label %vector.body
763
764middle.block: ; preds = %vector.body
765 %5 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> %3)
766 %cmp.n = icmp eq i32 %n.vec, %n
767 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
768
769for.body.preheader1: ; preds = %middle.block, %for.body.preheader
770 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
771 %r.07.ph = phi i32 [ 2147483647, %for.body.preheader ], [ %5, %middle.block ]
772 br label %for.body
773
774for.body: ; preds = %for.body.preheader1, %for.body
775 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
776 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
777 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
778 %6 = load i32, i32* %arrayidx, align 4
779 %c = icmp slt i32 %r.07, %6
780 %add = select i1 %c, i32 %r.07, i32 %6
781 %inc = add nuw nsw i32 %i.08, 1
782 %exitcond = icmp eq i32 %inc, %n
783 br i1 %exitcond, label %for.cond.cleanup, label %for.body
784
785for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
786 %r.0.lcssa = phi i32 [ 2147483647, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
787 ret i32 %r.0.lcssa
788}
789
790define i32 @smin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
791; CHECK-LABEL: smin_i32_inloop:
792; CHECK: @ %bb.0: @ %entry
793; CHECK-NEXT: .save {r4, lr}
794; CHECK-NEXT: push {r4, lr}
795; CHECK-NEXT: cmp r1, #1
796; CHECK-NEXT: blt .LBB8_3
797; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
798; CHECK-NEXT: mov r12, r0
799; CHECK-NEXT: mvn r0, #-2147483648
800; CHECK-NEXT: cmp r1, #4
801; CHECK-NEXT: bhs .LBB8_4
802; CHECK-NEXT: @ %bb.2:
803; CHECK-NEXT: movs r3, #0
804; CHECK-NEXT: b .LBB8_7
805; CHECK-NEXT: .LBB8_3:
806; CHECK-NEXT: mvn r0, #-2147483648
807; CHECK-NEXT: b .LBB8_9
808; CHECK-NEXT: .LBB8_4: @ %vector.ph
809; CHECK-NEXT: bic r3, r1, #3
810; CHECK-NEXT: movs r2, #1
811; CHECK-NEXT: sub.w lr, r3, #4
812; CHECK-NEXT: add.w lr, r2, lr, lsr #2
813; CHECK-NEXT: mov r2, r12
814; CHECK-NEXT: dls lr, lr
815; CHECK-NEXT: .LBB8_5: @ %vector.body
816; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
817; CHECK-NEXT: vldrw.u32 q0, [r2], #16
818; CHECK-NEXT: mvn r4, #-2147483648
819; CHECK-NEXT: vminv.s32 r4, q0
820; CHECK-NEXT: cmp r0, r4
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]821; CHECK-NEXT: csel r0, r0, r4, lt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]822; CHECK-NEXT: le lr, .LBB8_5
823; CHECK-NEXT: @ %bb.6: @ %middle.block
824; CHECK-NEXT: cmp r3, r1
825; CHECK-NEXT: it eq
826; CHECK-NEXT: popeq {r4, pc}
827; CHECK-NEXT: .LBB8_7: @ %for.body.preheader1
828; CHECK-NEXT: sub.w lr, r1, r3
829; CHECK-NEXT: add.w r1, r12, r3, lsl #2
830; CHECK-NEXT: dls lr, lr
831; CHECK-NEXT: .LBB8_8: @ %for.body
832; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
833; CHECK-NEXT: ldr r2, [r1], #4
834; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]835; CHECK-NEXT: csel r0, r0, r2, lt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]836; CHECK-NEXT: le lr, .LBB8_8
837; CHECK-NEXT: .LBB8_9: @ %for.cond.cleanup
838; CHECK-NEXT: pop {r4, pc}
839entry:
840 %cmp6 = icmp sgt i32 %n, 0
841 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
842
843for.body.preheader: ; preds = %entry
844 %min.iters.check = icmp ult i32 %n, 4
845 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
846
847vector.ph: ; preds = %for.body.preheader
848 %n.vec = and i32 %n, -4
849 br label %vector.body
850
851vector.body: ; preds = %vector.body, %vector.ph
852 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
853 %vec.phi = phi i32 [ 2147483647, %vector.ph ], [ %3, %vector.body ]
854 %0 = getelementptr inbounds i32, i32* %x, i32 %index
855 %1 = bitcast i32* %0 to <4 x i32>*
856 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
857 %l5 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> %wide.load)
858 %2 = icmp slt i32 %vec.phi, %l5
859 %3 = select i1 %2, i32 %vec.phi, i32 %l5
860 %index.next = add i32 %index, 4
861 %4 = icmp eq i32 %index.next, %n.vec
862 br i1 %4, label %middle.block, label %vector.body
863
864middle.block: ; preds = %vector.body
865 %5 = phi i32 [ %3, %vector.body ]
866 %cmp.n = icmp eq i32 %n.vec, %n
867 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
868
869for.body.preheader1: ; preds = %middle.block, %for.body.preheader
870 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
871 %r.07.ph = phi i32 [ 2147483647, %for.body.preheader ], [ %5, %middle.block ]
872 br label %for.body
873
874for.body: ; preds = %for.body.preheader1, %for.body
875 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
876 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
877 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
878 %6 = load i32, i32* %arrayidx, align 4
879 %c = icmp slt i32 %r.07, %6
880 %add = select i1 %c, i32 %r.07, i32 %6
881 %inc = add nuw nsw i32 %i.08, 1
882 %exitcond = icmp eq i32 %inc, %n
883 br i1 %exitcond, label %for.cond.cleanup, label %for.body
884
885for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
886 %r.0.lcssa = phi i32 [ 2147483647, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
887 ret i32 %r.0.lcssa
888}
889
890define i32 @smax_i32(i32* nocapture readonly %x, i32 %n) {
891; CHECK-LABEL: smax_i32:
892; CHECK: @ %bb.0: @ %entry
893; CHECK-NEXT: .save {r7, lr}
894; CHECK-NEXT: push {r7, lr}
895; CHECK-NEXT: cmp r1, #1
896; CHECK-NEXT: blt .LBB9_3
897; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
898; CHECK-NEXT: cmp r1, #4
899; CHECK-NEXT: bhs .LBB9_4
900; CHECK-NEXT: @ %bb.2:
901; CHECK-NEXT: mov.w r2, #-2147483648
902; CHECK-NEXT: movs r3, #0
903; CHECK-NEXT: b .LBB9_7
904; CHECK-NEXT: .LBB9_3:
905; CHECK-NEXT: mov.w r2, #-2147483648
906; CHECK-NEXT: b .LBB9_9
907; CHECK-NEXT: .LBB9_4: @ %vector.ph
908; CHECK-NEXT: bic r3, r1, #3
909; CHECK-NEXT: movs r2, #1
910; CHECK-NEXT: sub.w r12, r3, #4
911; CHECK-NEXT: vmov.i32 q0, #0x80000000
912; CHECK-NEXT: add.w lr, r2, r12, lsr #2
913; CHECK-NEXT: mov r2, r0
914; CHECK-NEXT: dls lr, lr
915; CHECK-NEXT: .LBB9_5: @ %vector.body
916; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
917; CHECK-NEXT: vldrw.u32 q1, [r2], #16
918; CHECK-NEXT: vmax.s32 q0, q0, q1
919; CHECK-NEXT: le lr, .LBB9_5
920; CHECK-NEXT: @ %bb.6: @ %middle.block
921; CHECK-NEXT: mov.w r2, #-2147483648
922; CHECK-NEXT: cmp r3, r1
923; CHECK-NEXT: vmaxv.s32 r2, q0
924; CHECK-NEXT: beq .LBB9_9
925; CHECK-NEXT: .LBB9_7: @ %for.body.preheader1
926; CHECK-NEXT: sub.w lr, r1, r3
927; CHECK-NEXT: add.w r0, r0, r3, lsl #2
928; CHECK-NEXT: dls lr, lr
929; CHECK-NEXT: .LBB9_8: @ %for.body
930; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
931; CHECK-NEXT: ldr r1, [r0], #4
932; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]933; CHECK-NEXT: csel r2, r2, r1, gt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]934; CHECK-NEXT: le lr, .LBB9_8
935; CHECK-NEXT: .LBB9_9: @ %for.cond.cleanup
936; CHECK-NEXT: mov r0, r2
937; CHECK-NEXT: pop {r7, pc}
938entry:
939 %cmp6 = icmp sgt i32 %n, 0
940 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
941
942for.body.preheader: ; preds = %entry
943 %min.iters.check = icmp ult i32 %n, 4
944 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
945
946vector.ph: ; preds = %for.body.preheader
947 %n.vec = and i32 %n, -4
948 br label %vector.body
949
950vector.body: ; preds = %vector.body, %vector.ph
951 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
952 %vec.phi = phi <4 x i32> [ <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %vector.ph ], [ %3, %vector.body ]
953 %0 = getelementptr inbounds i32, i32* %x, i32 %index
954 %1 = bitcast i32* %0 to <4 x i32>*
955 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
956 %2 = icmp sgt <4 x i32> %vec.phi, %wide.load
957 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
958 %index.next = add i32 %index, 4
959 %4 = icmp eq i32 %index.next, %n.vec
960 br i1 %4, label %middle.block, label %vector.body
961
962middle.block: ; preds = %vector.body
963 %5 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> %3)
964 %cmp.n = icmp eq i32 %n.vec, %n
965 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
966
967for.body.preheader1: ; preds = %middle.block, %for.body.preheader
968 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
969 %r.07.ph = phi i32 [ -2147483648, %for.body.preheader ], [ %5, %middle.block ]
970 br label %for.body
971
972for.body: ; preds = %for.body.preheader1, %for.body
973 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
974 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
975 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
976 %6 = load i32, i32* %arrayidx, align 4
977 %c = icmp sgt i32 %r.07, %6
978 %add = select i1 %c, i32 %r.07, i32 %6
979 %inc = add nuw nsw i32 %i.08, 1
980 %exitcond = icmp eq i32 %inc, %n
981 br i1 %exitcond, label %for.cond.cleanup, label %for.body
982
983for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
984 %r.0.lcssa = phi i32 [ -2147483648, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
985 ret i32 %r.0.lcssa
986}
987
988define i32 @smax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
989; CHECK-LABEL: smax_i32_inloop:
990; CHECK: @ %bb.0: @ %entry
991; CHECK-NEXT: .save {r4, lr}
992; CHECK-NEXT: push {r4, lr}
993; CHECK-NEXT: cmp r1, #1
994; CHECK-NEXT: blt .LBB10_3
995; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
996; CHECK-NEXT: mov r12, r0
997; CHECK-NEXT: mov.w r0, #-2147483648
998; CHECK-NEXT: cmp r1, #4
999; CHECK-NEXT: bhs .LBB10_4
1000; CHECK-NEXT: @ %bb.2:
1001; CHECK-NEXT: movs r3, #0
1002; CHECK-NEXT: b .LBB10_7
1003; CHECK-NEXT: .LBB10_3:
1004; CHECK-NEXT: mov.w r0, #-2147483648
1005; CHECK-NEXT: b .LBB10_9
1006; CHECK-NEXT: .LBB10_4: @ %vector.ph
1007; CHECK-NEXT: bic r3, r1, #3
1008; CHECK-NEXT: movs r2, #1
1009; CHECK-NEXT: sub.w lr, r3, #4
1010; CHECK-NEXT: add.w lr, r2, lr, lsr #2
1011; CHECK-NEXT: mov r2, r12
1012; CHECK-NEXT: dls lr, lr
1013; CHECK-NEXT: .LBB10_5: @ %vector.body
1014; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1015; CHECK-NEXT: vldrw.u32 q0, [r2], #16
1016; CHECK-NEXT: mov.w r4, #-2147483648
1017; CHECK-NEXT: vmaxv.s32 r4, q0
1018; CHECK-NEXT: cmp r0, r4
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1019; CHECK-NEXT: csel r0, r0, r4, gt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1020; CHECK-NEXT: le lr, .LBB10_5
1021; CHECK-NEXT: @ %bb.6: @ %middle.block
1022; CHECK-NEXT: cmp r3, r1
1023; CHECK-NEXT: it eq
1024; CHECK-NEXT: popeq {r4, pc}
1025; CHECK-NEXT: .LBB10_7: @ %for.body.preheader1
1026; CHECK-NEXT: sub.w lr, r1, r3
1027; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1028; CHECK-NEXT: dls lr, lr
1029; CHECK-NEXT: .LBB10_8: @ %for.body
1030; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1031; CHECK-NEXT: ldr r2, [r1], #4
1032; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1033; CHECK-NEXT: csel r0, r0, r2, gt
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1034; CHECK-NEXT: le lr, .LBB10_8
1035; CHECK-NEXT: .LBB10_9: @ %for.cond.cleanup
1036; CHECK-NEXT: pop {r4, pc}
1037entry:
1038 %cmp6 = icmp sgt i32 %n, 0
1039 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1040
1041for.body.preheader: ; preds = %entry
1042 %min.iters.check = icmp ult i32 %n, 4
1043 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1044
1045vector.ph: ; preds = %for.body.preheader
1046 %n.vec = and i32 %n, -4
1047 br label %vector.body
1048
1049vector.body: ; preds = %vector.body, %vector.ph
1050 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1051 %vec.phi = phi i32 [ -2147483648, %vector.ph ], [ %3, %vector.body ]
1052 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1053 %1 = bitcast i32* %0 to <4 x i32>*
1054 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1055 %l5 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> %wide.load)
1056 %2 = icmp sgt i32 %vec.phi, %l5
1057 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1058 %index.next = add i32 %index, 4
1059 %4 = icmp eq i32 %index.next, %n.vec
1060 br i1 %4, label %middle.block, label %vector.body
1061
1062middle.block: ; preds = %vector.body
1063 %5 = phi i32 [ %3, %vector.body ]
1064 %cmp.n = icmp eq i32 %n.vec, %n
1065 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1066
1067for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1068 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1069 %r.07.ph = phi i32 [ -2147483648, %for.body.preheader ], [ %5, %middle.block ]
1070 br label %for.body
1071
1072for.body: ; preds = %for.body.preheader1, %for.body
1073 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1074 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1075 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1076 %6 = load i32, i32* %arrayidx, align 4
1077 %c = icmp sgt i32 %r.07, %6
1078 %add = select i1 %c, i32 %r.07, i32 %6
1079 %inc = add nuw nsw i32 %i.08, 1
1080 %exitcond = icmp eq i32 %inc, %n
1081 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1082
1083for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1084 %r.0.lcssa = phi i32 [ -2147483648, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1085 ret i32 %r.0.lcssa
1086}
1087
1088define i32 @umin_i32(i32* nocapture readonly %x, i32 %n) {
1089; CHECK-LABEL: umin_i32:
1090; CHECK: @ %bb.0: @ %entry
1091; CHECK-NEXT: .save {r7, lr}
1092; CHECK-NEXT: push {r7, lr}
1093; CHECK-NEXT: cmp r1, #1
1094; CHECK-NEXT: blt .LBB11_3
1095; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1096; CHECK-NEXT: cmp r1, #4
1097; CHECK-NEXT: bhs .LBB11_4
1098; CHECK-NEXT: @ %bb.2:
1099; CHECK-NEXT: mov.w r2, #-1
1100; CHECK-NEXT: movs r3, #0
1101; CHECK-NEXT: b .LBB11_7
1102; CHECK-NEXT: .LBB11_3:
1103; CHECK-NEXT: mov.w r2, #-1
1104; CHECK-NEXT: b .LBB11_9
1105; CHECK-NEXT: .LBB11_4: @ %vector.ph
1106; CHECK-NEXT: bic r3, r1, #3
1107; CHECK-NEXT: movs r2, #1
1108; CHECK-NEXT: sub.w r12, r3, #4
1109; CHECK-NEXT: vmov.i8 q0, #0xff
1110; CHECK-NEXT: add.w lr, r2, r12, lsr #2
1111; CHECK-NEXT: mov r2, r0
1112; CHECK-NEXT: dls lr, lr
1113; CHECK-NEXT: .LBB11_5: @ %vector.body
1114; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1115; CHECK-NEXT: vldrw.u32 q1, [r2], #16
1116; CHECK-NEXT: vmin.u32 q0, q0, q1
1117; CHECK-NEXT: le lr, .LBB11_5
1118; CHECK-NEXT: @ %bb.6: @ %middle.block
1119; CHECK-NEXT: mov.w r2, #-1
1120; CHECK-NEXT: cmp r3, r1
1121; CHECK-NEXT: vminv.u32 r2, q0
1122; CHECK-NEXT: beq .LBB11_9
1123; CHECK-NEXT: .LBB11_7: @ %for.body.preheader1
1124; CHECK-NEXT: sub.w lr, r1, r3
1125; CHECK-NEXT: add.w r0, r0, r3, lsl #2
1126; CHECK-NEXT: dls lr, lr
1127; CHECK-NEXT: .LBB11_8: @ %for.body
1128; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1129; CHECK-NEXT: ldr r1, [r0], #4
1130; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1131; CHECK-NEXT: csel r2, r2, r1, lo
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1132; CHECK-NEXT: le lr, .LBB11_8
1133; CHECK-NEXT: .LBB11_9: @ %for.cond.cleanup
1134; CHECK-NEXT: mov r0, r2
1135; CHECK-NEXT: pop {r7, pc}
1136entry:
1137 %cmp6 = icmp sgt i32 %n, 0
1138 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1139
1140for.body.preheader: ; preds = %entry
1141 %min.iters.check = icmp ult i32 %n, 4
1142 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1143
1144vector.ph: ; preds = %for.body.preheader
1145 %n.vec = and i32 %n, -4
1146 br label %vector.body
1147
1148vector.body: ; preds = %vector.body, %vector.ph
1149 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1150 %vec.phi = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %vector.ph ], [ %3, %vector.body ]
1151 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1152 %1 = bitcast i32* %0 to <4 x i32>*
1153 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1154 %2 = icmp ult <4 x i32> %vec.phi, %wide.load
1155 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
1156 %index.next = add i32 %index, 4
1157 %4 = icmp eq i32 %index.next, %n.vec
1158 br i1 %4, label %middle.block, label %vector.body
1159
1160middle.block: ; preds = %vector.body
1161 %5 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> %3)
1162 %cmp.n = icmp eq i32 %n.vec, %n
1163 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1164
1165for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1166 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1167 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %5, %middle.block ]
1168 br label %for.body
1169
1170for.body: ; preds = %for.body.preheader1, %for.body
1171 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1172 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1173 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1174 %6 = load i32, i32* %arrayidx, align 4
1175 %c = icmp ult i32 %r.07, %6
1176 %add = select i1 %c, i32 %r.07, i32 %6
1177 %inc = add nuw nsw i32 %i.08, 1
1178 %exitcond = icmp eq i32 %inc, %n
1179 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1180
1181for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1182 %r.0.lcssa = phi i32 [ -1, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1183 ret i32 %r.0.lcssa
1184}
1185
1186define i32 @umin_i32_inloop(i32* nocapture readonly %x, i32 %n) {
1187; CHECK-LABEL: umin_i32_inloop:
1188; CHECK: @ %bb.0: @ %entry
1189; CHECK-NEXT: .save {r4, lr}
1190; CHECK-NEXT: push {r4, lr}
1191; CHECK-NEXT: cmp r1, #1
1192; CHECK-NEXT: blt .LBB12_3
1193; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1194; CHECK-NEXT: mov r12, r0
1195; CHECK-NEXT: mov.w r0, #-1
1196; CHECK-NEXT: cmp r1, #4
1197; CHECK-NEXT: bhs .LBB12_4
1198; CHECK-NEXT: @ %bb.2:
1199; CHECK-NEXT: movs r3, #0
1200; CHECK-NEXT: b .LBB12_7
1201; CHECK-NEXT: .LBB12_3:
1202; CHECK-NEXT: mov.w r0, #-1
1203; CHECK-NEXT: b .LBB12_9
1204; CHECK-NEXT: .LBB12_4: @ %vector.ph
1205; CHECK-NEXT: bic r3, r1, #3
1206; CHECK-NEXT: movs r2, #1
1207; CHECK-NEXT: sub.w lr, r3, #4
1208; CHECK-NEXT: add.w lr, r2, lr, lsr #2
1209; CHECK-NEXT: mov r2, r12
1210; CHECK-NEXT: dls lr, lr
1211; CHECK-NEXT: .LBB12_5: @ %vector.body
1212; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1213; CHECK-NEXT: vldrw.u32 q0, [r2], #16
1214; CHECK-NEXT: mov.w r4, #-1
1215; CHECK-NEXT: vminv.u32 r4, q0
1216; CHECK-NEXT: cmp r0, r4
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1217; CHECK-NEXT: csel r0, r0, r4, lo
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1218; CHECK-NEXT: le lr, .LBB12_5
1219; CHECK-NEXT: @ %bb.6: @ %middle.block
1220; CHECK-NEXT: cmp r3, r1
1221; CHECK-NEXT: it eq
1222; CHECK-NEXT: popeq {r4, pc}
1223; CHECK-NEXT: .LBB12_7: @ %for.body.preheader1
1224; CHECK-NEXT: sub.w lr, r1, r3
1225; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1226; CHECK-NEXT: dls lr, lr
1227; CHECK-NEXT: .LBB12_8: @ %for.body
1228; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1229; CHECK-NEXT: ldr r2, [r1], #4
1230; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1231; CHECK-NEXT: csel r0, r0, r2, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1232; CHECK-NEXT: le lr, .LBB12_8
1233; CHECK-NEXT: .LBB12_9: @ %for.cond.cleanup
1234; CHECK-NEXT: pop {r4, pc}
1235entry:
1236 %cmp6 = icmp sgt i32 %n, 0
1237 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1238
1239for.body.preheader: ; preds = %entry
1240 %min.iters.check = icmp ult i32 %n, 4
1241 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1242
1243vector.ph: ; preds = %for.body.preheader
1244 %n.vec = and i32 %n, -4
1245 br label %vector.body
1246
1247vector.body: ; preds = %vector.body, %vector.ph
1248 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1249 %vec.phi = phi i32 [ -1, %vector.ph ], [ %3, %vector.body ]
1250 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1251 %1 = bitcast i32* %0 to <4 x i32>*
1252 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1253 %l5 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> %wide.load)
1254 %2 = icmp ult i32 %vec.phi, %l5
1255 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1256 %index.next = add i32 %index, 4
1257 %4 = icmp eq i32 %index.next, %n.vec
1258 br i1 %4, label %middle.block, label %vector.body
1259
1260middle.block: ; preds = %vector.body
1261 %5 = phi i32 [ %3, %vector.body ]
1262 %cmp.n = icmp eq i32 %n.vec, %n
1263 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1264
1265for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1266 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1267 %r.07.ph = phi i32 [ -1, %for.body.preheader ], [ %5, %middle.block ]
1268 br label %for.body
1269
1270for.body: ; preds = %for.body.preheader1, %for.body
1271 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1272 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1273 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1274 %6 = load i32, i32* %arrayidx, align 4
1275 %c = icmp ugt i32 %r.07, %6
1276 %add = select i1 %c, i32 %r.07, i32 %6
1277 %inc = add nuw nsw i32 %i.08, 1
1278 %exitcond = icmp eq i32 %inc, %n
1279 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1280
1281for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1282 %r.0.lcssa = phi i32 [ -1, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1283 ret i32 %r.0.lcssa
1284}
1285
1286define i32 @umax_i32(i32* nocapture readonly %x, i32 %n) {
1287; CHECK-LABEL: umax_i32:
1288; CHECK: @ %bb.0: @ %entry
1289; CHECK-NEXT: .save {r7, lr}
1290; CHECK-NEXT: push {r7, lr}
1291; CHECK-NEXT: cmp r1, #1
1292; CHECK-NEXT: blt .LBB13_3
1293; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1294; CHECK-NEXT: cmp r1, #4
1295; CHECK-NEXT: bhs .LBB13_4
1296; CHECK-NEXT: @ %bb.2:
1297; CHECK-NEXT: movs r3, #0
1298; CHECK-NEXT: movs r2, #0
1299; CHECK-NEXT: b .LBB13_7
1300; CHECK-NEXT: .LBB13_3:
1301; CHECK-NEXT: movs r2, #0
1302; CHECK-NEXT: b .LBB13_9
1303; CHECK-NEXT: .LBB13_4: @ %vector.ph
1304; CHECK-NEXT: bic r3, r1, #3
1305; CHECK-NEXT: movs r2, #1
1306; CHECK-NEXT: sub.w r12, r3, #4
1307; CHECK-NEXT: vmov.i32 q0, #0x0
1308; CHECK-NEXT: add.w lr, r2, r12, lsr #2
1309; CHECK-NEXT: mov r2, r0
1310; CHECK-NEXT: dls lr, lr
1311; CHECK-NEXT: .LBB13_5: @ %vector.body
1312; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1313; CHECK-NEXT: vldrw.u32 q1, [r2], #16
1314; CHECK-NEXT: vmax.u32 q0, q0, q1
1315; CHECK-NEXT: le lr, .LBB13_5
1316; CHECK-NEXT: @ %bb.6: @ %middle.block
1317; CHECK-NEXT: movs r2, #0
1318; CHECK-NEXT: cmp r3, r1
1319; CHECK-NEXT: vmaxv.u32 r2, q0
1320; CHECK-NEXT: beq .LBB13_9
1321; CHECK-NEXT: .LBB13_7: @ %for.body.preheader1
1322; CHECK-NEXT: sub.w lr, r1, r3
1323; CHECK-NEXT: add.w r0, r0, r3, lsl #2
1324; CHECK-NEXT: dls lr, lr
1325; CHECK-NEXT: .LBB13_8: @ %for.body
1326; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1327; CHECK-NEXT: ldr r1, [r0], #4
1328; CHECK-NEXT: cmp r2, r1
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1329; CHECK-NEXT: csel r2, r2, r1, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1330; CHECK-NEXT: le lr, .LBB13_8
1331; CHECK-NEXT: .LBB13_9: @ %for.cond.cleanup
1332; CHECK-NEXT: mov r0, r2
1333; CHECK-NEXT: pop {r7, pc}
1334entry:
1335 %cmp6 = icmp sgt i32 %n, 0
1336 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1337
1338for.body.preheader: ; preds = %entry
1339 %min.iters.check = icmp ult i32 %n, 4
1340 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1341
1342vector.ph: ; preds = %for.body.preheader
1343 %n.vec = and i32 %n, -4
1344 br label %vector.body
1345
1346vector.body: ; preds = %vector.body, %vector.ph
1347 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1348 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1349 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1350 %1 = bitcast i32* %0 to <4 x i32>*
1351 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1352 %2 = icmp ugt <4 x i32> %vec.phi, %wide.load
1353 %3 = select <4 x i1> %2, <4 x i32> %vec.phi, <4 x i32> %wide.load
1354 %index.next = add i32 %index, 4
1355 %4 = icmp eq i32 %index.next, %n.vec
1356 br i1 %4, label %middle.block, label %vector.body
1357
1358middle.block: ; preds = %vector.body
1359 %5 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> %3)
1360 %cmp.n = icmp eq i32 %n.vec, %n
1361 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1362
1363for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1364 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1365 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %5, %middle.block ]
1366 br label %for.body
1367
1368for.body: ; preds = %for.body.preheader1, %for.body
1369 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1370 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1371 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1372 %6 = load i32, i32* %arrayidx, align 4
1373 %c = icmp ugt i32 %r.07, %6
1374 %add = select i1 %c, i32 %r.07, i32 %6
1375 %inc = add nuw nsw i32 %i.08, 1
1376 %exitcond = icmp eq i32 %inc, %n
1377 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1378
1379for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1380 %r.0.lcssa = phi i32 [ 0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1381 ret i32 %r.0.lcssa
1382}
1383
1384define i32 @umax_i32_inloop(i32* nocapture readonly %x, i32 %n) {
1385; CHECK-LABEL: umax_i32_inloop:
1386; CHECK: @ %bb.0: @ %entry
1387; CHECK-NEXT: .save {r4, lr}
1388; CHECK-NEXT: push {r4, lr}
1389; CHECK-NEXT: cmp r1, #1
1390; CHECK-NEXT: blt .LBB14_8
1391; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1392; CHECK-NEXT: mov r12, r0
1393; CHECK-NEXT: movs r3, #0
1394; CHECK-NEXT: cmp r1, #4
1395; CHECK-NEXT: mov.w r0, #0
1396; CHECK-NEXT: blo .LBB14_5
1397; CHECK-NEXT: @ %bb.2: @ %vector.ph
1398; CHECK-NEXT: bic r3, r1, #3
1399; CHECK-NEXT: movs r2, #1
1400; CHECK-NEXT: subs r0, r3, #4
1401; CHECK-NEXT: add.w lr, r2, r0, lsr #2
1402; CHECK-NEXT: movs r0, #0
1403; CHECK-NEXT: mov r2, r12
1404; CHECK-NEXT: dls lr, lr
1405; CHECK-NEXT: .LBB14_3: @ %vector.body
1406; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1407; CHECK-NEXT: vldrw.u32 q0, [r2], #16
1408; CHECK-NEXT: movs r4, #0
1409; CHECK-NEXT: vmaxv.u32 r4, q0
1410; CHECK-NEXT: cmp r0, r4
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1411; CHECK-NEXT: csel r0, r0, r4, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1412; CHECK-NEXT: le lr, .LBB14_3
1413; CHECK-NEXT: @ %bb.4: @ %middle.block
1414; CHECK-NEXT: cmp r3, r1
1415; CHECK-NEXT: it eq
1416; CHECK-NEXT: popeq {r4, pc}
1417; CHECK-NEXT: .LBB14_5: @ %for.body.preheader1
1418; CHECK-NEXT: sub.w lr, r1, r3
1419; CHECK-NEXT: add.w r1, r12, r3, lsl #2
1420; CHECK-NEXT: dls lr, lr
1421; CHECK-NEXT: .LBB14_6: @ %for.body
1422; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1423; CHECK-NEXT: ldr r2, [r1], #4
1424; CHECK-NEXT: cmp r0, r2
David Green146d35b2020-07-14 10:04:55 +0100[diff] [blame]1425; CHECK-NEXT: csel r0, r0, r2, hi
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]1426; CHECK-NEXT: le lr, .LBB14_6
1427; CHECK-NEXT: @ %bb.7: @ %for.cond.cleanup
1428; CHECK-NEXT: pop {r4, pc}
1429; CHECK-NEXT: .LBB14_8:
1430; CHECK-NEXT: movs r0, #0
1431; CHECK-NEXT: pop {r4, pc}
1432entry:
1433 %cmp6 = icmp sgt i32 %n, 0
1434 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1435
1436for.body.preheader: ; preds = %entry
1437 %min.iters.check = icmp ult i32 %n, 4
1438 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1439
1440vector.ph: ; preds = %for.body.preheader
1441 %n.vec = and i32 %n, -4
1442 br label %vector.body
1443
1444vector.body: ; preds = %vector.body, %vector.ph
1445 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1446 %vec.phi = phi i32 [ 0, %vector.ph ], [ %3, %vector.body ]
1447 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1448 %1 = bitcast i32* %0 to <4 x i32>*
1449 %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
1450 %l5 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> %wide.load)
1451 %2 = icmp ugt i32 %vec.phi, %l5
1452 %3 = select i1 %2, i32 %vec.phi, i32 %l5
1453 %index.next = add i32 %index, 4
1454 %4 = icmp eq i32 %index.next, %n.vec
1455 br i1 %4, label %middle.block, label %vector.body
1456
1457middle.block: ; preds = %vector.body
1458 %5 = phi i32 [ %3, %vector.body ]
1459 %cmp.n = icmp eq i32 %n.vec, %n
1460 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1461
1462for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1463 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1464 %r.07.ph = phi i32 [ 0, %for.body.preheader ], [ %5, %middle.block ]
1465 br label %for.body
1466
1467for.body: ; preds = %for.body.preheader1, %for.body
1468 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1469 %r.07 = phi i32 [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1470 %arrayidx = getelementptr inbounds i32, i32* %x, i32 %i.08
1471 %6 = load i32, i32* %arrayidx, align 4
1472 %c = icmp ugt i32 %r.07, %6
1473 %add = select i1 %c, i32 %r.07, i32 %6
1474 %inc = add nuw nsw i32 %i.08, 1
1475 %exitcond = icmp eq i32 %inc, %n
1476 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1477
1478for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1479 %r.0.lcssa = phi i32 [ 0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1480 ret i32 %r.0.lcssa
1481}
1482
1483define float @fmin_f32(float* nocapture readonly %x, i32 %n) {
1484; CHECK-LABEL: fmin_f32:
1485; CHECK: @ %bb.0: @ %entry
1486; CHECK-NEXT: .save {r7, lr}
1487; CHECK-NEXT: push {r7, lr}
1488; CHECK-NEXT: cmp r1, #1
1489; CHECK-NEXT: blt .LBB15_3
1490; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1491; CHECK-NEXT: cmp r1, #4
1492; CHECK-NEXT: bhs .LBB15_4
1493; CHECK-NEXT: @ %bb.2:
1494; CHECK-NEXT: vldr s0, .LCPI15_0
1495; CHECK-NEXT: movs r2, #0
1496; CHECK-NEXT: b .LBB15_7
1497; CHECK-NEXT: .LBB15_3:
1498; CHECK-NEXT: vldr s0, .LCPI15_0
1499; CHECK-NEXT: b .LBB15_9
1500; CHECK-NEXT: .LBB15_4: @ %vector.ph
1501; CHECK-NEXT: bic r2, r1, #3
1502; CHECK-NEXT: movs r3, #1
1503; CHECK-NEXT: sub.w r12, r2, #4
1504; CHECK-NEXT: vmov.i32 q0, #0x0
1505; CHECK-NEXT: add.w lr, r3, r12, lsr #2
1506; CHECK-NEXT: mov r3, r0
1507; CHECK-NEXT: dls lr, lr
1508; CHECK-NEXT: .LBB15_5: @ %vector.body
1509; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1510; CHECK-NEXT: vldrw.u32 q1, [r3], #16
1511; CHECK-NEXT: vcmp.f32 lt, q0, q1
1512; CHECK-NEXT: vpsel q0, q0, q1
1513; CHECK-NEXT: le lr, .LBB15_5
1514; CHECK-NEXT: @ %bb.6: @ %middle.block
1515; CHECK-NEXT: vmov.f32 s4, s2
1516; CHECK-NEXT: cmp r2, r1
1517; CHECK-NEXT: vmov.f32 s5, s3
1518; CHECK-NEXT: vminnm.f32 q0, q0, q1
1519; CHECK-NEXT: vmov r3, s1
1520; CHECK-NEXT: vdup.32 q1, r3
1521; CHECK-NEXT: vminnm.f32 q0, q0, q1
1522; CHECK-NEXT: beq .LBB15_9
1523; CHECK-NEXT: .LBB15_7: @ %for.body.preheader1
1524; CHECK-NEXT: sub.w lr, r1, r2
1525; CHECK-NEXT: add.w r0, r0, r2, lsl #2
1526; CHECK-NEXT: dls lr, lr
1527; CHECK-NEXT: .LBB15_8: @ %for.body
1528; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1529; CHECK-NEXT: vldmia r0!, {s4}
1530; CHECK-NEXT: vcmp.f32 s0, s4
1531; CHECK-NEXT: vmrs APSR_nzcv, fpscr
1532; CHECK-NEXT: vselge.f32 s0, s4, s0
1533; CHECK-NEXT: le lr, .LBB15_8
1534; CHECK-NEXT: .LBB15_9: @ %for.cond.cleanup
1535; CHECK-NEXT: vmov r0, s0
1536; CHECK-NEXT: pop {r7, pc}
1537; CHECK-NEXT: .p2align 2
1538; CHECK-NEXT: @ %bb.10:
1539; CHECK-NEXT: .LCPI15_0:
1540; CHECK-NEXT: .long 0x00000000 @ float 0
1541entry:
1542 %cmp6 = icmp sgt i32 %n, 0
1543 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1544
1545for.body.preheader: ; preds = %entry
1546 %min.iters.check = icmp ult i32 %n, 4
1547 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1548
1549vector.ph: ; preds = %for.body.preheader
1550 %n.vec = and i32 %n, -4
1551 br label %vector.body
1552
1553vector.body: ; preds = %vector.body, %vector.ph
1554 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1555 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1556 %0 = getelementptr inbounds float, float* %x, i32 %index
1557 %1 = bitcast float* %0 to <4 x float>*
1558 %wide.load = load <4 x float>, <4 x float>* %1, align 4
1559 %2 = fcmp ult <4 x float> %vec.phi, %wide.load
1560 %3 = select <4 x i1> %2, <4 x float> %vec.phi, <4 x float> %wide.load
1561 %index.next = add i32 %index, 4
1562 %4 = icmp eq i32 %index.next, %n.vec
1563 br i1 %4, label %middle.block, label %vector.body
1564
1565middle.block: ; preds = %vector.body
1566 %5 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> %3)
1567 %cmp.n = icmp eq i32 %n.vec, %n
1568 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1569
1570for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1571 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1572 %r.07.ph = phi float [ 0.0, %for.body.preheader ], [ %5, %middle.block ]
1573 br label %for.body
1574
1575for.body: ; preds = %for.body.preheader1, %for.body
1576 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1577 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1578 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
1579 %6 = load float, float* %arrayidx, align 4
1580 %c = fcmp ult float %r.07, %6
1581 %add = select i1 %c, float %r.07, float %6
1582 %inc = add nuw nsw i32 %i.08, 1
1583 %exitcond = icmp eq i32 %inc, %n
1584 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1585
1586for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1587 %r.0.lcssa = phi float [ 0.0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1588 ret float %r.0.lcssa
1589}
1590
1591define float @fmax_f32(float* nocapture readonly %x, i32 %n) {
1592; CHECK-LABEL: fmax_f32:
1593; CHECK: @ %bb.0: @ %entry
1594; CHECK-NEXT: .save {r7, lr}
1595; CHECK-NEXT: push {r7, lr}
1596; CHECK-NEXT: cmp r1, #1
1597; CHECK-NEXT: blt .LBB16_3
1598; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
1599; CHECK-NEXT: cmp r1, #4
1600; CHECK-NEXT: bhs .LBB16_4
1601; CHECK-NEXT: @ %bb.2:
1602; CHECK-NEXT: vldr s0, .LCPI16_0
1603; CHECK-NEXT: movs r2, #0
1604; CHECK-NEXT: b .LBB16_7
1605; CHECK-NEXT: .LBB16_3:
1606; CHECK-NEXT: vldr s0, .LCPI16_0
1607; CHECK-NEXT: b .LBB16_9
1608; CHECK-NEXT: .LBB16_4: @ %vector.ph
1609; CHECK-NEXT: bic r2, r1, #3
1610; CHECK-NEXT: movs r3, #1
1611; CHECK-NEXT: sub.w r12, r2, #4
1612; CHECK-NEXT: vmov.i32 q0, #0x0
1613; CHECK-NEXT: add.w lr, r3, r12, lsr #2
1614; CHECK-NEXT: mov r3, r0
1615; CHECK-NEXT: dls lr, lr
1616; CHECK-NEXT: .LBB16_5: @ %vector.body
1617; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1618; CHECK-NEXT: vldrw.u32 q1, [r3], #16
1619; CHECK-NEXT: vcmp.f32 lt, q1, q0
1620; CHECK-NEXT: vpsel q0, q0, q1
1621; CHECK-NEXT: le lr, .LBB16_5
1622; CHECK-NEXT: @ %bb.6: @ %middle.block
1623; CHECK-NEXT: vmov.f32 s4, s2
1624; CHECK-NEXT: cmp r2, r1
1625; CHECK-NEXT: vmov.f32 s5, s3
1626; CHECK-NEXT: vmaxnm.f32 q0, q0, q1
1627; CHECK-NEXT: vmov r3, s1
1628; CHECK-NEXT: vdup.32 q1, r3
1629; CHECK-NEXT: vmaxnm.f32 q0, q0, q1
1630; CHECK-NEXT: beq .LBB16_9
1631; CHECK-NEXT: .LBB16_7: @ %for.body.preheader1
1632; CHECK-NEXT: sub.w lr, r1, r2
1633; CHECK-NEXT: add.w r0, r0, r2, lsl #2
1634; CHECK-NEXT: dls lr, lr
1635; CHECK-NEXT: .LBB16_8: @ %for.body
1636; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1637; CHECK-NEXT: vldmia r0!, {s4}
1638; CHECK-NEXT: vcmp.f32 s4, s0
1639; CHECK-NEXT: vmrs APSR_nzcv, fpscr
1640; CHECK-NEXT: vselge.f32 s0, s4, s0
1641; CHECK-NEXT: le lr, .LBB16_8
1642; CHECK-NEXT: .LBB16_9: @ %for.cond.cleanup
1643; CHECK-NEXT: vmov r0, s0
1644; CHECK-NEXT: pop {r7, pc}
1645; CHECK-NEXT: .p2align 2
1646; CHECK-NEXT: @ %bb.10:
1647; CHECK-NEXT: .LCPI16_0:
1648; CHECK-NEXT: .long 0x00000000 @ float 0
1649entry:
1650 %cmp6 = icmp sgt i32 %n, 0
1651 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
1652
1653for.body.preheader: ; preds = %entry
1654 %min.iters.check = icmp ult i32 %n, 4
1655 br i1 %min.iters.check, label %for.body.preheader1, label %vector.ph
1656
1657vector.ph: ; preds = %for.body.preheader
1658 %n.vec = and i32 %n, -4
1659 br label %vector.body
1660
1661vector.body: ; preds = %vector.body, %vector.ph
1662 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1663 %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
1664 %0 = getelementptr inbounds float, float* %x, i32 %index
1665 %1 = bitcast float* %0 to <4 x float>*
1666 %wide.load = load <4 x float>, <4 x float>* %1, align 4
1667 %2 = fcmp ugt <4 x float> %vec.phi, %wide.load
1668 %3 = select <4 x i1> %2, <4 x float> %vec.phi, <4 x float> %wide.load
1669 %index.next = add i32 %index, 4
1670 %4 = icmp eq i32 %index.next, %n.vec
1671 br i1 %4, label %middle.block, label %vector.body
1672
1673middle.block: ; preds = %vector.body
1674 %5 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> %3)
1675 %cmp.n = icmp eq i32 %n.vec, %n
1676 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader1
1677
1678for.body.preheader1: ; preds = %middle.block, %for.body.preheader
1679 %i.08.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
1680 %r.07.ph = phi float [ 0.0, %for.body.preheader ], [ %5, %middle.block ]
1681 br label %for.body
1682
1683for.body: ; preds = %for.body.preheader1, %for.body
1684 %i.08 = phi i32 [ %inc, %for.body ], [ %i.08.ph, %for.body.preheader1 ]
1685 %r.07 = phi float [ %add, %for.body ], [ %r.07.ph, %for.body.preheader1 ]
1686 %arrayidx = getelementptr inbounds float, float* %x, i32 %i.08
1687 %6 = load float, float* %arrayidx, align 4
1688 %c = fcmp ugt float %r.07, %6
1689 %add = select i1 %c, float %r.07, float %6
1690 %inc = add nuw nsw i32 %i.08, 1
1691 %exitcond = icmp eq i32 %inc, %n
1692 br i1 %exitcond, label %for.cond.cleanup, label %for.body
1693
1694for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
1695 %r.0.lcssa = phi float [ 0.0, %entry ], [ %5, %middle.block ], [ %add, %for.body ]
1696 ret float %r.0.lcssa
1697}
1698
David Green25e38c32020-08-07 17:16:56 +0100[diff] [blame^]1699define i32 @add4i32(i32* noalias nocapture readonly %x, i32 %n) {
1700; CHECK-LABEL: add4i32:
1701; CHECK: @ %bb.0: @ %entry
1702; CHECK-NEXT: .save {r7, lr}
1703; CHECK-NEXT: push {r7, lr}
1704; CHECK-NEXT: cbz r1, .LBB17_4
1705; CHECK-NEXT: @ %bb.1: @ %vector.ph
1706; CHECK-NEXT: adds r2, r1, #3
1707; CHECK-NEXT: movs r3, #1
1708; CHECK-NEXT: bic r2, r2, #3
1709; CHECK-NEXT: subs r2, #4
1710; CHECK-NEXT: add.w lr, r3, r2, lsr #2
1711; CHECK-NEXT: adr r3, .LCPI17_0
1712; CHECK-NEXT: subs r2, r1, #1
1713; CHECK-NEXT: vldrw.u32 q0, [r3]
1714; CHECK-NEXT: vdup.32 q1, r2
1715; CHECK-NEXT: movs r1, #0
1716; CHECK-NEXT: movs r2, #0
1717; CHECK-NEXT: dls lr, lr
1718; CHECK-NEXT: .LBB17_2: @ %vector.body
1719; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1720; CHECK-NEXT: vadd.i32 q2, q0, r1
1721; CHECK-NEXT: vdup.32 q3, r1
1722; CHECK-NEXT: vcmp.u32 hi, q3, q2
1723; CHECK-NEXT: adds r1, #4
1724; CHECK-NEXT: vpnot
1725; CHECK-NEXT: vpsttt
1726; CHECK-NEXT: vcmpt.u32 cs, q1, q2
1727; CHECK-NEXT: vldrwt.u32 q2, [r0], #16
1728; CHECK-NEXT: vaddvat.u32 r2, q2
1729; CHECK-NEXT: le lr, .LBB17_2
1730; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
1731; CHECK-NEXT: mov r0, r2
1732; CHECK-NEXT: pop {r7, pc}
1733; CHECK-NEXT: .LBB17_4:
1734; CHECK-NEXT: movs r2, #0
1735; CHECK-NEXT: mov r0, r2
1736; CHECK-NEXT: pop {r7, pc}
1737; CHECK-NEXT: .p2align 4
1738; CHECK-NEXT: @ %bb.5:
1739; CHECK-NEXT: .LCPI17_0:
1740; CHECK-NEXT: .long 0 @ 0x0
1741; CHECK-NEXT: .long 1 @ 0x1
1742; CHECK-NEXT: .long 2 @ 0x2
1743; CHECK-NEXT: .long 3 @ 0x3
1744entry:
1745 %cmp6.not = icmp eq i32 %n, 0
1746 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
1747
1748vector.ph: ; preds = %entry
1749 %n.rnd.up = add i32 %n, 3
1750 %n.vec = and i32 %n.rnd.up, -4
1751 %trip.count.minus.1 = add i32 %n, -1
1752 br label %vector.body
1753
1754vector.body: ; preds = %vector.body, %vector.ph
1755 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1756 %vec.phi = phi i32 [ 0, %vector.ph ], [ %4, %vector.body ]
1757 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
1758 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1759 %1 = bitcast i32* %0 to <4 x i32>*
1760 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1761 %2 = select <4 x i1> %active.lane.mask, <4 x i32> %wide.masked.load, <4 x i32> zeroinitializer
1762 %3 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %2)
1763 %4 = add i32 %3, %vec.phi
1764 %index.next = add i32 %index, 4
1765 %5 = icmp eq i32 %index.next, %n.vec
1766 br i1 %5, label %for.cond.cleanup, label %vector.body
1767
1768for.cond.cleanup: ; preds = %vector.body, %entry
1769 %s.0.lcssa = phi i32 [ 0, %entry ], [ %4, %vector.body ]
1770 ret i32 %s.0.lcssa
1771}
1772
1773define i32 @mla4i32(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32 %n) {
1774; CHECK-LABEL: mla4i32:
1775; CHECK: @ %bb.0: @ %entry
1776; CHECK-NEXT: .save {r7, lr}
1777; CHECK-NEXT: push {r7, lr}
1778; CHECK-NEXT: cbz r2, .LBB18_4
1779; CHECK-NEXT: @ %bb.1: @ %vector.ph
1780; CHECK-NEXT: adds r3, r2, #3
1781; CHECK-NEXT: bic r3, r3, #3
1782; CHECK-NEXT: sub.w r12, r3, #4
1783; CHECK-NEXT: movs r3, #1
1784; CHECK-NEXT: add.w lr, r3, r12, lsr #2
1785; CHECK-NEXT: sub.w r12, r2, #1
1786; CHECK-NEXT: adr r2, .LCPI18_0
1787; CHECK-NEXT: movs r3, #0
1788; CHECK-NEXT: vldrw.u32 q0, [r2]
1789; CHECK-NEXT: vdup.32 q1, r12
1790; CHECK-NEXT: movs r2, #0
1791; CHECK-NEXT: dls lr, lr
1792; CHECK-NEXT: .LBB18_2: @ %vector.body
1793; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1794; CHECK-NEXT: vadd.i32 q2, q0, r3
1795; CHECK-NEXT: vdup.32 q3, r3
1796; CHECK-NEXT: vcmp.u32 hi, q3, q2
1797; CHECK-NEXT: adds r3, #4
1798; CHECK-NEXT: vpnot
1799; CHECK-NEXT: vpstttt
1800; CHECK-NEXT: vcmpt.u32 cs, q1, q2
1801; CHECK-NEXT: vldrwt.u32 q2, [r0], #16
1802; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
1803; CHECK-NEXT: vmlavat.u32 r2, q3, q2
1804; CHECK-NEXT: le lr, .LBB18_2
1805; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
1806; CHECK-NEXT: mov r0, r2
1807; CHECK-NEXT: pop {r7, pc}
1808; CHECK-NEXT: .LBB18_4:
1809; CHECK-NEXT: movs r2, #0
1810; CHECK-NEXT: mov r0, r2
1811; CHECK-NEXT: pop {r7, pc}
1812; CHECK-NEXT: .p2align 4
1813; CHECK-NEXT: @ %bb.5:
1814; CHECK-NEXT: .LCPI18_0:
1815; CHECK-NEXT: .long 0 @ 0x0
1816; CHECK-NEXT: .long 1 @ 0x1
1817; CHECK-NEXT: .long 2 @ 0x2
1818; CHECK-NEXT: .long 3 @ 0x3
1819entry:
1820 %cmp8.not = icmp eq i32 %n, 0
1821 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
1822
1823vector.ph: ; preds = %entry
1824 %n.rnd.up = add i32 %n, 3
1825 %n.vec = and i32 %n.rnd.up, -4
1826 %trip.count.minus.1 = add i32 %n, -1
1827 br label %vector.body
1828
1829vector.body: ; preds = %vector.body, %vector.ph
1830 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1831 %vec.phi = phi i32 [ 0, %vector.ph ], [ %7, %vector.body ]
1832 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
1833 %0 = getelementptr inbounds i32, i32* %x, i32 %index
1834 %1 = bitcast i32* %0 to <4 x i32>*
1835 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1836 %2 = getelementptr inbounds i32, i32* %y, i32 %index
1837 %3 = bitcast i32* %2 to <4 x i32>*
1838 %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
1839 %4 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
1840 %5 = select <4 x i1> %active.lane.mask, <4 x i32> %4, <4 x i32> zeroinitializer
1841 %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
1842 %7 = add i32 %6, %vec.phi
1843 %index.next = add i32 %index, 4
1844 %8 = icmp eq i32 %index.next, %n.vec
1845 br i1 %8, label %for.cond.cleanup, label %vector.body
1846
1847for.cond.cleanup: ; preds = %vector.body, %entry
1848 %s.0.lcssa = phi i32 [ 0, %entry ], [ %7, %vector.body ]
1849 ret i32 %s.0.lcssa
1850}
1851
1852define i32 @add8i32(i16* noalias nocapture readonly %x, i32 %n) {
1853; CHECK-LABEL: add8i32:
1854; CHECK: @ %bb.0: @ %entry
1855; CHECK-NEXT: .save {r7, lr}
1856; CHECK-NEXT: push {r7, lr}
1857; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
1858; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
1859; CHECK-NEXT: .pad #40
1860; CHECK-NEXT: sub sp, #40
1861; CHECK-NEXT: cmp r1, #0
1862; CHECK-NEXT: beq.w .LBB19_3
1863; CHECK-NEXT: @ %bb.1: @ %vector.ph
1864; CHECK-NEXT: adds r2, r1, #7
1865; CHECK-NEXT: movs r3, #1
1866; CHECK-NEXT: bic r2, r2, #7
1867; CHECK-NEXT: vmov.i8 q1, #0x0
1868; CHECK-NEXT: subs r2, #8
1869; CHECK-NEXT: vmov.i8 q2, #0xff
1870; CHECK-NEXT: add.w lr, r3, r2, lsr #3
1871; CHECK-NEXT: adr r3, .LCPI19_0
1872; CHECK-NEXT: vldrw.u32 q0, [r3]
1873; CHECK-NEXT: adr r3, .LCPI19_1
1874; CHECK-NEXT: subs r2, r1, #1
1875; CHECK-NEXT: dls lr, lr
1876; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
1877; CHECK-NEXT: vldrw.u32 q0, [r3]
1878; CHECK-NEXT: vdup.32 q4, r2
1879; CHECK-NEXT: movs r1, #0
1880; CHECK-NEXT: movs r2, #0
1881; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
1882; CHECK-NEXT: .LBB19_2: @ %vector.body
1883; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
1884; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
1885; CHECK-NEXT: vldrw.u32 q3, [sp] @ 16-byte Reload
1886; CHECK-NEXT: vadd.i32 q5, q0, r1
1887; CHECK-NEXT: vdup.32 q0, r1
1888; CHECK-NEXT: vcmp.u32 hi, q0, q5
1889; CHECK-NEXT: vadd.i32 q6, q3, r1
1890; CHECK-NEXT: vpsel q7, q2, q1
1891; CHECK-NEXT: vcmp.u32 hi, q0, q6
1892; CHECK-NEXT: vpsel q3, q2, q1
1893; CHECK-NEXT: adds r1, #8
1894; CHECK-NEXT: vmov r3, s12
1895; CHECK-NEXT: vmov.16 q0[0], r3
1896; CHECK-NEXT: vmov r3, s13
1897; CHECK-NEXT: vmov.16 q0[1], r3
1898; CHECK-NEXT: vmov r3, s14
1899; CHECK-NEXT: vmov.16 q0[2], r3
1900; CHECK-NEXT: vmov r3, s15
1901; CHECK-NEXT: vmov.16 q0[3], r3
1902; CHECK-NEXT: vmov r3, s28
1903; CHECK-NEXT: vmov.16 q0[4], r3
1904; CHECK-NEXT: vmov r3, s29
1905; CHECK-NEXT: vmov.16 q0[5], r3
1906; CHECK-NEXT: vmov r3, s30
1907; CHECK-NEXT: vmov.16 q0[6], r3
1908; CHECK-NEXT: vmov r3, s31
1909; CHECK-NEXT: vmov.16 q0[7], r3
1910; CHECK-NEXT: vcmp.i16 ne, q0, zr
1911; CHECK-NEXT: vstr p0, [sp, #36] @ 4-byte Spill
1912; CHECK-NEXT: vcmp.u32 cs, q4, q5
1913; CHECK-NEXT: vpsel q5, q2, q1
1914; CHECK-NEXT: vcmp.u32 cs, q4, q6
1915; CHECK-NEXT: vpsel q0, q2, q1
1916; CHECK-NEXT: vldr p0, [sp, #36] @ 4-byte Reload
1917; CHECK-NEXT: vmov r3, s0
1918; CHECK-NEXT: vmov.16 q6[0], r3
1919; CHECK-NEXT: vmov r3, s1
1920; CHECK-NEXT: vmov.16 q6[1], r3
1921; CHECK-NEXT: vmov r3, s2
1922; CHECK-NEXT: vmov.16 q6[2], r3
1923; CHECK-NEXT: vmov r3, s3
1924; CHECK-NEXT: vmov.16 q6[3], r3
1925; CHECK-NEXT: vmov r3, s20
1926; CHECK-NEXT: vmov.16 q6[4], r3
1927; CHECK-NEXT: vmov r3, s21
1928; CHECK-NEXT: vmov.16 q6[5], r3
1929; CHECK-NEXT: vmov r3, s22
1930; CHECK-NEXT: vmov.16 q6[6], r3
1931; CHECK-NEXT: vmov r3, s23
1932; CHECK-NEXT: vpnot
1933; CHECK-NEXT: vmov.16 q6[7], r3
1934; CHECK-NEXT: vpsttt
1935; CHECK-NEXT: vcmpt.i16 ne, q6, zr
1936; CHECK-NEXT: vldrht.u16 q0, [r0], #16
1937; CHECK-NEXT: vaddvat.s16 r2, q0
1938; CHECK-NEXT: le lr, .LBB19_2
1939; CHECK-NEXT: b .LBB19_4
1940; CHECK-NEXT: .LBB19_3:
1941; CHECK-NEXT: movs r2, #0
1942; CHECK-NEXT: .LBB19_4: @ %for.cond.cleanup
1943; CHECK-NEXT: mov r0, r2
1944; CHECK-NEXT: add sp, #40
1945; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
1946; CHECK-NEXT: pop {r7, pc}
1947; CHECK-NEXT: .p2align 4
1948; CHECK-NEXT: @ %bb.5:
1949; CHECK-NEXT: .LCPI19_0:
1950; CHECK-NEXT: .long 4 @ 0x4
1951; CHECK-NEXT: .long 5 @ 0x5
1952; CHECK-NEXT: .long 6 @ 0x6
1953; CHECK-NEXT: .long 7 @ 0x7
1954; CHECK-NEXT: .LCPI19_1:
1955; CHECK-NEXT: .long 0 @ 0x0
1956; CHECK-NEXT: .long 1 @ 0x1
1957; CHECK-NEXT: .long 2 @ 0x2
1958; CHECK-NEXT: .long 3 @ 0x3
1959entry:
1960 %cmp6.not = icmp eq i32 %n, 0
1961 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
1962
1963vector.ph: ; preds = %entry
1964 %n.rnd.up = add i32 %n, 7
1965 %n.vec = and i32 %n.rnd.up, -8
1966 %trip.count.minus.1 = add i32 %n, -1
1967 br label %vector.body
1968
1969vector.body: ; preds = %vector.body, %vector.ph
1970 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
1971 %vec.phi = phi i32 [ 0, %vector.ph ], [ %5, %vector.body ]
1972 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
1973 %0 = getelementptr inbounds i16, i16* %x, i32 %index
1974 %1 = bitcast i16* %0 to <8 x i16>*
1975 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
1976 %2 = sext <8 x i16> %wide.masked.load to <8 x i32>
1977 %3 = select <8 x i1> %active.lane.mask, <8 x i32> %2, <8 x i32> zeroinitializer
1978 %4 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %3)
1979 %5 = add i32 %4, %vec.phi
1980 %index.next = add i32 %index, 8
1981 %6 = icmp eq i32 %index.next, %n.vec
1982 br i1 %6, label %for.cond.cleanup, label %vector.body
1983
1984for.cond.cleanup: ; preds = %vector.body, %entry
1985 %s.0.lcssa = phi i32 [ 0, %entry ], [ %5, %vector.body ]
1986 ret i32 %s.0.lcssa
1987}
1988
1989define i32 @mla8i32(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
1990; CHECK-LABEL: mla8i32:
1991; CHECK: @ %bb.0: @ %entry
1992; CHECK-NEXT: .save {r7, lr}
1993; CHECK-NEXT: push {r7, lr}
1994; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
1995; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
1996; CHECK-NEXT: .pad #40
1997; CHECK-NEXT: sub sp, #40
1998; CHECK-NEXT: cmp r2, #0
1999; CHECK-NEXT: beq.w .LBB20_3
2000; CHECK-NEXT: @ %bb.1: @ %vector.ph
2001; CHECK-NEXT: adds r3, r2, #7
2002; CHECK-NEXT: vmov.i8 q1, #0x0
2003; CHECK-NEXT: bic r3, r3, #7
2004; CHECK-NEXT: vmov.i8 q2, #0xff
2005; CHECK-NEXT: sub.w r12, r3, #8
2006; CHECK-NEXT: movs r3, #1
2007; CHECK-NEXT: add.w lr, r3, r12, lsr #3
2008; CHECK-NEXT: sub.w r12, r2, #1
2009; CHECK-NEXT: adr r2, .LCPI20_0
2010; CHECK-NEXT: dls lr, lr
2011; CHECK-NEXT: vldrw.u32 q0, [r2]
2012; CHECK-NEXT: adr r2, .LCPI20_1
2013; CHECK-NEXT: vdup.32 q4, r12
2014; CHECK-NEXT: movs r3, #0
2015; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
2016; CHECK-NEXT: vldrw.u32 q0, [r2]
2017; CHECK-NEXT: mov.w r12, #0
2018; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
2019; CHECK-NEXT: .LBB20_2: @ %vector.body
2020; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
2021; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
2022; CHECK-NEXT: vldrw.u32 q3, [sp] @ 16-byte Reload
2023; CHECK-NEXT: vadd.i32 q5, q0, r3
2024; CHECK-NEXT: vdup.32 q0, r3
2025; CHECK-NEXT: vcmp.u32 hi, q0, q5
2026; CHECK-NEXT: vadd.i32 q6, q3, r3
2027; CHECK-NEXT: vpsel q7, q2, q1
2028; CHECK-NEXT: vcmp.u32 hi, q0, q6
2029; CHECK-NEXT: vpsel q3, q2, q1
2030; CHECK-NEXT: adds r3, #8
2031; CHECK-NEXT: vmov r2, s12
2032; CHECK-NEXT: vmov.16 q0[0], r2
2033; CHECK-NEXT: vmov r2, s13
2034; CHECK-NEXT: vmov.16 q0[1], r2
2035; CHECK-NEXT: vmov r2, s14
2036; CHECK-NEXT: vmov.16 q0[2], r2
2037; CHECK-NEXT: vmov r2, s15
2038; CHECK-NEXT: vmov.16 q0[3], r2
2039; CHECK-NEXT: vmov r2, s28
2040; CHECK-NEXT: vmov.16 q0[4], r2
2041; CHECK-NEXT: vmov r2, s29
2042; CHECK-NEXT: vmov.16 q0[5], r2
2043; CHECK-NEXT: vmov r2, s30
2044; CHECK-NEXT: vmov.16 q0[6], r2
2045; CHECK-NEXT: vmov r2, s31
2046; CHECK-NEXT: vmov.16 q0[7], r2
2047; CHECK-NEXT: vcmp.i16 ne, q0, zr
2048; CHECK-NEXT: vstr p0, [sp, #36] @ 4-byte Spill
2049; CHECK-NEXT: vcmp.u32 cs, q4, q5
2050; CHECK-NEXT: vpsel q5, q2, q1
2051; CHECK-NEXT: vcmp.u32 cs, q4, q6
2052; CHECK-NEXT: vpsel q0, q2, q1
2053; CHECK-NEXT: vldr p0, [sp, #36] @ 4-byte Reload
2054; CHECK-NEXT: vmov r2, s0
2055; CHECK-NEXT: vmov.16 q6[0], r2
2056; CHECK-NEXT: vmov r2, s1
2057; CHECK-NEXT: vmov.16 q6[1], r2
2058; CHECK-NEXT: vmov r2, s2
2059; CHECK-NEXT: vmov.16 q6[2], r2
2060; CHECK-NEXT: vmov r2, s3
2061; CHECK-NEXT: vmov.16 q6[3], r2
2062; CHECK-NEXT: vmov r2, s20
2063; CHECK-NEXT: vmov.16 q6[4], r2
2064; CHECK-NEXT: vmov r2, s21
2065; CHECK-NEXT: vmov.16 q6[5], r2
2066; CHECK-NEXT: vmov r2, s22
2067; CHECK-NEXT: vmov.16 q6[6], r2
2068; CHECK-NEXT: vmov r2, s23
2069; CHECK-NEXT: vpnot
2070; CHECK-NEXT: vmov.16 q6[7], r2
2071; CHECK-NEXT: vpstttt
2072; CHECK-NEXT: vcmpt.i16 ne, q6, zr
2073; CHECK-NEXT: vldrht.u16 q0, [r0], #16
2074; CHECK-NEXT: vldrht.u16 q3, [r1], #16
2075; CHECK-NEXT: vmlavat.s16 r12, q3, q0
2076; CHECK-NEXT: le lr, .LBB20_2
2077; CHECK-NEXT: b .LBB20_4
2078; CHECK-NEXT: .LBB20_3:
2079; CHECK-NEXT: mov.w r12, #0
2080; CHECK-NEXT: .LBB20_4: @ %for.cond.cleanup
2081; CHECK-NEXT: mov r0, r12
2082; CHECK-NEXT: add sp, #40
2083; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
2084; CHECK-NEXT: pop {r7, pc}
2085; CHECK-NEXT: .p2align 4
2086; CHECK-NEXT: @ %bb.5:
2087; CHECK-NEXT: .LCPI20_0:
2088; CHECK-NEXT: .long 4 @ 0x4
2089; CHECK-NEXT: .long 5 @ 0x5
2090; CHECK-NEXT: .long 6 @ 0x6
2091; CHECK-NEXT: .long 7 @ 0x7
2092; CHECK-NEXT: .LCPI20_1:
2093; CHECK-NEXT: .long 0 @ 0x0
2094; CHECK-NEXT: .long 1 @ 0x1
2095; CHECK-NEXT: .long 2 @ 0x2
2096; CHECK-NEXT: .long 3 @ 0x3
2097entry:
2098 %cmp9.not = icmp eq i32 %n, 0
2099 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
2100
2101vector.ph: ; preds = %entry
2102 %n.rnd.up = add i32 %n, 7
2103 %n.vec = and i32 %n.rnd.up, -8
2104 %trip.count.minus.1 = add i32 %n, -1
2105 br label %vector.body
2106
2107vector.body: ; preds = %vector.body, %vector.ph
2108 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2109 %vec.phi = phi i32 [ 0, %vector.ph ], [ %9, %vector.body ]
2110 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
2111 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2112 %1 = bitcast i16* %0 to <8 x i16>*
2113 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2114 %2 = sext <8 x i16> %wide.masked.load to <8 x i32>
2115 %3 = getelementptr inbounds i16, i16* %y, i32 %index
2116 %4 = bitcast i16* %3 to <8 x i16>*
2117 %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2118 %5 = sext <8 x i16> %wide.masked.load14 to <8 x i32>
2119 %6 = mul nsw <8 x i32> %5, %2
2120 %7 = select <8 x i1> %active.lane.mask, <8 x i32> %6, <8 x i32> zeroinitializer
2121 %8 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %7)
2122 %9 = add i32 %8, %vec.phi
2123 %index.next = add i32 %index, 8
2124 %10 = icmp eq i32 %index.next, %n.vec
2125 br i1 %10, label %for.cond.cleanup, label %vector.body
2126
2127for.cond.cleanup: ; preds = %vector.body, %entry
2128 %s.0.lcssa = phi i32 [ 0, %entry ], [ %9, %vector.body ]
2129 ret i32 %s.0.lcssa
2130}
2131
2132define i32 @add16i32(i8* noalias nocapture readonly %x, i32 %n) {
2133; CHECK-LABEL: add16i32:
2134; CHECK: @ %bb.0: @ %entry
2135; CHECK-NEXT: .save {r7, lr}
2136; CHECK-NEXT: push {r7, lr}
2137; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
2138; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
2139; CHECK-NEXT: .pad #136
2140; CHECK-NEXT: sub sp, #136
2141; CHECK-NEXT: cmp r1, #0
2142; CHECK-NEXT: beq.w .LBB21_3
2143; CHECK-NEXT: @ %bb.1: @ %vector.ph
2144; CHECK-NEXT: add.w r2, r1, #15
2145; CHECK-NEXT: movs r3, #1
2146; CHECK-NEXT: bic r2, r2, #15
2147; CHECK-NEXT: vmov.i8 q7, #0x0
2148; CHECK-NEXT: subs r2, #16
2149; CHECK-NEXT: vmov.i8 q2, #0xff
2150; CHECK-NEXT: add.w lr, r3, r2, lsr #4
2151; CHECK-NEXT: adr r3, .LCPI21_0
2152; CHECK-NEXT: vldrw.u32 q0, [r3]
2153; CHECK-NEXT: adr r3, .LCPI21_1
2154; CHECK-NEXT: dls lr, lr
2155; CHECK-NEXT: subs r2, r1, #1
2156; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
2157; CHECK-NEXT: vldrw.u32 q0, [r3]
2158; CHECK-NEXT: adr r3, .LCPI21_2
2159; CHECK-NEXT: movs r1, #0
2160; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
2161; CHECK-NEXT: vldrw.u32 q0, [r3]
2162; CHECK-NEXT: adr r3, .LCPI21_3
2163; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
2164; CHECK-NEXT: vldrw.u32 q0, [r3]
2165; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
2166; CHECK-NEXT: vdup.32 q0, r2
2167; CHECK-NEXT: movs r2, #0
2168; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
2169; CHECK-NEXT: .LBB21_2: @ %vector.body
2170; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
2171; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
2172; CHECK-NEXT: vdup.32 q6, r1
2173; CHECK-NEXT: vadd.i32 q0, q0, r1
2174; CHECK-NEXT: vcmp.u32 hi, q6, q0
2175; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
2176; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
2177; CHECK-NEXT: vpsel q3, q2, q7
2178; CHECK-NEXT: vadd.i32 q0, q0, r1
2179; CHECK-NEXT: vcmp.u32 hi, q6, q0
2180; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
2181; CHECK-NEXT: vpsel q0, q2, q7
2182; CHECK-NEXT: vmov r3, s0
2183; CHECK-NEXT: vmov.16 q4[0], r3
2184; CHECK-NEXT: vmov r3, s1
2185; CHECK-NEXT: vmov.16 q4[1], r3
2186; CHECK-NEXT: vmov r3, s2
2187; CHECK-NEXT: vmov.16 q4[2], r3
2188; CHECK-NEXT: vmov r3, s3
2189; CHECK-NEXT: vmov.16 q4[3], r3
2190; CHECK-NEXT: vmov r3, s12
2191; CHECK-NEXT: vmov.16 q4[4], r3
2192; CHECK-NEXT: vmov r3, s13
2193; CHECK-NEXT: vmov.16 q4[5], r3
2194; CHECK-NEXT: vmov r3, s14
2195; CHECK-NEXT: vmov.16 q4[6], r3
2196; CHECK-NEXT: vmov r3, s15
2197; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
2198; CHECK-NEXT: vmov.16 q4[7], r3
2199; CHECK-NEXT: vcmp.i16 ne, q4, zr
2200; CHECK-NEXT: vadd.i32 q3, q0, r1
2201; CHECK-NEXT: vpsel q5, q2, q7
2202; CHECK-NEXT: vcmp.u32 hi, q6, q3
2203; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
2204; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
2205; CHECK-NEXT: vadd.i32 q4, q0, r1
2206; CHECK-NEXT: adds r1, #16
2207; CHECK-NEXT: vcmp.u32 hi, q6, q4
2208; CHECK-NEXT: vpsel q0, q2, q7
2209; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
2210; CHECK-NEXT: vmov r3, s0
2211; CHECK-NEXT: vmov.16 q6[0], r3
2212; CHECK-NEXT: vmov r3, s1
2213; CHECK-NEXT: vmov.16 q6[1], r3
2214; CHECK-NEXT: vmov r3, s2
2215; CHECK-NEXT: vmov.16 q6[2], r3
2216; CHECK-NEXT: vmov r3, s3
2217; CHECK-NEXT: vpsel q1, q2, q7
2218; CHECK-NEXT: vmov.16 q6[3], r3
2219; CHECK-NEXT: vmov r3, s4
2220; CHECK-NEXT: vmov.16 q6[4], r3
2221; CHECK-NEXT: vmov r3, s5
2222; CHECK-NEXT: vmov.16 q6[5], r3
2223; CHECK-NEXT: vmov r3, s6
2224; CHECK-NEXT: vmov.16 q6[6], r3
2225; CHECK-NEXT: vmov r3, s7
2226; CHECK-NEXT: vmov.16 q6[7], r3
2227; CHECK-NEXT: vcmp.i16 ne, q6, zr
2228; CHECK-NEXT: vpsel q0, q2, q7
2229; CHECK-NEXT: vmov.u16 r3, q0[0]
2230; CHECK-NEXT: vmov.8 q6[0], r3
2231; CHECK-NEXT: vmov.u16 r3, q0[1]
2232; CHECK-NEXT: vmov.8 q6[1], r3
2233; CHECK-NEXT: vmov.u16 r3, q0[2]
2234; CHECK-NEXT: vmov.8 q6[2], r3
2235; CHECK-NEXT: vmov.u16 r3, q0[3]
2236; CHECK-NEXT: vmov.8 q6[3], r3
2237; CHECK-NEXT: vmov.u16 r3, q0[4]
2238; CHECK-NEXT: vmov.8 q6[4], r3
2239; CHECK-NEXT: vmov.u16 r3, q0[5]
2240; CHECK-NEXT: vmov.8 q6[5], r3
2241; CHECK-NEXT: vmov.u16 r3, q0[6]
2242; CHECK-NEXT: vmov.8 q6[6], r3
2243; CHECK-NEXT: vmov.u16 r3, q0[7]
2244; CHECK-NEXT: vmov.8 q6[7], r3
2245; CHECK-NEXT: vmov.u16 r3, q5[0]
2246; CHECK-NEXT: vmov.8 q6[8], r3
2247; CHECK-NEXT: vmov.u16 r3, q5[1]
2248; CHECK-NEXT: vmov.8 q6[9], r3
2249; CHECK-NEXT: vmov.u16 r3, q5[2]
2250; CHECK-NEXT: vmov.8 q6[10], r3
2251; CHECK-NEXT: vmov.u16 r3, q5[3]
2252; CHECK-NEXT: vmov.8 q6[11], r3
2253; CHECK-NEXT: vmov.u16 r3, q5[4]
2254; CHECK-NEXT: vmov.8 q6[12], r3
2255; CHECK-NEXT: vmov.u16 r3, q5[5]
2256; CHECK-NEXT: vmov.8 q6[13], r3
2257; CHECK-NEXT: vmov.u16 r3, q5[6]
2258; CHECK-NEXT: vmov.8 q6[14], r3
2259; CHECK-NEXT: vmov.u16 r3, q5[7]
2260; CHECK-NEXT: vmov.8 q6[15], r3
2261; CHECK-NEXT: vcmp.i8 ne, q6, zr
2262; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
2263; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
2264; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
2265; CHECK-NEXT: vcmp.u32 cs, q6, q0
2266; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
2267; CHECK-NEXT: vpsel q5, q2, q7
2268; CHECK-NEXT: vcmp.u32 cs, q6, q0
2269; CHECK-NEXT: vpsel q1, q2, q7
2270; CHECK-NEXT: vmov r3, s4
2271; CHECK-NEXT: vmov.16 q0[0], r3
2272; CHECK-NEXT: vmov r3, s5
2273; CHECK-NEXT: vmov.16 q0[1], r3
2274; CHECK-NEXT: vmov r3, s6
2275; CHECK-NEXT: vmov.16 q0[2], r3
2276; CHECK-NEXT: vmov r3, s7
2277; CHECK-NEXT: vmov.16 q0[3], r3
2278; CHECK-NEXT: vmov r3, s20
2279; CHECK-NEXT: vmov.16 q0[4], r3
2280; CHECK-NEXT: vmov r3, s21
2281; CHECK-NEXT: vmov.16 q0[5], r3
2282; CHECK-NEXT: vmov r3, s22
2283; CHECK-NEXT: vmov.16 q0[6], r3
2284; CHECK-NEXT: vmov r3, s23
2285; CHECK-NEXT: vmov.16 q0[7], r3
2286; CHECK-NEXT: vcmp.i16 ne, q0, zr
2287; CHECK-NEXT: vpsel q0, q2, q7
2288; CHECK-NEXT: vcmp.u32 cs, q6, q3
2289; CHECK-NEXT: vpsel q3, q2, q7
2290; CHECK-NEXT: vcmp.u32 cs, q6, q4
2291; CHECK-NEXT: vpsel q1, q2, q7
2292; CHECK-NEXT: vmov r3, s4
2293; CHECK-NEXT: vmov.16 q4[0], r3
2294; CHECK-NEXT: vmov r3, s5
2295; CHECK-NEXT: vmov.16 q4[1], r3
2296; CHECK-NEXT: vmov r3, s6
2297; CHECK-NEXT: vmov.16 q4[2], r3
2298; CHECK-NEXT: vmov r3, s7
2299; CHECK-NEXT: vmov.16 q4[3], r3
2300; CHECK-NEXT: vmov r3, s12
2301; CHECK-NEXT: vmov.16 q4[4], r3
2302; CHECK-NEXT: vmov r3, s13
2303; CHECK-NEXT: vmov.16 q4[5], r3
2304; CHECK-NEXT: vmov r3, s14
2305; CHECK-NEXT: vmov.16 q4[6], r3
2306; CHECK-NEXT: vmov r3, s15
2307; CHECK-NEXT: vmov.16 q4[7], r3
2308; CHECK-NEXT: vcmp.i16 ne, q4, zr
2309; CHECK-NEXT: vpsel q4, q2, q7
2310; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
2311; CHECK-NEXT: vmov.u16 r3, q4[0]
2312; CHECK-NEXT: vmov.8 q3[0], r3
2313; CHECK-NEXT: vmov.u16 r3, q4[1]
2314; CHECK-NEXT: vmov.8 q3[1], r3
2315; CHECK-NEXT: vmov.u16 r3, q4[2]
2316; CHECK-NEXT: vmov.8 q3[2], r3
2317; CHECK-NEXT: vmov.u16 r3, q4[3]
2318; CHECK-NEXT: vmov.8 q3[3], r3
2319; CHECK-NEXT: vmov.u16 r3, q4[4]
2320; CHECK-NEXT: vmov.8 q3[4], r3
2321; CHECK-NEXT: vmov.u16 r3, q4[5]
2322; CHECK-NEXT: vmov.8 q3[5], r3
2323; CHECK-NEXT: vmov.u16 r3, q4[6]
2324; CHECK-NEXT: vmov.8 q3[6], r3
2325; CHECK-NEXT: vmov.u16 r3, q4[7]
2326; CHECK-NEXT: vmov.8 q3[7], r3
2327; CHECK-NEXT: vmov.u16 r3, q0[0]
2328; CHECK-NEXT: vmov.8 q3[8], r3
2329; CHECK-NEXT: vmov.u16 r3, q0[1]
2330; CHECK-NEXT: vmov.8 q3[9], r3
2331; CHECK-NEXT: vmov.u16 r3, q0[2]
2332; CHECK-NEXT: vmov.8 q3[10], r3
2333; CHECK-NEXT: vmov.u16 r3, q0[3]
2334; CHECK-NEXT: vmov.8 q3[11], r3
2335; CHECK-NEXT: vmov.u16 r3, q0[4]
2336; CHECK-NEXT: vmov.8 q3[12], r3
2337; CHECK-NEXT: vmov.u16 r3, q0[5]
2338; CHECK-NEXT: vmov.8 q3[13], r3
2339; CHECK-NEXT: vmov.u16 r3, q0[6]
2340; CHECK-NEXT: vmov.8 q3[14], r3
2341; CHECK-NEXT: vmov.u16 r3, q0[7]
2342; CHECK-NEXT: vpnot
2343; CHECK-NEXT: vmov.8 q3[15], r3
2344; CHECK-NEXT: vpsttt
2345; CHECK-NEXT: vcmpt.i8 ne, q3, zr
2346; CHECK-NEXT: vldrbt.u8 q0, [r0], #16
2347; CHECK-NEXT: vaddvat.u8 r2, q0
2348; CHECK-NEXT: le lr, .LBB21_2
2349; CHECK-NEXT: b .LBB21_4
2350; CHECK-NEXT: .LBB21_3:
2351; CHECK-NEXT: movs r2, #0
2352; CHECK-NEXT: .LBB21_4: @ %for.cond.cleanup
2353; CHECK-NEXT: mov r0, r2
2354; CHECK-NEXT: add sp, #136
2355; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
2356; CHECK-NEXT: pop {r7, pc}
2357; CHECK-NEXT: .p2align 4
2358; CHECK-NEXT: @ %bb.5:
2359; CHECK-NEXT: .LCPI21_0:
2360; CHECK-NEXT: .long 12 @ 0xc
2361; CHECK-NEXT: .long 13 @ 0xd
2362; CHECK-NEXT: .long 14 @ 0xe
2363; CHECK-NEXT: .long 15 @ 0xf
2364; CHECK-NEXT: .LCPI21_1:
2365; CHECK-NEXT: .long 8 @ 0x8
2366; CHECK-NEXT: .long 9 @ 0x9
2367; CHECK-NEXT: .long 10 @ 0xa
2368; CHECK-NEXT: .long 11 @ 0xb
2369; CHECK-NEXT: .LCPI21_2:
2370; CHECK-NEXT: .long 4 @ 0x4
2371; CHECK-NEXT: .long 5 @ 0x5
2372; CHECK-NEXT: .long 6 @ 0x6
2373; CHECK-NEXT: .long 7 @ 0x7
2374; CHECK-NEXT: .LCPI21_3:
2375; CHECK-NEXT: .long 0 @ 0x0
2376; CHECK-NEXT: .long 1 @ 0x1
2377; CHECK-NEXT: .long 2 @ 0x2
2378; CHECK-NEXT: .long 3 @ 0x3
2379entry:
2380 %cmp6.not = icmp eq i32 %n, 0
2381 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
2382
2383vector.ph: ; preds = %entry
2384 %n.rnd.up = add i32 %n, 15
2385 %n.vec = and i32 %n.rnd.up, -16
2386 %trip.count.minus.1 = add i32 %n, -1
2387 br label %vector.body
2388
2389vector.body: ; preds = %vector.body, %vector.ph
2390 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2391 %vec.phi = phi i32 [ 0, %vector.ph ], [ %5, %vector.body ]
2392 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
2393 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2394 %1 = bitcast i8* %0 to <16 x i8>*
2395 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2396 %2 = zext <16 x i8> %wide.masked.load to <16 x i32>
2397 %3 = select <16 x i1> %active.lane.mask, <16 x i32> %2, <16 x i32> zeroinitializer
2398 %4 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %3)
2399 %5 = add i32 %4, %vec.phi
2400 %index.next = add i32 %index, 16
2401 %6 = icmp eq i32 %index.next, %n.vec
2402 br i1 %6, label %for.cond.cleanup, label %vector.body
2403
2404for.cond.cleanup: ; preds = %vector.body, %entry
2405 %s.0.lcssa = phi i32 [ 0, %entry ], [ %5, %vector.body ]
2406 ret i32 %s.0.lcssa
2407}
2408
2409define i32 @mla16i32(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
2410; CHECK-LABEL: mla16i32:
2411; CHECK: @ %bb.0: @ %entry
2412; CHECK-NEXT: .save {r7, lr}
2413; CHECK-NEXT: push {r7, lr}
2414; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
2415; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
2416; CHECK-NEXT: .pad #136
2417; CHECK-NEXT: sub sp, #136
2418; CHECK-NEXT: cmp r2, #0
2419; CHECK-NEXT: beq.w .LBB22_3
2420; CHECK-NEXT: @ %bb.1: @ %vector.ph
2421; CHECK-NEXT: add.w r3, r2, #15
2422; CHECK-NEXT: vmov.i8 q7, #0x0
2423; CHECK-NEXT: bic r3, r3, #15
2424; CHECK-NEXT: vmov.i8 q2, #0xff
2425; CHECK-NEXT: sub.w r12, r3, #16
2426; CHECK-NEXT: movs r3, #1
2427; CHECK-NEXT: add.w lr, r3, r12, lsr #4
2428; CHECK-NEXT: sub.w r12, r2, #1
2429; CHECK-NEXT: adr r2, .LCPI22_0
2430; CHECK-NEXT: dls lr, lr
2431; CHECK-NEXT: vldrw.u32 q0, [r2]
2432; CHECK-NEXT: adr r2, .LCPI22_1
2433; CHECK-NEXT: movs r3, #0
2434; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
2435; CHECK-NEXT: vldrw.u32 q0, [r2]
2436; CHECK-NEXT: adr r2, .LCPI22_2
2437; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
2438; CHECK-NEXT: vldrw.u32 q0, [r2]
2439; CHECK-NEXT: adr r2, .LCPI22_3
2440; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
2441; CHECK-NEXT: vldrw.u32 q0, [r2]
2442; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
2443; CHECK-NEXT: vdup.32 q0, r12
2444; CHECK-NEXT: mov.w r12, #0
2445; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
2446; CHECK-NEXT: .LBB22_2: @ %vector.body
2447; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
2448; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
2449; CHECK-NEXT: vdup.32 q6, r3
2450; CHECK-NEXT: vadd.i32 q0, q0, r3
2451; CHECK-NEXT: vcmp.u32 hi, q6, q0
2452; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
2453; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
2454; CHECK-NEXT: vpsel q3, q2, q7
2455; CHECK-NEXT: vadd.i32 q0, q0, r3
2456; CHECK-NEXT: vcmp.u32 hi, q6, q0
2457; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
2458; CHECK-NEXT: vpsel q0, q2, q7
2459; CHECK-NEXT: vmov r2, s0
2460; CHECK-NEXT: vmov.16 q4[0], r2
2461; CHECK-NEXT: vmov r2, s1
2462; CHECK-NEXT: vmov.16 q4[1], r2
2463; CHECK-NEXT: vmov r2, s2
2464; CHECK-NEXT: vmov.16 q4[2], r2
2465; CHECK-NEXT: vmov r2, s3
2466; CHECK-NEXT: vmov.16 q4[3], r2
2467; CHECK-NEXT: vmov r2, s12
2468; CHECK-NEXT: vmov.16 q4[4], r2
2469; CHECK-NEXT: vmov r2, s13
2470; CHECK-NEXT: vmov.16 q4[5], r2
2471; CHECK-NEXT: vmov r2, s14
2472; CHECK-NEXT: vmov.16 q4[6], r2
2473; CHECK-NEXT: vmov r2, s15
2474; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
2475; CHECK-NEXT: vmov.16 q4[7], r2
2476; CHECK-NEXT: vcmp.i16 ne, q4, zr
2477; CHECK-NEXT: vadd.i32 q3, q0, r3
2478; CHECK-NEXT: vpsel q5, q2, q7
2479; CHECK-NEXT: vcmp.u32 hi, q6, q3
2480; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
2481; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
2482; CHECK-NEXT: vadd.i32 q4, q0, r3
2483; CHECK-NEXT: adds r3, #16
2484; CHECK-NEXT: vcmp.u32 hi, q6, q4
2485; CHECK-NEXT: vpsel q0, q2, q7
2486; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
2487; CHECK-NEXT: vmov r2, s0
2488; CHECK-NEXT: vmov.16 q6[0], r2
2489; CHECK-NEXT: vmov r2, s1
2490; CHECK-NEXT: vmov.16 q6[1], r2
2491; CHECK-NEXT: vmov r2, s2
2492; CHECK-NEXT: vmov.16 q6[2], r2
2493; CHECK-NEXT: vmov r2, s3
2494; CHECK-NEXT: vpsel q1, q2, q7
2495; CHECK-NEXT: vmov.16 q6[3], r2
2496; CHECK-NEXT: vmov r2, s4
2497; CHECK-NEXT: vmov.16 q6[4], r2
2498; CHECK-NEXT: vmov r2, s5
2499; CHECK-NEXT: vmov.16 q6[5], r2
2500; CHECK-NEXT: vmov r2, s6
2501; CHECK-NEXT: vmov.16 q6[6], r2
2502; CHECK-NEXT: vmov r2, s7
2503; CHECK-NEXT: vmov.16 q6[7], r2
2504; CHECK-NEXT: vcmp.i16 ne, q6, zr
2505; CHECK-NEXT: vpsel q0, q2, q7
2506; CHECK-NEXT: vmov.u16 r2, q0[0]
2507; CHECK-NEXT: vmov.8 q6[0], r2
2508; CHECK-NEXT: vmov.u16 r2, q0[1]
2509; CHECK-NEXT: vmov.8 q6[1], r2
2510; CHECK-NEXT: vmov.u16 r2, q0[2]
2511; CHECK-NEXT: vmov.8 q6[2], r2
2512; CHECK-NEXT: vmov.u16 r2, q0[3]
2513; CHECK-NEXT: vmov.8 q6[3], r2
2514; CHECK-NEXT: vmov.u16 r2, q0[4]
2515; CHECK-NEXT: vmov.8 q6[4], r2
2516; CHECK-NEXT: vmov.u16 r2, q0[5]
2517; CHECK-NEXT: vmov.8 q6[5], r2
2518; CHECK-NEXT: vmov.u16 r2, q0[6]
2519; CHECK-NEXT: vmov.8 q6[6], r2
2520; CHECK-NEXT: vmov.u16 r2, q0[7]
2521; CHECK-NEXT: vmov.8 q6[7], r2
2522; CHECK-NEXT: vmov.u16 r2, q5[0]
2523; CHECK-NEXT: vmov.8 q6[8], r2
2524; CHECK-NEXT: vmov.u16 r2, q5[1]
2525; CHECK-NEXT: vmov.8 q6[9], r2
2526; CHECK-NEXT: vmov.u16 r2, q5[2]
2527; CHECK-NEXT: vmov.8 q6[10], r2
2528; CHECK-NEXT: vmov.u16 r2, q5[3]
2529; CHECK-NEXT: vmov.8 q6[11], r2
2530; CHECK-NEXT: vmov.u16 r2, q5[4]
2531; CHECK-NEXT: vmov.8 q6[12], r2
2532; CHECK-NEXT: vmov.u16 r2, q5[5]
2533; CHECK-NEXT: vmov.8 q6[13], r2
2534; CHECK-NEXT: vmov.u16 r2, q5[6]
2535; CHECK-NEXT: vmov.8 q6[14], r2
2536; CHECK-NEXT: vmov.u16 r2, q5[7]
2537; CHECK-NEXT: vmov.8 q6[15], r2
2538; CHECK-NEXT: vcmp.i8 ne, q6, zr
2539; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
2540; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
2541; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
2542; CHECK-NEXT: vcmp.u32 cs, q6, q0
2543; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
2544; CHECK-NEXT: vpsel q5, q2, q7
2545; CHECK-NEXT: vcmp.u32 cs, q6, q0
2546; CHECK-NEXT: vpsel q1, q2, q7
2547; CHECK-NEXT: vmov r2, s4
2548; CHECK-NEXT: vmov.16 q0[0], r2
2549; CHECK-NEXT: vmov r2, s5
2550; CHECK-NEXT: vmov.16 q0[1], r2
2551; CHECK-NEXT: vmov r2, s6
2552; CHECK-NEXT: vmov.16 q0[2], r2
2553; CHECK-NEXT: vmov r2, s7
2554; CHECK-NEXT: vmov.16 q0[3], r2
2555; CHECK-NEXT: vmov r2, s20
2556; CHECK-NEXT: vmov.16 q0[4], r2
2557; CHECK-NEXT: vmov r2, s21
2558; CHECK-NEXT: vmov.16 q0[5], r2
2559; CHECK-NEXT: vmov r2, s22
2560; CHECK-NEXT: vmov.16 q0[6], r2
2561; CHECK-NEXT: vmov r2, s23
2562; CHECK-NEXT: vmov.16 q0[7], r2
2563; CHECK-NEXT: vcmp.i16 ne, q0, zr
2564; CHECK-NEXT: vpsel q0, q2, q7
2565; CHECK-NEXT: vcmp.u32 cs, q6, q3
2566; CHECK-NEXT: vpsel q3, q2, q7
2567; CHECK-NEXT: vcmp.u32 cs, q6, q4
2568; CHECK-NEXT: vpsel q1, q2, q7
2569; CHECK-NEXT: vmov r2, s4
2570; CHECK-NEXT: vmov.16 q4[0], r2
2571; CHECK-NEXT: vmov r2, s5
2572; CHECK-NEXT: vmov.16 q4[1], r2
2573; CHECK-NEXT: vmov r2, s6
2574; CHECK-NEXT: vmov.16 q4[2], r2
2575; CHECK-NEXT: vmov r2, s7
2576; CHECK-NEXT: vmov.16 q4[3], r2
2577; CHECK-NEXT: vmov r2, s12
2578; CHECK-NEXT: vmov.16 q4[4], r2
2579; CHECK-NEXT: vmov r2, s13
2580; CHECK-NEXT: vmov.16 q4[5], r2
2581; CHECK-NEXT: vmov r2, s14
2582; CHECK-NEXT: vmov.16 q4[6], r2
2583; CHECK-NEXT: vmov r2, s15
2584; CHECK-NEXT: vmov.16 q4[7], r2
2585; CHECK-NEXT: vcmp.i16 ne, q4, zr
2586; CHECK-NEXT: vpsel q4, q2, q7
2587; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
2588; CHECK-NEXT: vmov.u16 r2, q4[0]
2589; CHECK-NEXT: vmov.8 q3[0], r2
2590; CHECK-NEXT: vmov.u16 r2, q4[1]
2591; CHECK-NEXT: vmov.8 q3[1], r2
2592; CHECK-NEXT: vmov.u16 r2, q4[2]
2593; CHECK-NEXT: vmov.8 q3[2], r2
2594; CHECK-NEXT: vmov.u16 r2, q4[3]
2595; CHECK-NEXT: vmov.8 q3[3], r2
2596; CHECK-NEXT: vmov.u16 r2, q4[4]
2597; CHECK-NEXT: vmov.8 q3[4], r2
2598; CHECK-NEXT: vmov.u16 r2, q4[5]
2599; CHECK-NEXT: vmov.8 q3[5], r2
2600; CHECK-NEXT: vmov.u16 r2, q4[6]
2601; CHECK-NEXT: vmov.8 q3[6], r2
2602; CHECK-NEXT: vmov.u16 r2, q4[7]
2603; CHECK-NEXT: vmov.8 q3[7], r2
2604; CHECK-NEXT: vmov.u16 r2, q0[0]
2605; CHECK-NEXT: vmov.8 q3[8], r2
2606; CHECK-NEXT: vmov.u16 r2, q0[1]
2607; CHECK-NEXT: vmov.8 q3[9], r2
2608; CHECK-NEXT: vmov.u16 r2, q0[2]
2609; CHECK-NEXT: vmov.8 q3[10], r2
2610; CHECK-NEXT: vmov.u16 r2, q0[3]
2611; CHECK-NEXT: vmov.8 q3[11], r2
2612; CHECK-NEXT: vmov.u16 r2, q0[4]
2613; CHECK-NEXT: vmov.8 q3[12], r2
2614; CHECK-NEXT: vmov.u16 r2, q0[5]
2615; CHECK-NEXT: vmov.8 q3[13], r2
2616; CHECK-NEXT: vmov.u16 r2, q0[6]
2617; CHECK-NEXT: vmov.8 q3[14], r2
2618; CHECK-NEXT: vmov.u16 r2, q0[7]
2619; CHECK-NEXT: vpnot
2620; CHECK-NEXT: vmov.8 q3[15], r2
2621; CHECK-NEXT: vpstttt
2622; CHECK-NEXT: vcmpt.i8 ne, q3, zr
2623; CHECK-NEXT: vldrbt.u8 q0, [r0], #16
2624; CHECK-NEXT: vldrbt.u8 q1, [r1], #16
2625; CHECK-NEXT: vmlavat.u8 r12, q1, q0
2626; CHECK-NEXT: le lr, .LBB22_2
2627; CHECK-NEXT: b .LBB22_4
2628; CHECK-NEXT: .LBB22_3:
2629; CHECK-NEXT: mov.w r12, #0
2630; CHECK-NEXT: .LBB22_4: @ %for.cond.cleanup
2631; CHECK-NEXT: mov r0, r12
2632; CHECK-NEXT: add sp, #136
2633; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
2634; CHECK-NEXT: pop {r7, pc}
2635; CHECK-NEXT: .p2align 4
2636; CHECK-NEXT: @ %bb.5:
2637; CHECK-NEXT: .LCPI22_0:
2638; CHECK-NEXT: .long 12 @ 0xc
2639; CHECK-NEXT: .long 13 @ 0xd
2640; CHECK-NEXT: .long 14 @ 0xe
2641; CHECK-NEXT: .long 15 @ 0xf
2642; CHECK-NEXT: .LCPI22_1:
2643; CHECK-NEXT: .long 8 @ 0x8
2644; CHECK-NEXT: .long 9 @ 0x9
2645; CHECK-NEXT: .long 10 @ 0xa
2646; CHECK-NEXT: .long 11 @ 0xb
2647; CHECK-NEXT: .LCPI22_2:
2648; CHECK-NEXT: .long 4 @ 0x4
2649; CHECK-NEXT: .long 5 @ 0x5
2650; CHECK-NEXT: .long 6 @ 0x6
2651; CHECK-NEXT: .long 7 @ 0x7
2652; CHECK-NEXT: .LCPI22_3:
2653; CHECK-NEXT: .long 0 @ 0x0
2654; CHECK-NEXT: .long 1 @ 0x1
2655; CHECK-NEXT: .long 2 @ 0x2
2656; CHECK-NEXT: .long 3 @ 0x3
2657entry:
2658 %cmp9.not = icmp eq i32 %n, 0
2659 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
2660
2661vector.ph: ; preds = %entry
2662 %n.rnd.up = add i32 %n, 15
2663 %n.vec = and i32 %n.rnd.up, -16
2664 %trip.count.minus.1 = add i32 %n, -1
2665 br label %vector.body
2666
2667vector.body: ; preds = %vector.body, %vector.ph
2668 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2669 %vec.phi = phi i32 [ 0, %vector.ph ], [ %9, %vector.body ]
2670 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
2671 %0 = getelementptr inbounds i8, i8* %x, i32 %index
2672 %1 = bitcast i8* %0 to <16 x i8>*
2673 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2674 %2 = zext <16 x i8> %wide.masked.load to <16 x i32>
2675 %3 = getelementptr inbounds i8, i8* %y, i32 %index
2676 %4 = bitcast i8* %3 to <16 x i8>*
2677 %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
2678 %5 = zext <16 x i8> %wide.masked.load14 to <16 x i32>
2679 %6 = mul nuw nsw <16 x i32> %5, %2
2680 %7 = select <16 x i1> %active.lane.mask, <16 x i32> %6, <16 x i32> zeroinitializer
2681 %8 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %7)
2682 %9 = add i32 %8, %vec.phi
2683 %index.next = add i32 %index, 16
2684 %10 = icmp eq i32 %index.next, %n.vec
2685 br i1 %10, label %for.cond.cleanup, label %vector.body
2686
2687for.cond.cleanup: ; preds = %vector.body, %entry
2688 %s.0.lcssa = phi i32 [ 0, %entry ], [ %9, %vector.body ]
2689 ret i32 %s.0.lcssa
2690}
2691
2692define signext i16 @add8i16(i16* noalias nocapture readonly %x, i32 %n) {
2693; CHECK-LABEL: add8i16:
2694; CHECK: @ %bb.0: @ %entry
2695; CHECK-NEXT: .save {r7, lr}
2696; CHECK-NEXT: push {r7, lr}
2697; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
2698; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
2699; CHECK-NEXT: .pad #40
2700; CHECK-NEXT: sub sp, #40
2701; CHECK-NEXT: cmp r1, #0
2702; CHECK-NEXT: beq.w .LBB23_3
2703; CHECK-NEXT: @ %bb.1: @ %vector.ph
2704; CHECK-NEXT: adds r2, r1, #7
2705; CHECK-NEXT: movs r3, #1
2706; CHECK-NEXT: bic r2, r2, #7
2707; CHECK-NEXT: vmov.i8 q1, #0x0
2708; CHECK-NEXT: subs r2, #8
2709; CHECK-NEXT: vmov.i8 q2, #0xff
2710; CHECK-NEXT: add.w lr, r3, r2, lsr #3
2711; CHECK-NEXT: adr r3, .LCPI23_0
2712; CHECK-NEXT: vldrw.u32 q0, [r3]
2713; CHECK-NEXT: adr r3, .LCPI23_1
2714; CHECK-NEXT: subs r2, r1, #1
2715; CHECK-NEXT: dls lr, lr
2716; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
2717; CHECK-NEXT: vldrw.u32 q0, [r3]
2718; CHECK-NEXT: vdup.32 q4, r2
2719; CHECK-NEXT: movs r1, #0
2720; CHECK-NEXT: movs r2, #0
2721; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
2722; CHECK-NEXT: .LBB23_2: @ %vector.body
2723; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
2724; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
2725; CHECK-NEXT: vldrw.u32 q3, [sp] @ 16-byte Reload
2726; CHECK-NEXT: vadd.i32 q5, q0, r1
2727; CHECK-NEXT: vdup.32 q0, r1
2728; CHECK-NEXT: vcmp.u32 hi, q0, q5
2729; CHECK-NEXT: vadd.i32 q6, q3, r1
2730; CHECK-NEXT: vpsel q7, q2, q1
2731; CHECK-NEXT: vcmp.u32 hi, q0, q6
2732; CHECK-NEXT: vpsel q3, q2, q1
2733; CHECK-NEXT: adds r1, #8
2734; CHECK-NEXT: vmov r3, s12
2735; CHECK-NEXT: vmov.16 q0[0], r3
2736; CHECK-NEXT: vmov r3, s13
2737; CHECK-NEXT: vmov.16 q0[1], r3
2738; CHECK-NEXT: vmov r3, s14
2739; CHECK-NEXT: vmov.16 q0[2], r3
2740; CHECK-NEXT: vmov r3, s15
2741; CHECK-NEXT: vmov.16 q0[3], r3
2742; CHECK-NEXT: vmov r3, s28
2743; CHECK-NEXT: vmov.16 q0[4], r3
2744; CHECK-NEXT: vmov r3, s29
2745; CHECK-NEXT: vmov.16 q0[5], r3
2746; CHECK-NEXT: vmov r3, s30
2747; CHECK-NEXT: vmov.16 q0[6], r3
2748; CHECK-NEXT: vmov r3, s31
2749; CHECK-NEXT: vmov.16 q0[7], r3
2750; CHECK-NEXT: vcmp.i16 ne, q0, zr
2751; CHECK-NEXT: vstr p0, [sp, #36] @ 4-byte Spill
2752; CHECK-NEXT: vcmp.u32 cs, q4, q5
2753; CHECK-NEXT: vpsel q5, q2, q1
2754; CHECK-NEXT: vcmp.u32 cs, q4, q6
2755; CHECK-NEXT: vpsel q0, q2, q1
2756; CHECK-NEXT: vldr p0, [sp, #36] @ 4-byte Reload
2757; CHECK-NEXT: vmov r3, s0
2758; CHECK-NEXT: vmov.16 q6[0], r3
2759; CHECK-NEXT: vmov r3, s1
2760; CHECK-NEXT: vmov.16 q6[1], r3
2761; CHECK-NEXT: vmov r3, s2
2762; CHECK-NEXT: vmov.16 q6[2], r3
2763; CHECK-NEXT: vmov r3, s3
2764; CHECK-NEXT: vmov.16 q6[3], r3
2765; CHECK-NEXT: vmov r3, s20
2766; CHECK-NEXT: vmov.16 q6[4], r3
2767; CHECK-NEXT: vmov r3, s21
2768; CHECK-NEXT: vmov.16 q6[5], r3
2769; CHECK-NEXT: vmov r3, s22
2770; CHECK-NEXT: vmov.16 q6[6], r3
2771; CHECK-NEXT: vmov r3, s23
2772; CHECK-NEXT: vpnot
2773; CHECK-NEXT: vmov.16 q6[7], r3
2774; CHECK-NEXT: vpsttt
2775; CHECK-NEXT: vcmpt.i16 ne, q6, zr
2776; CHECK-NEXT: vldrht.u16 q0, [r0], #16
2777; CHECK-NEXT: vaddvat.u16 r2, q0
2778; CHECK-NEXT: le lr, .LBB23_2
2779; CHECK-NEXT: b .LBB23_4
2780; CHECK-NEXT: .LBB23_3:
2781; CHECK-NEXT: movs r2, #0
2782; CHECK-NEXT: .LBB23_4: @ %for.cond.cleanup
2783; CHECK-NEXT: sxth r0, r2
2784; CHECK-NEXT: add sp, #40
2785; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
2786; CHECK-NEXT: pop {r7, pc}
2787; CHECK-NEXT: .p2align 4
2788; CHECK-NEXT: @ %bb.5:
2789; CHECK-NEXT: .LCPI23_0:
2790; CHECK-NEXT: .long 4 @ 0x4
2791; CHECK-NEXT: .long 5 @ 0x5
2792; CHECK-NEXT: .long 6 @ 0x6
2793; CHECK-NEXT: .long 7 @ 0x7
2794; CHECK-NEXT: .LCPI23_1:
2795; CHECK-NEXT: .long 0 @ 0x0
2796; CHECK-NEXT: .long 1 @ 0x1
2797; CHECK-NEXT: .long 2 @ 0x2
2798; CHECK-NEXT: .long 3 @ 0x3
2799entry:
2800 %cmp8.not = icmp eq i32 %n, 0
2801 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
2802
2803vector.ph: ; preds = %entry
2804 %n.rnd.up = add i32 %n, 7
2805 %n.vec = and i32 %n.rnd.up, -8
2806 %trip.count.minus.1 = add i32 %n, -1
2807 br label %vector.body
2808
2809vector.body: ; preds = %vector.body, %vector.ph
2810 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2811 %vec.phi = phi i16 [ 0, %vector.ph ], [ %4, %vector.body ]
2812 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
2813 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2814 %1 = bitcast i16* %0 to <8 x i16>*
2815 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2816 %2 = select <8 x i1> %active.lane.mask, <8 x i16> %wide.masked.load, <8 x i16> zeroinitializer
2817 %3 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %2)
2818 %4 = add i16 %3, %vec.phi
2819 %index.next = add i32 %index, 8
2820 %5 = icmp eq i32 %index.next, %n.vec
2821 br i1 %5, label %for.cond.cleanup, label %vector.body
2822
2823for.cond.cleanup: ; preds = %vector.body, %entry
2824 %s.0.lcssa = phi i16 [ 0, %entry ], [ %4, %vector.body ]
2825 ret i16 %s.0.lcssa
2826}
2827
2828define signext i16 @mla8i16(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
2829; CHECK-LABEL: mla8i16:
2830; CHECK: @ %bb.0: @ %entry
2831; CHECK-NEXT: .save {r7, lr}
2832; CHECK-NEXT: push {r7, lr}
2833; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
2834; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
2835; CHECK-NEXT: .pad #40
2836; CHECK-NEXT: sub sp, #40
2837; CHECK-NEXT: cmp r2, #0
2838; CHECK-NEXT: beq.w .LBB24_3
2839; CHECK-NEXT: @ %bb.1: @ %vector.ph
2840; CHECK-NEXT: adds r3, r2, #7
2841; CHECK-NEXT: vmov.i8 q1, #0x0
2842; CHECK-NEXT: bic r3, r3, #7
2843; CHECK-NEXT: vmov.i8 q2, #0xff
2844; CHECK-NEXT: sub.w r12, r3, #8
2845; CHECK-NEXT: movs r3, #1
2846; CHECK-NEXT: add.w lr, r3, r12, lsr #3
2847; CHECK-NEXT: adr r3, .LCPI24_0
2848; CHECK-NEXT: vldrw.u32 q0, [r3]
2849; CHECK-NEXT: adr r3, .LCPI24_1
2850; CHECK-NEXT: sub.w r12, r2, #1
2851; CHECK-NEXT: dls lr, lr
2852; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
2853; CHECK-NEXT: vldrw.u32 q0, [r3]
2854; CHECK-NEXT: vdup.32 q4, r12
2855; CHECK-NEXT: movs r2, #0
2856; CHECK-NEXT: mov.w r12, #0
2857; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
2858; CHECK-NEXT: .LBB24_2: @ %vector.body
2859; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
2860; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
2861; CHECK-NEXT: vldrw.u32 q3, [sp] @ 16-byte Reload
2862; CHECK-NEXT: vadd.i32 q5, q0, r2
2863; CHECK-NEXT: vdup.32 q0, r2
2864; CHECK-NEXT: vcmp.u32 hi, q0, q5
2865; CHECK-NEXT: vadd.i32 q6, q3, r2
2866; CHECK-NEXT: vpsel q7, q2, q1
2867; CHECK-NEXT: vcmp.u32 hi, q0, q6
2868; CHECK-NEXT: vpsel q3, q2, q1
2869; CHECK-NEXT: adds r2, #8
2870; CHECK-NEXT: vmov r3, s12
2871; CHECK-NEXT: vmov.16 q0[0], r3
2872; CHECK-NEXT: vmov r3, s13
2873; CHECK-NEXT: vmov.16 q0[1], r3
2874; CHECK-NEXT: vmov r3, s14
2875; CHECK-NEXT: vmov.16 q0[2], r3
2876; CHECK-NEXT: vmov r3, s15
2877; CHECK-NEXT: vmov.16 q0[3], r3
2878; CHECK-NEXT: vmov r3, s28
2879; CHECK-NEXT: vmov.16 q0[4], r3
2880; CHECK-NEXT: vmov r3, s29
2881; CHECK-NEXT: vmov.16 q0[5], r3
2882; CHECK-NEXT: vmov r3, s30
2883; CHECK-NEXT: vmov.16 q0[6], r3
2884; CHECK-NEXT: vmov r3, s31
2885; CHECK-NEXT: vmov.16 q0[7], r3
2886; CHECK-NEXT: vcmp.i16 ne, q0, zr
2887; CHECK-NEXT: vstr p0, [sp, #36] @ 4-byte Spill
2888; CHECK-NEXT: vcmp.u32 cs, q4, q5
2889; CHECK-NEXT: vpsel q5, q2, q1
2890; CHECK-NEXT: vcmp.u32 cs, q4, q6
2891; CHECK-NEXT: vpsel q0, q2, q1
2892; CHECK-NEXT: vldr p0, [sp, #36] @ 4-byte Reload
2893; CHECK-NEXT: vmov r3, s0
2894; CHECK-NEXT: vmov.16 q6[0], r3
2895; CHECK-NEXT: vmov r3, s1
2896; CHECK-NEXT: vmov.16 q6[1], r3
2897; CHECK-NEXT: vmov r3, s2
2898; CHECK-NEXT: vmov.16 q6[2], r3
2899; CHECK-NEXT: vmov r3, s3
2900; CHECK-NEXT: vmov.16 q6[3], r3
2901; CHECK-NEXT: vmov r3, s20
2902; CHECK-NEXT: vmov.16 q6[4], r3
2903; CHECK-NEXT: vmov r3, s21
2904; CHECK-NEXT: vmov.16 q6[5], r3
2905; CHECK-NEXT: vmov r3, s22
2906; CHECK-NEXT: vmov.16 q6[6], r3
2907; CHECK-NEXT: vmov r3, s23
2908; CHECK-NEXT: vpnot
2909; CHECK-NEXT: vmov.16 q6[7], r3
2910; CHECK-NEXT: vpstttt
2911; CHECK-NEXT: vcmpt.i16 ne, q6, zr
2912; CHECK-NEXT: vldrht.u16 q0, [r0], #16
2913; CHECK-NEXT: vldrht.u16 q3, [r1], #16
2914; CHECK-NEXT: vmlavat.u16 r12, q3, q0
2915; CHECK-NEXT: le lr, .LBB24_2
2916; CHECK-NEXT: b .LBB24_4
2917; CHECK-NEXT: .LBB24_3:
2918; CHECK-NEXT: mov.w r12, #0
2919; CHECK-NEXT: .LBB24_4: @ %for.cond.cleanup
2920; CHECK-NEXT: sxth.w r0, r12
2921; CHECK-NEXT: add sp, #40
2922; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
2923; CHECK-NEXT: pop {r7, pc}
2924; CHECK-NEXT: .p2align 4
2925; CHECK-NEXT: @ %bb.5:
2926; CHECK-NEXT: .LCPI24_0:
2927; CHECK-NEXT: .long 4 @ 0x4
2928; CHECK-NEXT: .long 5 @ 0x5
2929; CHECK-NEXT: .long 6 @ 0x6
2930; CHECK-NEXT: .long 7 @ 0x7
2931; CHECK-NEXT: .LCPI24_1:
2932; CHECK-NEXT: .long 0 @ 0x0
2933; CHECK-NEXT: .long 1 @ 0x1
2934; CHECK-NEXT: .long 2 @ 0x2
2935; CHECK-NEXT: .long 3 @ 0x3
2936entry:
2937 %cmp11.not = icmp eq i32 %n, 0
2938 br i1 %cmp11.not, label %for.cond.cleanup, label %vector.ph
2939
2940vector.ph: ; preds = %entry
2941 %n.rnd.up = add i32 %n, 7
2942 %n.vec = and i32 %n.rnd.up, -8
2943 %trip.count.minus.1 = add i32 %n, -1
2944 br label %vector.body
2945
2946vector.body: ; preds = %vector.body, %vector.ph
2947 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2948 %vec.phi = phi i16 [ 0, %vector.ph ], [ %7, %vector.body ]
2949 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
2950 %0 = getelementptr inbounds i16, i16* %x, i32 %index
2951 %1 = bitcast i16* %0 to <8 x i16>*
2952 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2953 %2 = getelementptr inbounds i16, i16* %y, i32 %index
2954 %3 = bitcast i16* %2 to <8 x i16>*
2955 %wide.masked.load16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %3, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
2956 %4 = mul <8 x i16> %wide.masked.load16, %wide.masked.load
2957 %5 = select <8 x i1> %active.lane.mask, <8 x i16> %4, <8 x i16> zeroinitializer
2958 %6 = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %5)
2959 %7 = add i16 %6, %vec.phi
2960 %index.next = add i32 %index, 8
2961 %8 = icmp eq i32 %index.next, %n.vec
2962 br i1 %8, label %for.cond.cleanup, label %vector.body
2963
2964for.cond.cleanup: ; preds = %vector.body, %entry
2965 %s.0.lcssa = phi i16 [ 0, %entry ], [ %7, %vector.body ]
2966 ret i16 %s.0.lcssa
2967}
2968
2969define signext i16 @add16i16(i8* noalias nocapture readonly %x, i32 %n) {
2970; CHECK-LABEL: add16i16:
2971; CHECK: @ %bb.0: @ %entry
2972; CHECK-NEXT: .save {r7, lr}
2973; CHECK-NEXT: push {r7, lr}
2974; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
2975; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
2976; CHECK-NEXT: .pad #200
2977; CHECK-NEXT: sub sp, #200
2978; CHECK-NEXT: cmp r1, #0
2979; CHECK-NEXT: beq.w .LBB25_3
2980; CHECK-NEXT: @ %bb.1: @ %vector.ph
2981; CHECK-NEXT: add.w r2, r1, #15
2982; CHECK-NEXT: movs r3, #1
2983; CHECK-NEXT: bic r2, r2, #15
2984; CHECK-NEXT: subs r2, #16
2985; CHECK-NEXT: add.w lr, r3, r2, lsr #4
2986; CHECK-NEXT: adr.w r3, .LCPI25_0
2987; CHECK-NEXT: vldrw.u32 q0, [r3]
2988; CHECK-NEXT: dls lr, lr
2989; CHECK-NEXT: adr.w r3, .LCPI25_1
2990; CHECK-NEXT: subs r2, r1, #1
2991; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
2992; CHECK-NEXT: vmov.i8 q0, #0x0
2993; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
2994; CHECK-NEXT: vmov.i8 q0, #0xff
2995; CHECK-NEXT: vstrw.32 q0, [sp, #80] @ 16-byte Spill
2996; CHECK-NEXT: vldrw.u32 q0, [r3]
2997; CHECK-NEXT: adr.w r3, .LCPI25_2
2998; CHECK-NEXT: movs r1, #0
2999; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
3000; CHECK-NEXT: vldrw.u32 q0, [r3]
3001; CHECK-NEXT: adr.w r3, .LCPI25_3
3002; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
3003; CHECK-NEXT: vldrw.u32 q0, [r3]
3004; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
3005; CHECK-NEXT: vdup.32 q0, r2
3006; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
3007; CHECK-NEXT: vmov.i32 q0, #0x0
3008; CHECK-NEXT: movs r2, #0
3009; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
3010; CHECK-NEXT: .LBB25_2: @ %vector.body
3011; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
3012; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
3013; CHECK-NEXT: vdup.32 q6, r1
3014; CHECK-NEXT: vldrw.u32 q3, [sp, #96] @ 16-byte Reload
3015; CHECK-NEXT: vldrw.u32 q1, [sp, #80] @ 16-byte Reload
3016; CHECK-NEXT: vadd.i32 q0, q0, r1
3017; CHECK-NEXT: vcmp.u32 hi, q6, q0
3018; CHECK-NEXT: vstrw.32 q0, [sp, #176] @ 16-byte Spill
3019; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
3020; CHECK-NEXT: vpsel q4, q1, q3
3021; CHECK-NEXT: vadd.i32 q0, q0, r1
3022; CHECK-NEXT: vcmp.u32 hi, q6, q0
3023; CHECK-NEXT: vstrw.32 q0, [sp, #160] @ 16-byte Spill
3024; CHECK-NEXT: vpsel q0, q1, q3
3025; CHECK-NEXT: vmov r3, s0
3026; CHECK-NEXT: vmov.16 q5[0], r3
3027; CHECK-NEXT: vmov r3, s1
3028; CHECK-NEXT: vmov.16 q5[1], r3
3029; CHECK-NEXT: vmov r3, s2
3030; CHECK-NEXT: vmov.16 q5[2], r3
3031; CHECK-NEXT: vmov r3, s3
3032; CHECK-NEXT: vmov.16 q5[3], r3
3033; CHECK-NEXT: vmov r3, s16
3034; CHECK-NEXT: vmov.16 q5[4], r3
3035; CHECK-NEXT: vmov r3, s17
3036; CHECK-NEXT: vmov.16 q5[5], r3
3037; CHECK-NEXT: vmov r3, s18
3038; CHECK-NEXT: vmov.16 q5[6], r3
3039; CHECK-NEXT: vmov r3, s19
3040; CHECK-NEXT: vmov.16 q5[7], r3
3041; CHECK-NEXT: vmov q4, q1
3042; CHECK-NEXT: vcmp.i16 ne, q5, zr
3043; CHECK-NEXT: vstr p0, [sp, #196] @ 4-byte Spill
3044; CHECK-NEXT: vpsel q7, q1, q3
3045; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
3046; CHECK-NEXT: vadd.i32 q0, q0, r1
3047; CHECK-NEXT: vcmp.u32 hi, q6, q0
3048; CHECK-NEXT: vstrw.32 q0, [sp, #128] @ 16-byte Spill
3049; CHECK-NEXT: vstr p0, [sp, #192] @ 4-byte Spill
3050; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
3051; CHECK-NEXT: vadd.i32 q5, q0, r1
3052; CHECK-NEXT: adds r1, #16
3053; CHECK-NEXT: vcmp.u32 hi, q6, q5
3054; CHECK-NEXT: vpsel q0, q1, q3
3055; CHECK-NEXT: vldr p0, [sp, #192] @ 4-byte Reload
3056; CHECK-NEXT: vmov r3, s0
3057; CHECK-NEXT: vmov.16 q6[0], r3
3058; CHECK-NEXT: vmov r3, s1
3059; CHECK-NEXT: vmov.16 q6[1], r3
3060; CHECK-NEXT: vmov r3, s2
3061; CHECK-NEXT: vmov.16 q6[2], r3
3062; CHECK-NEXT: vmov r3, s3
3063; CHECK-NEXT: vpsel q1, q1, q3
3064; CHECK-NEXT: vmov.16 q6[3], r3
3065; CHECK-NEXT: vmov r3, s4
3066; CHECK-NEXT: vmov.16 q6[4], r3
3067; CHECK-NEXT: vmov r3, s5
3068; CHECK-NEXT: vmov.16 q6[5], r3
3069; CHECK-NEXT: vmov r3, s6
3070; CHECK-NEXT: vmov.16 q6[6], r3
3071; CHECK-NEXT: vmov r3, s7
3072; CHECK-NEXT: vmov.16 q6[7], r3
3073; CHECK-NEXT: vcmp.i16 ne, q6, zr
3074; CHECK-NEXT: vpsel q0, q4, q3
3075; CHECK-NEXT: vstr p0, [sp, #192] @ 4-byte Spill
3076; CHECK-NEXT: vmov.u16 r3, q0[0]
3077; CHECK-NEXT: vmov.8 q6[0], r3
3078; CHECK-NEXT: vmov.u16 r3, q0[1]
3079; CHECK-NEXT: vmov.8 q6[1], r3
3080; CHECK-NEXT: vmov.u16 r3, q0[2]
3081; CHECK-NEXT: vmov.8 q6[2], r3
3082; CHECK-NEXT: vmov.u16 r3, q0[3]
3083; CHECK-NEXT: vmov.8 q6[3], r3
3084; CHECK-NEXT: vmov.u16 r3, q0[4]
3085; CHECK-NEXT: vmov.8 q6[4], r3
3086; CHECK-NEXT: vmov.u16 r3, q0[5]
3087; CHECK-NEXT: vmov.8 q6[5], r3
3088; CHECK-NEXT: vmov.u16 r3, q0[6]
3089; CHECK-NEXT: vmov.8 q6[6], r3
3090; CHECK-NEXT: vmov.u16 r3, q0[7]
3091; CHECK-NEXT: vmov.8 q6[7], r3
3092; CHECK-NEXT: vmov.u16 r3, q7[0]
3093; CHECK-NEXT: vmov.8 q6[8], r3
3094; CHECK-NEXT: vmov.u16 r3, q7[1]
3095; CHECK-NEXT: vmov.8 q6[9], r3
3096; CHECK-NEXT: vmov.u16 r3, q7[2]
3097; CHECK-NEXT: vmov.8 q6[10], r3
3098; CHECK-NEXT: vmov.u16 r3, q7[3]
3099; CHECK-NEXT: vmov.8 q6[11], r3
3100; CHECK-NEXT: vmov.u16 r3, q7[4]
3101; CHECK-NEXT: vmov.8 q6[12], r3
3102; CHECK-NEXT: vmov.u16 r3, q7[5]
3103; CHECK-NEXT: vmov.8 q6[13], r3
3104; CHECK-NEXT: vmov.u16 r3, q7[6]
3105; CHECK-NEXT: vmov.8 q6[14], r3
3106; CHECK-NEXT: vmov.u16 r3, q7[7]
3107; CHECK-NEXT: vmov.8 q6[15], r3
3108; CHECK-NEXT: vcmp.i8 ne, q6, zr
3109; CHECK-NEXT: vstr p0, [sp, #156] @ 4-byte Spill
3110; CHECK-NEXT: vldrw.u32 q2, [sp, #16] @ 16-byte Reload
3111; CHECK-NEXT: vldrw.u32 q0, [sp, #176] @ 16-byte Reload
3112; CHECK-NEXT: vcmp.u32 cs, q2, q0
3113; CHECK-NEXT: vldrw.u32 q0, [sp, #160] @ 16-byte Reload
3114; CHECK-NEXT: vpsel q6, q4, q3
3115; CHECK-NEXT: vcmp.u32 cs, q2, q0
3116; CHECK-NEXT: vpsel q1, q4, q3
3117; CHECK-NEXT: vmov r3, s4
3118; CHECK-NEXT: vmov.16 q0[0], r3
3119; CHECK-NEXT: vmov r3, s5
3120; CHECK-NEXT: vmov.16 q0[1], r3
3121; CHECK-NEXT: vmov r3, s6
3122; CHECK-NEXT: vmov.16 q0[2], r3
3123; CHECK-NEXT: vmov r3, s7
3124; CHECK-NEXT: vmov.16 q0[3], r3
3125; CHECK-NEXT: vmov r3, s24
3126; CHECK-NEXT: vmov.16 q0[4], r3
3127; CHECK-NEXT: vmov r3, s25
3128; CHECK-NEXT: vmov.16 q0[5], r3
3129; CHECK-NEXT: vmov r3, s26
3130; CHECK-NEXT: vmov.16 q0[6], r3
3131; CHECK-NEXT: vmov r3, s27
3132; CHECK-NEXT: vmov.16 q0[7], r3
3133; CHECK-NEXT: vldrw.u32 q1, [sp, #128] @ 16-byte Reload
3134; CHECK-NEXT: vcmp.i16 ne, q0, zr
3135; CHECK-NEXT: vmov q6, q4
3136; CHECK-NEXT: vpsel q7, q4, q3
3137; CHECK-NEXT: vcmp.u32 cs, q2, q1
3138; CHECK-NEXT: vpsel q4, q4, q3
3139; CHECK-NEXT: vcmp.u32 cs, q2, q5
3140; CHECK-NEXT: vpsel q1, q6, q3
3141; CHECK-NEXT: vmov q2, q3
3142; CHECK-NEXT: vmov r3, s4
3143; CHECK-NEXT: vmov q5, q6
3144; CHECK-NEXT: vmov.16 q3[0], r3
3145; CHECK-NEXT: vmov r3, s5
3146; CHECK-NEXT: vmov.16 q3[1], r3
3147; CHECK-NEXT: vmov r3, s6
3148; CHECK-NEXT: vmov.16 q3[2], r3
3149; CHECK-NEXT: vmov r3, s7
3150; CHECK-NEXT: vmov.16 q3[3], r3
3151; CHECK-NEXT: vmov r3, s16
3152; CHECK-NEXT: vmov.16 q3[4], r3
3153; CHECK-NEXT: vmov r3, s17
3154; CHECK-NEXT: vmov.16 q3[5], r3
3155; CHECK-NEXT: vmov r3, s18
3156; CHECK-NEXT: vmov.16 q3[6], r3
3157; CHECK-NEXT: vmov r3, s19
3158; CHECK-NEXT: vmov.16 q3[7], r3
3159; CHECK-NEXT: vcmp.i16 ne, q3, zr
3160; CHECK-NEXT: vpsel q5, q6, q2
3161; CHECK-NEXT: vldr p0, [sp, #156] @ 4-byte Reload
3162; CHECK-NEXT: vmov.u16 r3, q5[0]
3163; CHECK-NEXT: vmov.8 q4[0], r3
3164; CHECK-NEXT: vmov.u16 r3, q5[1]
3165; CHECK-NEXT: vmov.8 q4[1], r3
3166; CHECK-NEXT: vmov.u16 r3, q5[2]
3167; CHECK-NEXT: vmov.8 q4[2], r3
3168; CHECK-NEXT: vmov.u16 r3, q5[3]
3169; CHECK-NEXT: vmov.8 q4[3], r3
3170; CHECK-NEXT: vmov.u16 r3, q5[4]
3171; CHECK-NEXT: vmov.8 q4[4], r3
3172; CHECK-NEXT: vmov.u16 r3, q5[5]
3173; CHECK-NEXT: vmov.8 q4[5], r3
3174; CHECK-NEXT: vmov.u16 r3, q5[6]
3175; CHECK-NEXT: vmov.8 q4[6], r3
3176; CHECK-NEXT: vmov.u16 r3, q5[7]
3177; CHECK-NEXT: vmov.8 q4[7], r3
3178; CHECK-NEXT: vmov.u16 r3, q7[0]
3179; CHECK-NEXT: vmov.8 q4[8], r3
3180; CHECK-NEXT: vmov.u16 r3, q7[1]
3181; CHECK-NEXT: vmov.8 q4[9], r3
3182; CHECK-NEXT: vmov.u16 r3, q7[2]
3183; CHECK-NEXT: vmov.8 q4[10], r3
3184; CHECK-NEXT: vmov.u16 r3, q7[3]
3185; CHECK-NEXT: vmov.8 q4[11], r3
3186; CHECK-NEXT: vmov.u16 r3, q7[4]
3187; CHECK-NEXT: vmov.8 q4[12], r3
3188; CHECK-NEXT: vmov.u16 r3, q7[5]
3189; CHECK-NEXT: vmov.8 q4[13], r3
3190; CHECK-NEXT: vmov.u16 r3, q7[6]
3191; CHECK-NEXT: vmov.8 q4[14], r3
3192; CHECK-NEXT: vmov.u16 r3, q7[7]
3193; CHECK-NEXT: vmov.8 q4[15], r3
3194; CHECK-NEXT: vpnot
3195; CHECK-NEXT: vpstt
3196; CHECK-NEXT: vcmpt.i8 ne, q4, zr
3197; CHECK-NEXT: vldrbt.u8 q4, [r0], #16
3198; CHECK-NEXT: vmov.u8 r3, q4[0]
3199; CHECK-NEXT: vldr p0, [sp, #192] @ 4-byte Reload
3200; CHECK-NEXT: vpnot
3201; CHECK-NEXT: vpst
3202; CHECK-NEXT: vcmpt.i16 ne, q3, zr
3203; CHECK-NEXT: vmov.16 q3[0], r3
3204; CHECK-NEXT: vmov.u8 r3, q4[1]
3205; CHECK-NEXT: vldrw.u32 q2, [sp] @ 16-byte Reload
3206; CHECK-NEXT: vmov.16 q3[1], r3
3207; CHECK-NEXT: vmov.u8 r3, q4[2]
3208; CHECK-NEXT: vmov.16 q3[2], r3
3209; CHECK-NEXT: vmov.u8 r3, q4[3]
3210; CHECK-NEXT: vmov.16 q3[3], r3
3211; CHECK-NEXT: vmov.u8 r3, q4[4]
3212; CHECK-NEXT: vmov.16 q3[4], r3
3213; CHECK-NEXT: vmov.u8 r3, q4[5]
3214; CHECK-NEXT: vmov.16 q3[5], r3
3215; CHECK-NEXT: vmov.u8 r3, q4[6]
3216; CHECK-NEXT: vmov.16 q3[6], r3
3217; CHECK-NEXT: vmov.u8 r3, q4[7]
3218; CHECK-NEXT: vmov.16 q3[7], r3
3219; CHECK-NEXT: vmov.u8 r3, q4[8]
3220; CHECK-NEXT: vmovlb.u8 q1, q3
3221; CHECK-NEXT: vpsel q3, q1, q2
3222; CHECK-NEXT: vldr p0, [sp, #196] @ 4-byte Reload
3223; CHECK-NEXT: vpnot
3224; CHECK-NEXT: vpst
3225; CHECK-NEXT: vcmpt.i16 ne, q0, zr
3226; CHECK-NEXT: vmov.16 q0[0], r3
3227; CHECK-NEXT: vmov.u8 r3, q4[9]
3228; CHECK-NEXT: vmov.16 q0[1], r3
3229; CHECK-NEXT: vmov.u8 r3, q4[10]
3230; CHECK-NEXT: vmov.16 q0[2], r3
3231; CHECK-NEXT: vmov.u8 r3, q4[11]
3232; CHECK-NEXT: vmov.16 q0[3], r3
3233; CHECK-NEXT: vmov.u8 r3, q4[12]
3234; CHECK-NEXT: vmov.16 q0[4], r3
3235; CHECK-NEXT: vmov.u8 r3, q4[13]
3236; CHECK-NEXT: vmov.16 q0[5], r3
3237; CHECK-NEXT: vmov.u8 r3, q4[14]
3238; CHECK-NEXT: vmov.16 q0[6], r3
3239; CHECK-NEXT: vmov.u8 r3, q4[15]
3240; CHECK-NEXT: vmov.16 q0[7], r3
3241; CHECK-NEXT: vmovlb.u8 q0, q0
3242; CHECK-NEXT: vpst
3243; CHECK-NEXT: vaddt.i16 q3, q3, q0
3244; CHECK-NEXT: vaddva.u16 r2, q3
3245; CHECK-NEXT: le lr, .LBB25_2
3246; CHECK-NEXT: b .LBB25_4
3247; CHECK-NEXT: .LBB25_3:
3248; CHECK-NEXT: movs r2, #0
3249; CHECK-NEXT: .LBB25_4: @ %for.cond.cleanup
3250; CHECK-NEXT: sxth r0, r2
3251; CHECK-NEXT: add sp, #200
3252; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
3253; CHECK-NEXT: pop {r7, pc}
3254; CHECK-NEXT: .p2align 4
3255; CHECK-NEXT: @ %bb.5:
3256; CHECK-NEXT: .LCPI25_0:
3257; CHECK-NEXT: .long 12 @ 0xc
3258; CHECK-NEXT: .long 13 @ 0xd
3259; CHECK-NEXT: .long 14 @ 0xe
3260; CHECK-NEXT: .long 15 @ 0xf
3261; CHECK-NEXT: .LCPI25_1:
3262; CHECK-NEXT: .long 8 @ 0x8
3263; CHECK-NEXT: .long 9 @ 0x9
3264; CHECK-NEXT: .long 10 @ 0xa
3265; CHECK-NEXT: .long 11 @ 0xb
3266; CHECK-NEXT: .LCPI25_2:
3267; CHECK-NEXT: .long 4 @ 0x4
3268; CHECK-NEXT: .long 5 @ 0x5
3269; CHECK-NEXT: .long 6 @ 0x6
3270; CHECK-NEXT: .long 7 @ 0x7
3271; CHECK-NEXT: .LCPI25_3:
3272; CHECK-NEXT: .long 0 @ 0x0
3273; CHECK-NEXT: .long 1 @ 0x1
3274; CHECK-NEXT: .long 2 @ 0x2
3275; CHECK-NEXT: .long 3 @ 0x3
3276entry:
3277 %cmp8.not = icmp eq i32 %n, 0
3278 br i1 %cmp8.not, label %for.cond.cleanup, label %vector.ph
3279
3280vector.ph: ; preds = %entry
3281 %n.rnd.up = add i32 %n, 15
3282 %n.vec = and i32 %n.rnd.up, -16
3283 %trip.count.minus.1 = add i32 %n, -1
3284 br label %vector.body
3285
3286vector.body: ; preds = %vector.body, %vector.ph
3287 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
3288 %vec.phi = phi i16 [ 0, %vector.ph ], [ %5, %vector.body ]
3289 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
3290 %0 = getelementptr inbounds i8, i8* %x, i32 %index
3291 %1 = bitcast i8* %0 to <16 x i8>*
3292 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
3293 %2 = zext <16 x i8> %wide.masked.load to <16 x i16>
3294 %3 = select <16 x i1> %active.lane.mask, <16 x i16> %2, <16 x i16> zeroinitializer
3295 %4 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %3)
3296 %5 = add i16 %4, %vec.phi
3297 %index.next = add i32 %index, 16
3298 %6 = icmp eq i32 %index.next, %n.vec
3299 br i1 %6, label %for.cond.cleanup, label %vector.body
3300
3301for.cond.cleanup: ; preds = %vector.body, %entry
3302 %s.0.lcssa = phi i16 [ 0, %entry ], [ %5, %vector.body ]
3303 ret i16 %s.0.lcssa
3304}
3305
3306define signext i16 @mla16i16(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
3307; CHECK-LABEL: mla16i16:
3308; CHECK: @ %bb.0: @ %entry
3309; CHECK-NEXT: .save {r7, lr}
3310; CHECK-NEXT: push {r7, lr}
3311; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
3312; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
3313; CHECK-NEXT: .pad #200
3314; CHECK-NEXT: sub sp, #200
3315; CHECK-NEXT: cmp r2, #0
3316; CHECK-NEXT: beq.w .LBB26_3
3317; CHECK-NEXT: @ %bb.1: @ %vector.ph
3318; CHECK-NEXT: add.w r3, r2, #15
3319; CHECK-NEXT: bic r3, r3, #15
3320; CHECK-NEXT: sub.w r12, r3, #16
3321; CHECK-NEXT: movs r3, #1
3322; CHECK-NEXT: add.w lr, r3, r12, lsr #4
3323; CHECK-NEXT: adr.w r3, .LCPI26_0
3324; CHECK-NEXT: vldrw.u32 q0, [r3]
3325; CHECK-NEXT: dls lr, lr
3326; CHECK-NEXT: adr.w r3, .LCPI26_1
3327; CHECK-NEXT: sub.w r12, r2, #1
3328; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
3329; CHECK-NEXT: vmov.i8 q0, #0x0
3330; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
3331; CHECK-NEXT: vmov.i8 q0, #0xff
3332; CHECK-NEXT: vstrw.32 q0, [sp, #80] @ 16-byte Spill
3333; CHECK-NEXT: vldrw.u32 q0, [r3]
3334; CHECK-NEXT: adr.w r3, .LCPI26_2
3335; CHECK-NEXT: movs r2, #0
3336; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
3337; CHECK-NEXT: vldrw.u32 q0, [r3]
3338; CHECK-NEXT: adr.w r3, .LCPI26_3
3339; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
3340; CHECK-NEXT: vldrw.u32 q0, [r3]
3341; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
3342; CHECK-NEXT: vdup.32 q0, r12
3343; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
3344; CHECK-NEXT: vmov.i32 q0, #0x0
3345; CHECK-NEXT: mov.w r12, #0
3346; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
3347; CHECK-NEXT: .LBB26_2: @ %vector.body
3348; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
3349; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
3350; CHECK-NEXT: vdup.32 q6, r2
3351; CHECK-NEXT: vldrw.u32 q3, [sp, #96] @ 16-byte Reload
3352; CHECK-NEXT: vldrw.u32 q2, [sp, #80] @ 16-byte Reload
3353; CHECK-NEXT: vadd.i32 q0, q0, r2
3354; CHECK-NEXT: vcmp.u32 hi, q6, q0
3355; CHECK-NEXT: vstrw.32 q0, [sp, #176] @ 16-byte Spill
3356; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
3357; CHECK-NEXT: vpsel q4, q2, q3
3358; CHECK-NEXT: vadd.i32 q0, q0, r2
3359; CHECK-NEXT: vcmp.u32 hi, q6, q0
3360; CHECK-NEXT: vstrw.32 q0, [sp, #160] @ 16-byte Spill
3361; CHECK-NEXT: vpsel q0, q2, q3
3362; CHECK-NEXT: vmov r3, s0
3363; CHECK-NEXT: vmov.16 q5[0], r3
3364; CHECK-NEXT: vmov r3, s1
3365; CHECK-NEXT: vmov.16 q5[1], r3
3366; CHECK-NEXT: vmov r3, s2
3367; CHECK-NEXT: vmov.16 q5[2], r3
3368; CHECK-NEXT: vmov r3, s3
3369; CHECK-NEXT: vmov.16 q5[3], r3
3370; CHECK-NEXT: vmov r3, s16
3371; CHECK-NEXT: vmov.16 q5[4], r3
3372; CHECK-NEXT: vmov r3, s17
3373; CHECK-NEXT: vmov.16 q5[5], r3
3374; CHECK-NEXT: vmov r3, s18
3375; CHECK-NEXT: vmov.16 q5[6], r3
3376; CHECK-NEXT: vmov r3, s19
3377; CHECK-NEXT: vmov.16 q5[7], r3
3378; CHECK-NEXT: vcmp.i16 ne, q5, zr
3379; CHECK-NEXT: vmov q5, q3
3380; CHECK-NEXT: vstr p0, [sp, #196] @ 4-byte Spill
3381; CHECK-NEXT: vpsel q7, q2, q3
3382; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
3383; CHECK-NEXT: vadd.i32 q4, q0, r2
3384; CHECK-NEXT: vcmp.u32 hi, q6, q4
3385; CHECK-NEXT: vstr p0, [sp, #192] @ 4-byte Spill
3386; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
3387; CHECK-NEXT: vadd.i32 q0, q0, r2
3388; CHECK-NEXT: adds r2, #16
3389; CHECK-NEXT: vcmp.u32 hi, q6, q0
3390; CHECK-NEXT: vstrw.32 q0, [sp, #144] @ 16-byte Spill
3391; CHECK-NEXT: vpsel q0, q2, q3
3392; CHECK-NEXT: vldr p0, [sp, #192] @ 4-byte Reload
3393; CHECK-NEXT: vmov r3, s0
3394; CHECK-NEXT: vmov.16 q6[0], r3
3395; CHECK-NEXT: vmov r3, s1
3396; CHECK-NEXT: vmov.16 q6[1], r3
3397; CHECK-NEXT: vmov r3, s2
3398; CHECK-NEXT: vmov.16 q6[2], r3
3399; CHECK-NEXT: vmov r3, s3
3400; CHECK-NEXT: vpsel q1, q2, q3
3401; CHECK-NEXT: vmov.16 q6[3], r3
3402; CHECK-NEXT: vmov r3, s4
3403; CHECK-NEXT: vmov.16 q6[4], r3
3404; CHECK-NEXT: vmov r3, s5
3405; CHECK-NEXT: vmov.16 q6[5], r3
3406; CHECK-NEXT: vmov r3, s6
3407; CHECK-NEXT: vmov.16 q6[6], r3
3408; CHECK-NEXT: vmov r3, s7
3409; CHECK-NEXT: vmov.16 q6[7], r3
3410; CHECK-NEXT: vcmp.i16 ne, q6, zr
3411; CHECK-NEXT: vpsel q0, q2, q3
3412; CHECK-NEXT: vstr p0, [sp, #192] @ 4-byte Spill
3413; CHECK-NEXT: vmov.u16 r3, q0[0]
3414; CHECK-NEXT: vmov.8 q6[0], r3
3415; CHECK-NEXT: vmov.u16 r3, q0[1]
3416; CHECK-NEXT: vmov.8 q6[1], r3
3417; CHECK-NEXT: vmov.u16 r3, q0[2]
3418; CHECK-NEXT: vmov.8 q6[2], r3
3419; CHECK-NEXT: vmov.u16 r3, q0[3]
3420; CHECK-NEXT: vmov.8 q6[3], r3
3421; CHECK-NEXT: vmov.u16 r3, q0[4]
3422; CHECK-NEXT: vmov.8 q6[4], r3
3423; CHECK-NEXT: vmov.u16 r3, q0[5]
3424; CHECK-NEXT: vmov.8 q6[5], r3
3425; CHECK-NEXT: vmov.u16 r3, q0[6]
3426; CHECK-NEXT: vmov.8 q6[6], r3
3427; CHECK-NEXT: vmov.u16 r3, q0[7]
3428; CHECK-NEXT: vmov.8 q6[7], r3
3429; CHECK-NEXT: vmov.u16 r3, q7[0]
3430; CHECK-NEXT: vmov.8 q6[8], r3
3431; CHECK-NEXT: vmov.u16 r3, q7[1]
3432; CHECK-NEXT: vmov.8 q6[9], r3
3433; CHECK-NEXT: vmov.u16 r3, q7[2]
3434; CHECK-NEXT: vmov.8 q6[10], r3
3435; CHECK-NEXT: vmov.u16 r3, q7[3]
3436; CHECK-NEXT: vmov.8 q6[11], r3
3437; CHECK-NEXT: vmov.u16 r3, q7[4]
3438; CHECK-NEXT: vmov.8 q6[12], r3
3439; CHECK-NEXT: vmov.u16 r3, q7[5]
3440; CHECK-NEXT: vmov.8 q6[13], r3
3441; CHECK-NEXT: vmov.u16 r3, q7[6]
3442; CHECK-NEXT: vmov.8 q6[14], r3
3443; CHECK-NEXT: vmov.u16 r3, q7[7]
3444; CHECK-NEXT: vmov.8 q6[15], r3
3445; CHECK-NEXT: vcmp.i8 ne, q6, zr
3446; CHECK-NEXT: vstr p0, [sp, #140] @ 4-byte Spill
3447; CHECK-NEXT: vldrw.u32 q7, [sp, #16] @ 16-byte Reload
3448; CHECK-NEXT: vldrw.u32 q0, [sp, #176] @ 16-byte Reload
3449; CHECK-NEXT: vcmp.u32 cs, q7, q0
3450; CHECK-NEXT: vldrw.u32 q0, [sp, #160] @ 16-byte Reload
3451; CHECK-NEXT: vpsel q6, q2, q3
3452; CHECK-NEXT: vcmp.u32 cs, q7, q0
3453; CHECK-NEXT: vpsel q1, q2, q3
3454; CHECK-NEXT: vmov r3, s4
3455; CHECK-NEXT: vmov.16 q0[0], r3
3456; CHECK-NEXT: vmov r3, s5
3457; CHECK-NEXT: vmov.16 q0[1], r3
3458; CHECK-NEXT: vmov r3, s6
3459; CHECK-NEXT: vmov.16 q0[2], r3
3460; CHECK-NEXT: vmov r3, s7
3461; CHECK-NEXT: vmov.16 q0[3], r3
3462; CHECK-NEXT: vmov r3, s24
3463; CHECK-NEXT: vmov.16 q0[4], r3
3464; CHECK-NEXT: vmov r3, s25
3465; CHECK-NEXT: vmov.16 q0[5], r3
3466; CHECK-NEXT: vmov r3, s26
3467; CHECK-NEXT: vmov.16 q0[6], r3
3468; CHECK-NEXT: vmov r3, s27
3469; CHECK-NEXT: vmov.16 q0[7], r3
3470; CHECK-NEXT: vldrw.u32 q1, [sp, #144] @ 16-byte Reload
3471; CHECK-NEXT: vcmp.i16 ne, q0, zr
3472; CHECK-NEXT: vpsel q3, q2, q3
3473; CHECK-NEXT: vcmp.u32 cs, q7, q4
3474; CHECK-NEXT: vpsel q6, q2, q5
3475; CHECK-NEXT: vcmp.u32 cs, q7, q1
3476; CHECK-NEXT: vpsel q1, q2, q5
3477; CHECK-NEXT: vmov r3, s4
3478; CHECK-NEXT: vmov.16 q4[0], r3
3479; CHECK-NEXT: vmov r3, s5
3480; CHECK-NEXT: vmov.16 q4[1], r3
3481; CHECK-NEXT: vmov r3, s6
3482; CHECK-NEXT: vmov.16 q4[2], r3
3483; CHECK-NEXT: vmov r3, s7
3484; CHECK-NEXT: vmov.16 q4[3], r3
3485; CHECK-NEXT: vmov r3, s24
3486; CHECK-NEXT: vmov.16 q4[4], r3
3487; CHECK-NEXT: vmov r3, s25
3488; CHECK-NEXT: vmov.16 q4[5], r3
3489; CHECK-NEXT: vmov r3, s26
3490; CHECK-NEXT: vmov.16 q4[6], r3
3491; CHECK-NEXT: vmov r3, s27
3492; CHECK-NEXT: vmov.16 q4[7], r3
3493; CHECK-NEXT: vcmp.i16 ne, q4, zr
3494; CHECK-NEXT: vpsel q6, q2, q5
3495; CHECK-NEXT: vldr p0, [sp, #140] @ 4-byte Reload
3496; CHECK-NEXT: vmov.u16 r3, q6[0]
3497; CHECK-NEXT: vmov.8 q5[0], r3
3498; CHECK-NEXT: vmov.u16 r3, q6[1]
3499; CHECK-NEXT: vmov.8 q5[1], r3
3500; CHECK-NEXT: vmov.u16 r3, q6[2]
3501; CHECK-NEXT: vmov.8 q5[2], r3
3502; CHECK-NEXT: vmov.u16 r3, q6[3]
3503; CHECK-NEXT: vmov.8 q5[3], r3
3504; CHECK-NEXT: vmov.u16 r3, q6[4]
3505; CHECK-NEXT: vmov.8 q5[4], r3
3506; CHECK-NEXT: vmov.u16 r3, q6[5]
3507; CHECK-NEXT: vmov.8 q5[5], r3
3508; CHECK-NEXT: vmov.u16 r3, q6[6]
3509; CHECK-NEXT: vmov.8 q5[6], r3
3510; CHECK-NEXT: vmov.u16 r3, q6[7]
3511; CHECK-NEXT: vmov.8 q5[7], r3
3512; CHECK-NEXT: vmov.u16 r3, q3[0]
3513; CHECK-NEXT: vmov.8 q5[8], r3
3514; CHECK-NEXT: vmov.u16 r3, q3[1]
3515; CHECK-NEXT: vmov.8 q5[9], r3
3516; CHECK-NEXT: vmov.u16 r3, q3[2]
3517; CHECK-NEXT: vmov.8 q5[10], r3
3518; CHECK-NEXT: vmov.u16 r3, q3[3]
3519; CHECK-NEXT: vmov.8 q5[11], r3
3520; CHECK-NEXT: vmov.u16 r3, q3[4]
3521; CHECK-NEXT: vmov.8 q5[12], r3
3522; CHECK-NEXT: vmov.u16 r3, q3[5]
3523; CHECK-NEXT: vmov.8 q5[13], r3
3524; CHECK-NEXT: vmov.u16 r3, q3[6]
3525; CHECK-NEXT: vmov.8 q5[14], r3
3526; CHECK-NEXT: vmov.u16 r3, q3[7]
3527; CHECK-NEXT: vmov.8 q5[15], r3
3528; CHECK-NEXT: vpnot
3529; CHECK-NEXT: vpsttt
3530; CHECK-NEXT: vcmpt.i8 ne, q5, zr
3531; CHECK-NEXT: vldrbt.u8 q5, [r0], #16
3532; CHECK-NEXT: vldrbt.u8 q3, [r1], #16
3533; CHECK-NEXT: vmov.u8 r3, q3[0]
3534; CHECK-NEXT: vldr p0, [sp, #192] @ 4-byte Reload
3535; CHECK-NEXT: vpnot
3536; CHECK-NEXT: vpst
3537; CHECK-NEXT: vcmpt.i16 ne, q4, zr
3538; CHECK-NEXT: vmov.16 q4[0], r3
3539; CHECK-NEXT: vmov.u8 r3, q3[1]
3540; CHECK-NEXT: vmov.16 q4[1], r3
3541; CHECK-NEXT: vmov.u8 r3, q3[2]
3542; CHECK-NEXT: vmov.16 q4[2], r3
3543; CHECK-NEXT: vmov.u8 r3, q3[3]
3544; CHECK-NEXT: vmov.16 q4[3], r3
3545; CHECK-NEXT: vmov.u8 r3, q3[4]
3546; CHECK-NEXT: vmov.16 q4[4], r3
3547; CHECK-NEXT: vmov.u8 r3, q3[5]
3548; CHECK-NEXT: vmov.16 q4[5], r3
3549; CHECK-NEXT: vmov.u8 r3, q3[6]
3550; CHECK-NEXT: vmov.16 q4[6], r3
3551; CHECK-NEXT: vmov.u8 r3, q3[7]
3552; CHECK-NEXT: vmov.16 q4[7], r3
3553; CHECK-NEXT: vmov.u8 r3, q5[0]
3554; CHECK-NEXT: vmov.16 q6[0], r3
3555; CHECK-NEXT: vmov.u8 r3, q5[1]
3556; CHECK-NEXT: vmov.16 q6[1], r3
3557; CHECK-NEXT: vmov.u8 r3, q5[2]
3558; CHECK-NEXT: vmov.16 q6[2], r3
3559; CHECK-NEXT: vmov.u8 r3, q5[3]
3560; CHECK-NEXT: vmov.16 q6[3], r3
3561; CHECK-NEXT: vmov.u8 r3, q5[4]
3562; CHECK-NEXT: vmov.16 q6[4], r3
3563; CHECK-NEXT: vmov.u8 r3, q5[5]
3564; CHECK-NEXT: vmov.16 q6[5], r3
3565; CHECK-NEXT: vmov.u8 r3, q5[6]
3566; CHECK-NEXT: vmov.16 q6[6], r3
3567; CHECK-NEXT: vmov.u8 r3, q5[7]
3568; CHECK-NEXT: vmov.16 q6[7], r3
3569; CHECK-NEXT: vmov.u8 r3, q5[8]
3570; CHECK-NEXT: vmovlb.u8 q1, q6
3571; CHECK-NEXT: vmovlb.u8 q6, q4
3572; CHECK-NEXT: vldrw.u32 q4, [sp] @ 16-byte Reload
3573; CHECK-NEXT: vpst
3574; CHECK-NEXT: vmult.i16 q4, q6, q1
3575; CHECK-NEXT: vldr p0, [sp, #196] @ 4-byte Reload
3576; CHECK-NEXT: vpnot
3577; CHECK-NEXT: vpst
3578; CHECK-NEXT: vcmpt.i16 ne, q0, zr
3579; CHECK-NEXT: vmov.16 q0[0], r3
3580; CHECK-NEXT: vmov.u8 r3, q5[9]
3581; CHECK-NEXT: vmov.16 q0[1], r3
3582; CHECK-NEXT: vmov.u8 r3, q5[10]
3583; CHECK-NEXT: vmov.16 q0[2], r3
3584; CHECK-NEXT: vmov.u8 r3, q5[11]
3585; CHECK-NEXT: vmov.16 q0[3], r3
3586; CHECK-NEXT: vmov.u8 r3, q5[12]
3587; CHECK-NEXT: vmov.16 q0[4], r3
3588; CHECK-NEXT: vmov.u8 r3, q5[13]
3589; CHECK-NEXT: vmov.16 q0[5], r3
3590; CHECK-NEXT: vmov.u8 r3, q5[14]
3591; CHECK-NEXT: vmov.16 q0[6], r3
3592; CHECK-NEXT: vmov.u8 r3, q5[15]
3593; CHECK-NEXT: vmov.16 q0[7], r3
3594; CHECK-NEXT: vmov.u8 r3, q3[8]
3595; CHECK-NEXT: vmov.16 q5[0], r3
3596; CHECK-NEXT: vmov.u8 r3, q3[9]
3597; CHECK-NEXT: vmov.16 q5[1], r3
3598; CHECK-NEXT: vmov.u8 r3, q3[10]
3599; CHECK-NEXT: vmov.16 q5[2], r3
3600; CHECK-NEXT: vmov.u8 r3, q3[11]
3601; CHECK-NEXT: vmov.16 q5[3], r3
3602; CHECK-NEXT: vmov.u8 r3, q3[12]
3603; CHECK-NEXT: vmov.16 q5[4], r3
3604; CHECK-NEXT: vmov.u8 r3, q3[13]
3605; CHECK-NEXT: vmov.16 q5[5], r3
3606; CHECK-NEXT: vmov.u8 r3, q3[14]
3607; CHECK-NEXT: vmov.16 q5[6], r3
3608; CHECK-NEXT: vmov.u8 r3, q3[15]
3609; CHECK-NEXT: vmov.16 q5[7], r3
3610; CHECK-NEXT: vmullb.u8 q0, q5, q0
3611; CHECK-NEXT: vpst
3612; CHECK-NEXT: vaddt.i16 q4, q4, q0
3613; CHECK-NEXT: vaddva.u16 r12, q4
3614; CHECK-NEXT: le lr, .LBB26_2
3615; CHECK-NEXT: b .LBB26_4
3616; CHECK-NEXT: .LBB26_3:
3617; CHECK-NEXT: mov.w r12, #0
3618; CHECK-NEXT: .LBB26_4: @ %for.cond.cleanup
3619; CHECK-NEXT: sxth.w r0, r12
3620; CHECK-NEXT: add sp, #200
3621; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
3622; CHECK-NEXT: pop {r7, pc}
3623; CHECK-NEXT: .p2align 4
3624; CHECK-NEXT: @ %bb.5:
3625; CHECK-NEXT: .LCPI26_0:
3626; CHECK-NEXT: .long 12 @ 0xc
3627; CHECK-NEXT: .long 13 @ 0xd
3628; CHECK-NEXT: .long 14 @ 0xe
3629; CHECK-NEXT: .long 15 @ 0xf
3630; CHECK-NEXT: .LCPI26_1:
3631; CHECK-NEXT: .long 8 @ 0x8
3632; CHECK-NEXT: .long 9 @ 0x9
3633; CHECK-NEXT: .long 10 @ 0xa
3634; CHECK-NEXT: .long 11 @ 0xb
3635; CHECK-NEXT: .LCPI26_2:
3636; CHECK-NEXT: .long 4 @ 0x4
3637; CHECK-NEXT: .long 5 @ 0x5
3638; CHECK-NEXT: .long 6 @ 0x6
3639; CHECK-NEXT: .long 7 @ 0x7
3640; CHECK-NEXT: .LCPI26_3:
3641; CHECK-NEXT: .long 0 @ 0x0
3642; CHECK-NEXT: .long 1 @ 0x1
3643; CHECK-NEXT: .long 2 @ 0x2
3644; CHECK-NEXT: .long 3 @ 0x3
3645entry:
3646 %cmp13.not = icmp eq i32 %n, 0
3647 br i1 %cmp13.not, label %for.cond.cleanup, label %vector.ph
3648
3649vector.ph: ; preds = %entry
3650 %n.rnd.up = add i32 %n, 15
3651 %n.vec = and i32 %n.rnd.up, -16
3652 %trip.count.minus.1 = add i32 %n, -1
3653 br label %vector.body
3654
3655vector.body: ; preds = %vector.body, %vector.ph
3656 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
3657 %vec.phi = phi i16 [ 0, %vector.ph ], [ %9, %vector.body ]
3658 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
3659 %0 = getelementptr inbounds i8, i8* %x, i32 %index
3660 %1 = bitcast i8* %0 to <16 x i8>*
3661 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
3662 %2 = zext <16 x i8> %wide.masked.load to <16 x i16>
3663 %3 = getelementptr inbounds i8, i8* %y, i32 %index
3664 %4 = bitcast i8* %3 to <16 x i8>*
3665 %wide.masked.load18 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
3666 %5 = zext <16 x i8> %wide.masked.load18 to <16 x i16>
3667 %6 = mul nuw <16 x i16> %5, %2
3668 %7 = select <16 x i1> %active.lane.mask, <16 x i16> %6, <16 x i16> zeroinitializer
3669 %8 = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %7)
3670 %9 = add i16 %8, %vec.phi
3671 %index.next = add i32 %index, 16
3672 %10 = icmp eq i32 %index.next, %n.vec
3673 br i1 %10, label %for.cond.cleanup, label %vector.body
3674
3675for.cond.cleanup: ; preds = %vector.body, %entry
3676 %s.0.lcssa = phi i16 [ 0, %entry ], [ %9, %vector.body ]
3677 ret i16 %s.0.lcssa
3678}
3679
3680define zeroext i8 @add16i8(i8* noalias nocapture readonly %x, i32 %n) {
3681; CHECK-LABEL: add16i8:
3682; CHECK: @ %bb.0: @ %entry
3683; CHECK-NEXT: .save {r7, lr}
3684; CHECK-NEXT: push {r7, lr}
3685; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
3686; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
3687; CHECK-NEXT: .pad #136
3688; CHECK-NEXT: sub sp, #136
3689; CHECK-NEXT: cmp r1, #0
3690; CHECK-NEXT: beq.w .LBB27_3
3691; CHECK-NEXT: @ %bb.1: @ %vector.ph
3692; CHECK-NEXT: add.w r2, r1, #15
3693; CHECK-NEXT: movs r3, #1
3694; CHECK-NEXT: bic r2, r2, #15
3695; CHECK-NEXT: vmov.i8 q7, #0x0
3696; CHECK-NEXT: subs r2, #16
3697; CHECK-NEXT: vmov.i8 q2, #0xff
3698; CHECK-NEXT: add.w lr, r3, r2, lsr #4
3699; CHECK-NEXT: adr r3, .LCPI27_0
3700; CHECK-NEXT: vldrw.u32 q0, [r3]
3701; CHECK-NEXT: adr r3, .LCPI27_1
3702; CHECK-NEXT: dls lr, lr
3703; CHECK-NEXT: subs r2, r1, #1
3704; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
3705; CHECK-NEXT: vldrw.u32 q0, [r3]
3706; CHECK-NEXT: adr r3, .LCPI27_2
3707; CHECK-NEXT: movs r1, #0
3708; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
3709; CHECK-NEXT: vldrw.u32 q0, [r3]
3710; CHECK-NEXT: adr r3, .LCPI27_3
3711; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
3712; CHECK-NEXT: vldrw.u32 q0, [r3]
3713; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
3714; CHECK-NEXT: vdup.32 q0, r2
3715; CHECK-NEXT: movs r2, #0
3716; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
3717; CHECK-NEXT: .LBB27_2: @ %vector.body
3718; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
3719; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
3720; CHECK-NEXT: vdup.32 q6, r1
3721; CHECK-NEXT: vadd.i32 q0, q0, r1
3722; CHECK-NEXT: vcmp.u32 hi, q6, q0
3723; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
3724; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
3725; CHECK-NEXT: vpsel q3, q2, q7
3726; CHECK-NEXT: vadd.i32 q0, q0, r1
3727; CHECK-NEXT: vcmp.u32 hi, q6, q0
3728; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
3729; CHECK-NEXT: vpsel q0, q2, q7
3730; CHECK-NEXT: vmov r3, s0
3731; CHECK-NEXT: vmov.16 q4[0], r3
3732; CHECK-NEXT: vmov r3, s1
3733; CHECK-NEXT: vmov.16 q4[1], r3
3734; CHECK-NEXT: vmov r3, s2
3735; CHECK-NEXT: vmov.16 q4[2], r3
3736; CHECK-NEXT: vmov r3, s3
3737; CHECK-NEXT: vmov.16 q4[3], r3
3738; CHECK-NEXT: vmov r3, s12
3739; CHECK-NEXT: vmov.16 q4[4], r3
3740; CHECK-NEXT: vmov r3, s13
3741; CHECK-NEXT: vmov.16 q4[5], r3
3742; CHECK-NEXT: vmov r3, s14
3743; CHECK-NEXT: vmov.16 q4[6], r3
3744; CHECK-NEXT: vmov r3, s15
3745; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
3746; CHECK-NEXT: vmov.16 q4[7], r3
3747; CHECK-NEXT: vcmp.i16 ne, q4, zr
3748; CHECK-NEXT: vadd.i32 q3, q0, r1
3749; CHECK-NEXT: vpsel q5, q2, q7
3750; CHECK-NEXT: vcmp.u32 hi, q6, q3
3751; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
3752; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
3753; CHECK-NEXT: vadd.i32 q4, q0, r1
3754; CHECK-NEXT: adds r1, #16
3755; CHECK-NEXT: vcmp.u32 hi, q6, q4
3756; CHECK-NEXT: vpsel q0, q2, q7
3757; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
3758; CHECK-NEXT: vmov r3, s0
3759; CHECK-NEXT: vmov.16 q6[0], r3
3760; CHECK-NEXT: vmov r3, s1
3761; CHECK-NEXT: vmov.16 q6[1], r3
3762; CHECK-NEXT: vmov r3, s2
3763; CHECK-NEXT: vmov.16 q6[2], r3
3764; CHECK-NEXT: vmov r3, s3
3765; CHECK-NEXT: vpsel q1, q2, q7
3766; CHECK-NEXT: vmov.16 q6[3], r3
3767; CHECK-NEXT: vmov r3, s4
3768; CHECK-NEXT: vmov.16 q6[4], r3
3769; CHECK-NEXT: vmov r3, s5
3770; CHECK-NEXT: vmov.16 q6[5], r3
3771; CHECK-NEXT: vmov r3, s6
3772; CHECK-NEXT: vmov.16 q6[6], r3
3773; CHECK-NEXT: vmov r3, s7
3774; CHECK-NEXT: vmov.16 q6[7], r3
3775; CHECK-NEXT: vcmp.i16 ne, q6, zr
3776; CHECK-NEXT: vpsel q0, q2, q7
3777; CHECK-NEXT: vmov.u16 r3, q0[0]
3778; CHECK-NEXT: vmov.8 q6[0], r3
3779; CHECK-NEXT: vmov.u16 r3, q0[1]
3780; CHECK-NEXT: vmov.8 q6[1], r3
3781; CHECK-NEXT: vmov.u16 r3, q0[2]
3782; CHECK-NEXT: vmov.8 q6[2], r3
3783; CHECK-NEXT: vmov.u16 r3, q0[3]
3784; CHECK-NEXT: vmov.8 q6[3], r3
3785; CHECK-NEXT: vmov.u16 r3, q0[4]
3786; CHECK-NEXT: vmov.8 q6[4], r3
3787; CHECK-NEXT: vmov.u16 r3, q0[5]
3788; CHECK-NEXT: vmov.8 q6[5], r3
3789; CHECK-NEXT: vmov.u16 r3, q0[6]
3790; CHECK-NEXT: vmov.8 q6[6], r3
3791; CHECK-NEXT: vmov.u16 r3, q0[7]
3792; CHECK-NEXT: vmov.8 q6[7], r3
3793; CHECK-NEXT: vmov.u16 r3, q5[0]
3794; CHECK-NEXT: vmov.8 q6[8], r3
3795; CHECK-NEXT: vmov.u16 r3, q5[1]
3796; CHECK-NEXT: vmov.8 q6[9], r3
3797; CHECK-NEXT: vmov.u16 r3, q5[2]
3798; CHECK-NEXT: vmov.8 q6[10], r3
3799; CHECK-NEXT: vmov.u16 r3, q5[3]
3800; CHECK-NEXT: vmov.8 q6[11], r3
3801; CHECK-NEXT: vmov.u16 r3, q5[4]
3802; CHECK-NEXT: vmov.8 q6[12], r3
3803; CHECK-NEXT: vmov.u16 r3, q5[5]
3804; CHECK-NEXT: vmov.8 q6[13], r3
3805; CHECK-NEXT: vmov.u16 r3, q5[6]
3806; CHECK-NEXT: vmov.8 q6[14], r3
3807; CHECK-NEXT: vmov.u16 r3, q5[7]
3808; CHECK-NEXT: vmov.8 q6[15], r3
3809; CHECK-NEXT: vcmp.i8 ne, q6, zr
3810; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
3811; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
3812; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
3813; CHECK-NEXT: vcmp.u32 cs, q6, q0
3814; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
3815; CHECK-NEXT: vpsel q5, q2, q7
3816; CHECK-NEXT: vcmp.u32 cs, q6, q0
3817; CHECK-NEXT: vpsel q1, q2, q7
3818; CHECK-NEXT: vmov r3, s4
3819; CHECK-NEXT: vmov.16 q0[0], r3
3820; CHECK-NEXT: vmov r3, s5
3821; CHECK-NEXT: vmov.16 q0[1], r3
3822; CHECK-NEXT: vmov r3, s6
3823; CHECK-NEXT: vmov.16 q0[2], r3
3824; CHECK-NEXT: vmov r3, s7
3825; CHECK-NEXT: vmov.16 q0[3], r3
3826; CHECK-NEXT: vmov r3, s20
3827; CHECK-NEXT: vmov.16 q0[4], r3
3828; CHECK-NEXT: vmov r3, s21
3829; CHECK-NEXT: vmov.16 q0[5], r3
3830; CHECK-NEXT: vmov r3, s22
3831; CHECK-NEXT: vmov.16 q0[6], r3
3832; CHECK-NEXT: vmov r3, s23
3833; CHECK-NEXT: vmov.16 q0[7], r3
3834; CHECK-NEXT: vcmp.i16 ne, q0, zr
3835; CHECK-NEXT: vpsel q0, q2, q7
3836; CHECK-NEXT: vcmp.u32 cs, q6, q3
3837; CHECK-NEXT: vpsel q3, q2, q7
3838; CHECK-NEXT: vcmp.u32 cs, q6, q4
3839; CHECK-NEXT: vpsel q1, q2, q7
3840; CHECK-NEXT: vmov r3, s4
3841; CHECK-NEXT: vmov.16 q4[0], r3
3842; CHECK-NEXT: vmov r3, s5
3843; CHECK-NEXT: vmov.16 q4[1], r3
3844; CHECK-NEXT: vmov r3, s6
3845; CHECK-NEXT: vmov.16 q4[2], r3
3846; CHECK-NEXT: vmov r3, s7
3847; CHECK-NEXT: vmov.16 q4[3], r3
3848; CHECK-NEXT: vmov r3, s12
3849; CHECK-NEXT: vmov.16 q4[4], r3
3850; CHECK-NEXT: vmov r3, s13
3851; CHECK-NEXT: vmov.16 q4[5], r3
3852; CHECK-NEXT: vmov r3, s14
3853; CHECK-NEXT: vmov.16 q4[6], r3
3854; CHECK-NEXT: vmov r3, s15
3855; CHECK-NEXT: vmov.16 q4[7], r3
3856; CHECK-NEXT: vcmp.i16 ne, q4, zr
3857; CHECK-NEXT: vpsel q4, q2, q7
3858; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
3859; CHECK-NEXT: vmov.u16 r3, q4[0]
3860; CHECK-NEXT: vmov.8 q3[0], r3
3861; CHECK-NEXT: vmov.u16 r3, q4[1]
3862; CHECK-NEXT: vmov.8 q3[1], r3
3863; CHECK-NEXT: vmov.u16 r3, q4[2]
3864; CHECK-NEXT: vmov.8 q3[2], r3
3865; CHECK-NEXT: vmov.u16 r3, q4[3]
3866; CHECK-NEXT: vmov.8 q3[3], r3
3867; CHECK-NEXT: vmov.u16 r3, q4[4]
3868; CHECK-NEXT: vmov.8 q3[4], r3
3869; CHECK-NEXT: vmov.u16 r3, q4[5]
3870; CHECK-NEXT: vmov.8 q3[5], r3
3871; CHECK-NEXT: vmov.u16 r3, q4[6]
3872; CHECK-NEXT: vmov.8 q3[6], r3
3873; CHECK-NEXT: vmov.u16 r3, q4[7]
3874; CHECK-NEXT: vmov.8 q3[7], r3
3875; CHECK-NEXT: vmov.u16 r3, q0[0]
3876; CHECK-NEXT: vmov.8 q3[8], r3
3877; CHECK-NEXT: vmov.u16 r3, q0[1]
3878; CHECK-NEXT: vmov.8 q3[9], r3
3879; CHECK-NEXT: vmov.u16 r3, q0[2]
3880; CHECK-NEXT: vmov.8 q3[10], r3
3881; CHECK-NEXT: vmov.u16 r3, q0[3]
3882; CHECK-NEXT: vmov.8 q3[11], r3
3883; CHECK-NEXT: vmov.u16 r3, q0[4]
3884; CHECK-NEXT: vmov.8 q3[12], r3
3885; CHECK-NEXT: vmov.u16 r3, q0[5]
3886; CHECK-NEXT: vmov.8 q3[13], r3
3887; CHECK-NEXT: vmov.u16 r3, q0[6]
3888; CHECK-NEXT: vmov.8 q3[14], r3
3889; CHECK-NEXT: vmov.u16 r3, q0[7]
3890; CHECK-NEXT: vpnot
3891; CHECK-NEXT: vmov.8 q3[15], r3
3892; CHECK-NEXT: vpsttt
3893; CHECK-NEXT: vcmpt.i8 ne, q3, zr
3894; CHECK-NEXT: vldrbt.u8 q0, [r0], #16
3895; CHECK-NEXT: vaddvat.u8 r2, q0
3896; CHECK-NEXT: le lr, .LBB27_2
3897; CHECK-NEXT: b .LBB27_4
3898; CHECK-NEXT: .LBB27_3:
3899; CHECK-NEXT: movs r2, #0
3900; CHECK-NEXT: .LBB27_4: @ %for.cond.cleanup
3901; CHECK-NEXT: uxtb r0, r2
3902; CHECK-NEXT: add sp, #136
3903; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
3904; CHECK-NEXT: pop {r7, pc}
3905; CHECK-NEXT: .p2align 4
3906; CHECK-NEXT: @ %bb.5:
3907; CHECK-NEXT: .LCPI27_0:
3908; CHECK-NEXT: .long 12 @ 0xc
3909; CHECK-NEXT: .long 13 @ 0xd
3910; CHECK-NEXT: .long 14 @ 0xe
3911; CHECK-NEXT: .long 15 @ 0xf
3912; CHECK-NEXT: .LCPI27_1:
3913; CHECK-NEXT: .long 8 @ 0x8
3914; CHECK-NEXT: .long 9 @ 0x9
3915; CHECK-NEXT: .long 10 @ 0xa
3916; CHECK-NEXT: .long 11 @ 0xb
3917; CHECK-NEXT: .LCPI27_2:
3918; CHECK-NEXT: .long 4 @ 0x4
3919; CHECK-NEXT: .long 5 @ 0x5
3920; CHECK-NEXT: .long 6 @ 0x6
3921; CHECK-NEXT: .long 7 @ 0x7
3922; CHECK-NEXT: .LCPI27_3:
3923; CHECK-NEXT: .long 0 @ 0x0
3924; CHECK-NEXT: .long 1 @ 0x1
3925; CHECK-NEXT: .long 2 @ 0x2
3926; CHECK-NEXT: .long 3 @ 0x3
3927entry:
3928 %cmp7.not = icmp eq i32 %n, 0
3929 br i1 %cmp7.not, label %for.cond.cleanup, label %vector.ph
3930
3931vector.ph: ; preds = %entry
3932 %n.rnd.up = add i32 %n, 15
3933 %n.vec = and i32 %n.rnd.up, -16
3934 %trip.count.minus.1 = add i32 %n, -1
3935 br label %vector.body
3936
3937vector.body: ; preds = %vector.body, %vector.ph
3938 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
3939 %vec.phi = phi i8 [ 0, %vector.ph ], [ %4, %vector.body ]
3940 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
3941 %0 = getelementptr inbounds i8, i8* %x, i32 %index
3942 %1 = bitcast i8* %0 to <16 x i8>*
3943 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
3944 %2 = select <16 x i1> %active.lane.mask, <16 x i8> %wide.masked.load, <16 x i8> zeroinitializer
3945 %3 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %2)
3946 %4 = add i8 %3, %vec.phi
3947 %index.next = add i32 %index, 16
3948 %5 = icmp eq i32 %index.next, %n.vec
3949 br i1 %5, label %for.cond.cleanup, label %vector.body
3950
3951for.cond.cleanup: ; preds = %vector.body, %entry
3952 %s.0.lcssa = phi i8 [ 0, %entry ], [ %4, %vector.body ]
3953 ret i8 %s.0.lcssa
3954}
3955
3956define zeroext i8 @mla16i8(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i32 %n) {
3957; CHECK-LABEL: mla16i8:
3958; CHECK: @ %bb.0: @ %entry
3959; CHECK-NEXT: .save {r7, lr}
3960; CHECK-NEXT: push {r7, lr}
3961; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
3962; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
3963; CHECK-NEXT: .pad #136
3964; CHECK-NEXT: sub sp, #136
3965; CHECK-NEXT: cmp r2, #0
3966; CHECK-NEXT: beq.w .LBB28_3
3967; CHECK-NEXT: @ %bb.1: @ %vector.ph
3968; CHECK-NEXT: add.w r3, r2, #15
3969; CHECK-NEXT: vmov.i8 q7, #0x0
3970; CHECK-NEXT: bic r3, r3, #15
3971; CHECK-NEXT: vmov.i8 q2, #0xff
3972; CHECK-NEXT: sub.w r12, r3, #16
3973; CHECK-NEXT: movs r3, #1
3974; CHECK-NEXT: add.w lr, r3, r12, lsr #4
3975; CHECK-NEXT: adr r3, .LCPI28_0
3976; CHECK-NEXT: vldrw.u32 q0, [r3]
3977; CHECK-NEXT: adr r3, .LCPI28_1
3978; CHECK-NEXT: dls lr, lr
3979; CHECK-NEXT: sub.w r12, r2, #1
3980; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
3981; CHECK-NEXT: vldrw.u32 q0, [r3]
3982; CHECK-NEXT: adr r3, .LCPI28_2
3983; CHECK-NEXT: movs r2, #0
3984; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
3985; CHECK-NEXT: vldrw.u32 q0, [r3]
3986; CHECK-NEXT: adr r3, .LCPI28_3
3987; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
3988; CHECK-NEXT: vldrw.u32 q0, [r3]
3989; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
3990; CHECK-NEXT: vdup.32 q0, r12
3991; CHECK-NEXT: mov.w r12, #0
3992; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
3993; CHECK-NEXT: .LBB28_2: @ %vector.body
3994; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
3995; CHECK-NEXT: vldrw.u32 q0, [sp, #64] @ 16-byte Reload
3996; CHECK-NEXT: vdup.32 q6, r2
3997; CHECK-NEXT: vadd.i32 q0, q0, r2
3998; CHECK-NEXT: vcmp.u32 hi, q6, q0
3999; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
4000; CHECK-NEXT: vldrw.u32 q0, [sp, #48] @ 16-byte Reload
4001; CHECK-NEXT: vpsel q3, q2, q7
4002; CHECK-NEXT: vadd.i32 q0, q0, r2
4003; CHECK-NEXT: vcmp.u32 hi, q6, q0
4004; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
4005; CHECK-NEXT: vpsel q0, q2, q7
4006; CHECK-NEXT: vmov r3, s0
4007; CHECK-NEXT: vmov.16 q4[0], r3
4008; CHECK-NEXT: vmov r3, s1
4009; CHECK-NEXT: vmov.16 q4[1], r3
4010; CHECK-NEXT: vmov r3, s2
4011; CHECK-NEXT: vmov.16 q4[2], r3
4012; CHECK-NEXT: vmov r3, s3
4013; CHECK-NEXT: vmov.16 q4[3], r3
4014; CHECK-NEXT: vmov r3, s12
4015; CHECK-NEXT: vmov.16 q4[4], r3
4016; CHECK-NEXT: vmov r3, s13
4017; CHECK-NEXT: vmov.16 q4[5], r3
4018; CHECK-NEXT: vmov r3, s14
4019; CHECK-NEXT: vmov.16 q4[6], r3
4020; CHECK-NEXT: vmov r3, s15
4021; CHECK-NEXT: vldrw.u32 q0, [sp, #32] @ 16-byte Reload
4022; CHECK-NEXT: vmov.16 q4[7], r3
4023; CHECK-NEXT: vcmp.i16 ne, q4, zr
4024; CHECK-NEXT: vadd.i32 q3, q0, r2
4025; CHECK-NEXT: vpsel q5, q2, q7
4026; CHECK-NEXT: vcmp.u32 hi, q6, q3
4027; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
4028; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
4029; CHECK-NEXT: vadd.i32 q4, q0, r2
4030; CHECK-NEXT: adds r2, #16
4031; CHECK-NEXT: vcmp.u32 hi, q6, q4
4032; CHECK-NEXT: vpsel q0, q2, q7
4033; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
4034; CHECK-NEXT: vmov r3, s0
4035; CHECK-NEXT: vmov.16 q6[0], r3
4036; CHECK-NEXT: vmov r3, s1
4037; CHECK-NEXT: vmov.16 q6[1], r3
4038; CHECK-NEXT: vmov r3, s2
4039; CHECK-NEXT: vmov.16 q6[2], r3
4040; CHECK-NEXT: vmov r3, s3
4041; CHECK-NEXT: vpsel q1, q2, q7
4042; CHECK-NEXT: vmov.16 q6[3], r3
4043; CHECK-NEXT: vmov r3, s4
4044; CHECK-NEXT: vmov.16 q6[4], r3
4045; CHECK-NEXT: vmov r3, s5
4046; CHECK-NEXT: vmov.16 q6[5], r3
4047; CHECK-NEXT: vmov r3, s6
4048; CHECK-NEXT: vmov.16 q6[6], r3
4049; CHECK-NEXT: vmov r3, s7
4050; CHECK-NEXT: vmov.16 q6[7], r3
4051; CHECK-NEXT: vcmp.i16 ne, q6, zr
4052; CHECK-NEXT: vpsel q0, q2, q7
4053; CHECK-NEXT: vmov.u16 r3, q0[0]
4054; CHECK-NEXT: vmov.8 q6[0], r3
4055; CHECK-NEXT: vmov.u16 r3, q0[1]
4056; CHECK-NEXT: vmov.8 q6[1], r3
4057; CHECK-NEXT: vmov.u16 r3, q0[2]
4058; CHECK-NEXT: vmov.8 q6[2], r3
4059; CHECK-NEXT: vmov.u16 r3, q0[3]
4060; CHECK-NEXT: vmov.8 q6[3], r3
4061; CHECK-NEXT: vmov.u16 r3, q0[4]
4062; CHECK-NEXT: vmov.8 q6[4], r3
4063; CHECK-NEXT: vmov.u16 r3, q0[5]
4064; CHECK-NEXT: vmov.8 q6[5], r3
4065; CHECK-NEXT: vmov.u16 r3, q0[6]
4066; CHECK-NEXT: vmov.8 q6[6], r3
4067; CHECK-NEXT: vmov.u16 r3, q0[7]
4068; CHECK-NEXT: vmov.8 q6[7], r3
4069; CHECK-NEXT: vmov.u16 r3, q5[0]
4070; CHECK-NEXT: vmov.8 q6[8], r3
4071; CHECK-NEXT: vmov.u16 r3, q5[1]
4072; CHECK-NEXT: vmov.8 q6[9], r3
4073; CHECK-NEXT: vmov.u16 r3, q5[2]
4074; CHECK-NEXT: vmov.8 q6[10], r3
4075; CHECK-NEXT: vmov.u16 r3, q5[3]
4076; CHECK-NEXT: vmov.8 q6[11], r3
4077; CHECK-NEXT: vmov.u16 r3, q5[4]
4078; CHECK-NEXT: vmov.8 q6[12], r3
4079; CHECK-NEXT: vmov.u16 r3, q5[5]
4080; CHECK-NEXT: vmov.8 q6[13], r3
4081; CHECK-NEXT: vmov.u16 r3, q5[6]
4082; CHECK-NEXT: vmov.8 q6[14], r3
4083; CHECK-NEXT: vmov.u16 r3, q5[7]
4084; CHECK-NEXT: vmov.8 q6[15], r3
4085; CHECK-NEXT: vcmp.i8 ne, q6, zr
4086; CHECK-NEXT: vstr p0, [sp, #92] @ 4-byte Spill
4087; CHECK-NEXT: vldrw.u32 q6, [sp] @ 16-byte Reload
4088; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
4089; CHECK-NEXT: vcmp.u32 cs, q6, q0
4090; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
4091; CHECK-NEXT: vpsel q5, q2, q7
4092; CHECK-NEXT: vcmp.u32 cs, q6, q0
4093; CHECK-NEXT: vpsel q1, q2, q7
4094; CHECK-NEXT: vmov r3, s4
4095; CHECK-NEXT: vmov.16 q0[0], r3
4096; CHECK-NEXT: vmov r3, s5
4097; CHECK-NEXT: vmov.16 q0[1], r3
4098; CHECK-NEXT: vmov r3, s6
4099; CHECK-NEXT: vmov.16 q0[2], r3
4100; CHECK-NEXT: vmov r3, s7
4101; CHECK-NEXT: vmov.16 q0[3], r3
4102; CHECK-NEXT: vmov r3, s20
4103; CHECK-NEXT: vmov.16 q0[4], r3
4104; CHECK-NEXT: vmov r3, s21
4105; CHECK-NEXT: vmov.16 q0[5], r3
4106; CHECK-NEXT: vmov r3, s22
4107; CHECK-NEXT: vmov.16 q0[6], r3
4108; CHECK-NEXT: vmov r3, s23
4109; CHECK-NEXT: vmov.16 q0[7], r3
4110; CHECK-NEXT: vcmp.i16 ne, q0, zr
4111; CHECK-NEXT: vpsel q0, q2, q7
4112; CHECK-NEXT: vcmp.u32 cs, q6, q3
4113; CHECK-NEXT: vpsel q3, q2, q7
4114; CHECK-NEXT: vcmp.u32 cs, q6, q4
4115; CHECK-NEXT: vpsel q1, q2, q7
4116; CHECK-NEXT: vmov r3, s4
4117; CHECK-NEXT: vmov.16 q4[0], r3
4118; CHECK-NEXT: vmov r3, s5
4119; CHECK-NEXT: vmov.16 q4[1], r3
4120; CHECK-NEXT: vmov r3, s6
4121; CHECK-NEXT: vmov.16 q4[2], r3
4122; CHECK-NEXT: vmov r3, s7
4123; CHECK-NEXT: vmov.16 q4[3], r3
4124; CHECK-NEXT: vmov r3, s12
4125; CHECK-NEXT: vmov.16 q4[4], r3
4126; CHECK-NEXT: vmov r3, s13
4127; CHECK-NEXT: vmov.16 q4[5], r3
4128; CHECK-NEXT: vmov r3, s14
4129; CHECK-NEXT: vmov.16 q4[6], r3
4130; CHECK-NEXT: vmov r3, s15
4131; CHECK-NEXT: vmov.16 q4[7], r3
4132; CHECK-NEXT: vcmp.i16 ne, q4, zr
4133; CHECK-NEXT: vpsel q4, q2, q7
4134; CHECK-NEXT: vldr p0, [sp, #92] @ 4-byte Reload
4135; CHECK-NEXT: vmov.u16 r3, q4[0]
4136; CHECK-NEXT: vmov.8 q3[0], r3
4137; CHECK-NEXT: vmov.u16 r3, q4[1]
4138; CHECK-NEXT: vmov.8 q3[1], r3
4139; CHECK-NEXT: vmov.u16 r3, q4[2]
4140; CHECK-NEXT: vmov.8 q3[2], r3
4141; CHECK-NEXT: vmov.u16 r3, q4[3]
4142; CHECK-NEXT: vmov.8 q3[3], r3
4143; CHECK-NEXT: vmov.u16 r3, q4[4]
4144; CHECK-NEXT: vmov.8 q3[4], r3
4145; CHECK-NEXT: vmov.u16 r3, q4[5]
4146; CHECK-NEXT: vmov.8 q3[5], r3
4147; CHECK-NEXT: vmov.u16 r3, q4[6]
4148; CHECK-NEXT: vmov.8 q3[6], r3
4149; CHECK-NEXT: vmov.u16 r3, q4[7]
4150; CHECK-NEXT: vmov.8 q3[7], r3
4151; CHECK-NEXT: vmov.u16 r3, q0[0]
4152; CHECK-NEXT: vmov.8 q3[8], r3
4153; CHECK-NEXT: vmov.u16 r3, q0[1]
4154; CHECK-NEXT: vmov.8 q3[9], r3
4155; CHECK-NEXT: vmov.u16 r3, q0[2]
4156; CHECK-NEXT: vmov.8 q3[10], r3
4157; CHECK-NEXT: vmov.u16 r3, q0[3]
4158; CHECK-NEXT: vmov.8 q3[11], r3
4159; CHECK-NEXT: vmov.u16 r3, q0[4]
4160; CHECK-NEXT: vmov.8 q3[12], r3
4161; CHECK-NEXT: vmov.u16 r3, q0[5]
4162; CHECK-NEXT: vmov.8 q3[13], r3
4163; CHECK-NEXT: vmov.u16 r3, q0[6]
4164; CHECK-NEXT: vmov.8 q3[14], r3
4165; CHECK-NEXT: vmov.u16 r3, q0[7]
4166; CHECK-NEXT: vpnot
4167; CHECK-NEXT: vmov.8 q3[15], r3
4168; CHECK-NEXT: vpstttt
4169; CHECK-NEXT: vcmpt.i8 ne, q3, zr
4170; CHECK-NEXT: vldrbt.u8 q0, [r0], #16
4171; CHECK-NEXT: vldrbt.u8 q1, [r1], #16
4172; CHECK-NEXT: vmlavat.u8 r12, q1, q0
4173; CHECK-NEXT: le lr, .LBB28_2
4174; CHECK-NEXT: b .LBB28_4
4175; CHECK-NEXT: .LBB28_3:
4176; CHECK-NEXT: mov.w r12, #0
4177; CHECK-NEXT: .LBB28_4: @ %for.cond.cleanup
4178; CHECK-NEXT: uxtb.w r0, r12
4179; CHECK-NEXT: add sp, #136
4180; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
4181; CHECK-NEXT: pop {r7, pc}
4182; CHECK-NEXT: .p2align 4
4183; CHECK-NEXT: @ %bb.5:
4184; CHECK-NEXT: .LCPI28_0:
4185; CHECK-NEXT: .long 12 @ 0xc
4186; CHECK-NEXT: .long 13 @ 0xd
4187; CHECK-NEXT: .long 14 @ 0xe
4188; CHECK-NEXT: .long 15 @ 0xf
4189; CHECK-NEXT: .LCPI28_1:
4190; CHECK-NEXT: .long 8 @ 0x8
4191; CHECK-NEXT: .long 9 @ 0x9
4192; CHECK-NEXT: .long 10 @ 0xa
4193; CHECK-NEXT: .long 11 @ 0xb
4194; CHECK-NEXT: .LCPI28_2:
4195; CHECK-NEXT: .long 4 @ 0x4
4196; CHECK-NEXT: .long 5 @ 0x5
4197; CHECK-NEXT: .long 6 @ 0x6
4198; CHECK-NEXT: .long 7 @ 0x7
4199; CHECK-NEXT: .LCPI28_3:
4200; CHECK-NEXT: .long 0 @ 0x0
4201; CHECK-NEXT: .long 1 @ 0x1
4202; CHECK-NEXT: .long 2 @ 0x2
4203; CHECK-NEXT: .long 3 @ 0x3
4204entry:
4205 %cmp10.not = icmp eq i32 %n, 0
4206 br i1 %cmp10.not, label %for.cond.cleanup, label %vector.ph
4207
4208vector.ph: ; preds = %entry
4209 %n.rnd.up = add i32 %n, 15
4210 %n.vec = and i32 %n.rnd.up, -16
4211 %trip.count.minus.1 = add i32 %n, -1
4212 br label %vector.body
4213
4214vector.body: ; preds = %vector.body, %vector.ph
4215 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
4216 %vec.phi = phi i8 [ 0, %vector.ph ], [ %7, %vector.body ]
4217 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %trip.count.minus.1)
4218 %0 = getelementptr inbounds i8, i8* %x, i32 %index
4219 %1 = bitcast i8* %0 to <16 x i8>*
4220 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %1, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
4221 %2 = getelementptr inbounds i8, i8* %y, i32 %index
4222 %3 = bitcast i8* %2 to <16 x i8>*
4223 %wide.masked.load15 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %3, i32 1, <16 x i1> %active.lane.mask, <16 x i8> undef)
4224 %4 = mul <16 x i8> %wide.masked.load15, %wide.masked.load
4225 %5 = select <16 x i1> %active.lane.mask, <16 x i8> %4, <16 x i8> zeroinitializer
4226 %6 = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %5)
4227 %7 = add i8 %6, %vec.phi
4228 %index.next = add i32 %index, 16
4229 %8 = icmp eq i32 %index.next, %n.vec
4230 br i1 %8, label %for.cond.cleanup, label %vector.body
4231
4232for.cond.cleanup: ; preds = %vector.body, %entry
4233 %s.0.lcssa = phi i8 [ 0, %entry ], [ %7, %vector.body ]
4234 ret i8 %s.0.lcssa
4235}
4236
4237define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
4238; CHECK-LABEL: add4i64:
4239; CHECK: @ %bb.0: @ %entry
4240; CHECK-NEXT: .save {r7, lr}
4241; CHECK-NEXT: push {r7, lr}
4242; CHECK-NEXT: cbz r1, .LBB29_4
4243; CHECK-NEXT: @ %bb.1: @ %vector.ph
4244; CHECK-NEXT: adds r2, r1, #3
4245; CHECK-NEXT: movs r3, #1
4246; CHECK-NEXT: bic r2, r2, #3
4247; CHECK-NEXT: subs r1, #1
4248; CHECK-NEXT: subs r2, #4
4249; CHECK-NEXT: vdup.32 q1, r1
4250; CHECK-NEXT: movs r1, #0
4251; CHECK-NEXT: add.w lr, r3, r2, lsr #2
4252; CHECK-NEXT: adr r2, .LCPI29_0
4253; CHECK-NEXT: vldrw.u32 q0, [r2]
4254; CHECK-NEXT: movs r3, #0
4255; CHECK-NEXT: movs r2, #0
4256; CHECK-NEXT: dls lr, lr
4257; CHECK-NEXT: .LBB29_2: @ %vector.body
4258; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
4259; CHECK-NEXT: vadd.i32 q2, q0, r3
4260; CHECK-NEXT: vdup.32 q3, r3
4261; CHECK-NEXT: vcmp.u32 hi, q3, q2
4262; CHECK-NEXT: adds r3, #4
4263; CHECK-NEXT: vpnot
4264; CHECK-NEXT: vpsttt
4265; CHECK-NEXT: vcmpt.u32 cs, q1, q2
4266; CHECK-NEXT: vldrwt.u32 q2, [r0], #16
4267; CHECK-NEXT: vaddlvat.s32 r2, r1, q2
4268; CHECK-NEXT: le lr, .LBB29_2
4269; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup
4270; CHECK-NEXT: mov r0, r2
4271; CHECK-NEXT: pop {r7, pc}
4272; CHECK-NEXT: .LBB29_4:
4273; CHECK-NEXT: movs r2, #0
4274; CHECK-NEXT: mov r1, r2
4275; CHECK-NEXT: mov r0, r2
4276; CHECK-NEXT: pop {r7, pc}
4277; CHECK-NEXT: .p2align 4
4278; CHECK-NEXT: @ %bb.5:
4279; CHECK-NEXT: .LCPI29_0:
4280; CHECK-NEXT: .long 0 @ 0x0
4281; CHECK-NEXT: .long 1 @ 0x1
4282; CHECK-NEXT: .long 2 @ 0x2
4283; CHECK-NEXT: .long 3 @ 0x3
4284entry:
4285 %cmp6.not = icmp eq i32 %n, 0
4286 br i1 %cmp6.not, label %for.cond.cleanup, label %vector.ph
4287
4288vector.ph: ; preds = %entry
4289 %n.rnd.up = add i32 %n, 3
4290 %n.vec = and i32 %n.rnd.up, -4
4291 %trip.count.minus.1 = add i32 %n, -1
4292 br label %vector.body
4293
4294vector.body: ; preds = %vector.body, %vector.ph
4295 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
4296 %vec.phi = phi i64 [ 0, %vector.ph ], [ %5, %vector.body ]
4297 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
4298 %0 = getelementptr inbounds i32, i32* %x, i32 %index
4299 %1 = bitcast i32* %0 to <4 x i32>*
4300 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
4301 %2 = sext <4 x i32> %wide.masked.load to <4 x i64>
4302 %3 = select <4 x i1> %active.lane.mask, <4 x i64> %2, <4 x i64> zeroinitializer
4303 %4 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %3)
4304 %5 = add i64 %4, %vec.phi
4305 %index.next = add i32 %index, 4
4306 %6 = icmp eq i32 %index.next, %n.vec
4307 br i1 %6, label %for.cond.cleanup, label %vector.body
4308
4309for.cond.cleanup: ; preds = %vector.body, %entry
4310 %s.0.lcssa = phi i64 [ 0, %entry ], [ %5, %vector.body ]
4311 ret i64 %s.0.lcssa
4312}
4313
4314define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32 %n) {
4315; CHECK-LABEL: mla4i64:
4316; CHECK: @ %bb.0: @ %entry
4317; CHECK-NEXT: .save {r7, lr}
4318; CHECK-NEXT: push {r7, lr}
4319; CHECK-NEXT: cbz r2, .LBB30_3
4320; CHECK-NEXT: @ %bb.1: @ %vector.ph
4321; CHECK-NEXT: adds r3, r2, #3
4322; CHECK-NEXT: bic r3, r3, #3
4323; CHECK-NEXT: sub.w r12, r3, #4
4324; CHECK-NEXT: movs r3, #1
4325; CHECK-NEXT: add.w lr, r3, r12, lsr #2
4326; CHECK-NEXT: adr r3, .LCPI30_0
4327; CHECK-NEXT: sub.w r12, r2, #1
4328; CHECK-NEXT: vldrw.u32 q0, [r3]
4329; CHECK-NEXT: vdup.32 q1, r12
4330; CHECK-NEXT: movs r2, #0
4331; CHECK-NEXT: mov.w r12, #0
4332; CHECK-NEXT: movs r3, #0
4333; CHECK-NEXT: dls lr, lr
4334; CHECK-NEXT: .LBB30_2: @ %vector.body
4335; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
4336; CHECK-NEXT: vadd.i32 q2, q0, r2
4337; CHECK-NEXT: vdup.32 q3, r2
4338; CHECK-NEXT: vcmp.u32 hi, q3, q2
4339; CHECK-NEXT: adds r2, #4
4340; CHECK-NEXT: vpnot
4341; CHECK-NEXT: vpstttt
4342; CHECK-NEXT: vcmpt.u32 cs, q1, q2
4343; CHECK-NEXT: vldrwt.u32 q2, [r0], #16
4344; CHECK-NEXT: vldrwt.u32 q3, [r1], #16
4345; CHECK-NEXT: vmlalvat.s32 r12, r3, q3, q2
4346; CHECK-NEXT: le lr, .LBB30_2
4347; CHECK-NEXT: b .LBB30_4
4348; CHECK-NEXT: .LBB30_3:
4349; CHECK-NEXT: mov.w r12, #0
4350; CHECK-NEXT: mov r3, r12
4351; CHECK-NEXT: .LBB30_4: @ %for.cond.cleanup
4352; CHECK-NEXT: mov r0, r12
4353; CHECK-NEXT: mov r1, r3
4354; CHECK-NEXT: pop {r7, pc}
4355; CHECK-NEXT: .p2align 4
4356; CHECK-NEXT: @ %bb.5:
4357; CHECK-NEXT: .LCPI30_0:
4358; CHECK-NEXT: .long 0 @ 0x0
4359; CHECK-NEXT: .long 1 @ 0x1
4360; CHECK-NEXT: .long 2 @ 0x2
4361; CHECK-NEXT: .long 3 @ 0x3
4362entry:
4363 %cmp9.not = icmp eq i32 %n, 0
4364 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
4365
4366vector.ph: ; preds = %entry
4367 %n.rnd.up = add i32 %n, 3
4368 %n.vec = and i32 %n.rnd.up, -4
4369 %trip.count.minus.1 = add i32 %n, -1
4370 br label %vector.body
4371
4372vector.body: ; preds = %vector.body, %vector.ph
4373 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
4374 %vec.phi = phi i64 [ 0, %vector.ph ], [ %9, %vector.body ]
4375 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %trip.count.minus.1)
4376 %0 = getelementptr inbounds i32, i32* %x, i32 %index
4377 %1 = bitcast i32* %0 to <4 x i32>*
4378 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
4379 %2 = sext <4 x i32> %wide.masked.load to <4 x i64>
4380 %3 = getelementptr inbounds i32, i32* %y, i32 %index
4381 %4 = bitcast i32* %3 to <4 x i32>*
4382 %wide.masked.load14 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)
4383 %5 = sext <4 x i32> %wide.masked.load14 to <4 x i64>
4384 %6 = mul nsw <4 x i64> %5, %2
4385 %7 = select <4 x i1> %active.lane.mask, <4 x i64> %6, <4 x i64> zeroinitializer
4386 %8 = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %7)
4387 %9 = add i64 %8, %vec.phi
4388 %index.next = add i32 %index, 4
4389 %10 = icmp eq i32 %index.next, %n.vec
4390 br i1 %10, label %for.cond.cleanup, label %vector.body
4391
4392for.cond.cleanup: ; preds = %vector.body, %entry
4393 %s.0.lcssa = phi i64 [ 0, %entry ], [ %9, %vector.body ]
4394 ret i64 %s.0.lcssa
4395}
4396
4397define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i32 %n) {
4398; CHECK-LABEL: mla8i64:
4399; CHECK: @ %bb.0: @ %entry
4400; CHECK-NEXT: .save {r4, lr}
4401; CHECK-NEXT: push {r4, lr}
4402; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
4403; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
4404; CHECK-NEXT: .pad #40
4405; CHECK-NEXT: sub sp, #40
4406; CHECK-NEXT: cmp r2, #0
4407; CHECK-NEXT: beq.w .LBB31_3
4408; CHECK-NEXT: @ %bb.1: @ %vector.ph
4409; CHECK-NEXT: adds r3, r2, #7
4410; CHECK-NEXT: vmov.i8 q1, #0x0
4411; CHECK-NEXT: bic r3, r3, #7
4412; CHECK-NEXT: vmov.i8 q2, #0xff
4413; CHECK-NEXT: sub.w r12, r3, #8
4414; CHECK-NEXT: movs r3, #1
4415; CHECK-NEXT: add.w lr, r3, r12, lsr #3
4416; CHECK-NEXT: adr r3, .LCPI31_0
4417; CHECK-NEXT: vldrw.u32 q0, [r3]
4418; CHECK-NEXT: adr r3, .LCPI31_1
4419; CHECK-NEXT: sub.w r12, r2, #1
4420; CHECK-NEXT: dls lr, lr
4421; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
4422; CHECK-NEXT: vldrw.u32 q0, [r3]
4423; CHECK-NEXT: vdup.32 q4, r12
4424; CHECK-NEXT: movs r2, #0
4425; CHECK-NEXT: mov.w r12, #0
4426; CHECK-NEXT: movs r3, #0
4427; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
4428; CHECK-NEXT: .LBB31_2: @ %vector.body
4429; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
4430; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
4431; CHECK-NEXT: vldrw.u32 q3, [sp] @ 16-byte Reload
4432; CHECK-NEXT: vadd.i32 q5, q0, r2
4433; CHECK-NEXT: vdup.32 q0, r2
4434; CHECK-NEXT: vcmp.u32 hi, q0, q5
4435; CHECK-NEXT: vadd.i32 q6, q3, r2
4436; CHECK-NEXT: vpsel q7, q2, q1
4437; CHECK-NEXT: vcmp.u32 hi, q0, q6
4438; CHECK-NEXT: vpsel q3, q2, q1
4439; CHECK-NEXT: adds r2, #8
4440; CHECK-NEXT: vmov r4, s12
4441; CHECK-NEXT: vmov.16 q0[0], r4
4442; CHECK-NEXT: vmov r4, s13
4443; CHECK-NEXT: vmov.16 q0[1], r4
4444; CHECK-NEXT: vmov r4, s14
4445; CHECK-NEXT: vmov.16 q0[2], r4
4446; CHECK-NEXT: vmov r4, s15
4447; CHECK-NEXT: vmov.16 q0[3], r4
4448; CHECK-NEXT: vmov r4, s28
4449; CHECK-NEXT: vmov.16 q0[4], r4
4450; CHECK-NEXT: vmov r4, s29
4451; CHECK-NEXT: vmov.16 q0[5], r4
4452; CHECK-NEXT: vmov r4, s30
4453; CHECK-NEXT: vmov.16 q0[6], r4
4454; CHECK-NEXT: vmov r4, s31
4455; CHECK-NEXT: vmov.16 q0[7], r4
4456; CHECK-NEXT: vcmp.i16 ne, q0, zr
4457; CHECK-NEXT: vstr p0, [sp, #36] @ 4-byte Spill
4458; CHECK-NEXT: vcmp.u32 cs, q4, q5
4459; CHECK-NEXT: vpsel q5, q2, q1
4460; CHECK-NEXT: vcmp.u32 cs, q4, q6
4461; CHECK-NEXT: vpsel q0, q2, q1
4462; CHECK-NEXT: vldr p0, [sp, #36] @ 4-byte Reload
4463; CHECK-NEXT: vmov r4, s0
4464; CHECK-NEXT: vmov.16 q6[0], r4
4465; CHECK-NEXT: vmov r4, s1
4466; CHECK-NEXT: vmov.16 q6[1], r4
4467; CHECK-NEXT: vmov r4, s2
4468; CHECK-NEXT: vmov.16 q6[2], r4
4469; CHECK-NEXT: vmov r4, s3
4470; CHECK-NEXT: vmov.16 q6[3], r4
4471; CHECK-NEXT: vmov r4, s20
4472; CHECK-NEXT: vmov.16 q6[4], r4
4473; CHECK-NEXT: vmov r4, s21
4474; CHECK-NEXT: vmov.16 q6[5], r4
4475; CHECK-NEXT: vmov r4, s22
4476; CHECK-NEXT: vmov.16 q6[6], r4
4477; CHECK-NEXT: vmov r4, s23
4478; CHECK-NEXT: vpnot
4479; CHECK-NEXT: vmov.16 q6[7], r4
4480; CHECK-NEXT: vpstttt
4481; CHECK-NEXT: vcmpt.i16 ne, q6, zr
4482; CHECK-NEXT: vldrht.u16 q0, [r0], #16
4483; CHECK-NEXT: vldrht.u16 q3, [r1], #16
4484; CHECK-NEXT: vmlalvat.s16 r12, r3, q3, q0
4485; CHECK-NEXT: le lr, .LBB31_2
4486; CHECK-NEXT: b .LBB31_4
4487; CHECK-NEXT: .LBB31_3:
4488; CHECK-NEXT: mov.w r12, #0
4489; CHECK-NEXT: mov r3, r12
4490; CHECK-NEXT: .LBB31_4: @ %for.cond.cleanup
4491; CHECK-NEXT: mov r0, r12
4492; CHECK-NEXT: mov r1, r3
4493; CHECK-NEXT: add sp, #40
4494; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
4495; CHECK-NEXT: pop {r4, pc}
4496; CHECK-NEXT: .p2align 4
4497; CHECK-NEXT: @ %bb.5:
4498; CHECK-NEXT: .LCPI31_0:
4499; CHECK-NEXT: .long 4 @ 0x4
4500; CHECK-NEXT: .long 5 @ 0x5
4501; CHECK-NEXT: .long 6 @ 0x6
4502; CHECK-NEXT: .long 7 @ 0x7
4503; CHECK-NEXT: .LCPI31_1:
4504; CHECK-NEXT: .long 0 @ 0x0
4505; CHECK-NEXT: .long 1 @ 0x1
4506; CHECK-NEXT: .long 2 @ 0x2
4507; CHECK-NEXT: .long 3 @ 0x3
4508entry:
4509 %cmp9.not = icmp eq i32 %n, 0
4510 br i1 %cmp9.not, label %for.cond.cleanup, label %vector.ph
4511
4512vector.ph: ; preds = %entry
4513 %n.rnd.up = add i32 %n, 7
4514 %n.vec = and i32 %n.rnd.up, -8
4515 %trip.count.minus.1 = add i32 %n, -1
4516 br label %vector.body
4517
4518vector.body: ; preds = %vector.body, %vector.ph
4519 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
4520 %vec.phi = phi i64 [ 0, %vector.ph ], [ %9, %vector.body ]
4521 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %trip.count.minus.1)
4522 %0 = getelementptr inbounds i16, i16* %x, i32 %index
4523 %1 = bitcast i16* %0 to <8 x i16>*
4524 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %1, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
4525 %2 = sext <8 x i16> %wide.masked.load to <8 x i64>
4526 %3 = getelementptr inbounds i16, i16* %y, i32 %index
4527 %4 = bitcast i16* %3 to <8 x i16>*
4528 %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %4, i32 2, <8 x i1> %active.lane.mask, <8 x i16> undef)
4529 %5 = sext <8 x i16> %wide.masked.load14 to <8 x i64>
4530 %6 = mul nsw <8 x i64> %5, %2
4531 %7 = select <8 x i1> %active.lane.mask, <8 x i64> %6, <8 x i64> zeroinitializer
4532 %8 = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %7)
4533 %9 = add i64 %8, %vec.phi
4534 %index.next = add i32 %index, 8
4535 %10 = icmp eq i32 %index.next, %n.vec
4536 br i1 %10, label %for.cond.cleanup, label %vector.body
4537
4538for.cond.cleanup: ; preds = %vector.body, %entry
4539 %s.0.lcssa = phi i64 [ 0, %entry ], [ %9, %vector.body ]
4540 ret i64 %s.0.lcssa
4541}
4542
4543declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) #1
4544declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #2
4545declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32) #1
4546declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>) #2
4547declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>) #3
4548declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32) #1
4549declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) #2
4550declare i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32>) #3
4551declare i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16>) #3
4552declare i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16>) #3
4553declare i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8>) #3
4554declare i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64>) #3
4555declare i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64>) #3
4556
David Greenc7551572020-06-09 11:04:29 +0100[diff] [blame]4557declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
4558declare i32 @llvm.experimental.vector.reduce.mul.v4i32(<4 x i32>)
4559declare i32 @llvm.experimental.vector.reduce.and.v4i32(<4 x i32>)
4560declare i32 @llvm.experimental.vector.reduce.or.v4i32(<4 x i32>)
4561declare i32 @llvm.experimental.vector.reduce.xor.v4i32(<4 x i32>)
4562declare float @llvm.experimental.vector.reduce.v2.fadd.f32.v4f32(float, <4 x float>)
4563declare float @llvm.experimental.vector.reduce.v2.fmul.f32.v4f32(float, <4 x float>)
4564declare i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32>)
4565declare i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32>)
4566declare i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32>)
4567declare i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32>)
4568declare float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float>)
4569declare float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float>)