blob: 435efe0c9c453b35a93c59a7cdf39782a9f82e22 [file] [log] [blame]
Matt Arsenault8d630032015-02-20 22:10:41 +00001; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
2
3; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
4; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
5; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
6
7; Make sure we don't form mad with denormals
8; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=FUNC %s
9; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
10
11declare i32 @llvm.r600.read.tidig.x() #0
12declare float @llvm.fabs.f32(float) #0
13declare float @llvm.fma.f32(float, float, float) #0
14declare float @llvm.fmuladd.f32(float, float, float) #0
15
16; (fadd (fmul x, y), z) -> (fma x, y, z)
17; FUNC-LABEL: {{^}}combine_to_mad_f32_0:
18; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
19; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
20; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
21
22; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
23
24; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
25
26; SI-DENORM-SLOWFMAF-NOT: v_fma
27; SI-DENORM-SLOWFMAF-NOT: v_mad
28
29; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
30; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
31
32; SI: buffer_store_dword [[RESULT]]
33define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
34 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +000035 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
36 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
37 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
38 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +000039
40 %a = load float addrspace(1)* %gep.0
41 %b = load float addrspace(1)* %gep.1
42 %c = load float addrspace(1)* %gep.2
43
44 %mul = fmul float %a, %b
45 %fma = fadd float %mul, %c
46 store float %fma, float addrspace(1)* %gep.out
47 ret void
48}
49
50; (fadd (fmul x, y), z) -> (fma x, y, z)
51; FUNC-LABEL: {{^}}combine_to_mad_f32_0_2use:
52; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
53; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
54; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
55; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
56
57; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
58; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
59
60; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
61; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
62
63; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
64; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
65; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
66
67; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
68; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
69; SI: s_endpgm
70define void @combine_to_mad_f32_0_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
71 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +000072 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
73 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
74 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
75 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
76 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
77 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
Matt Arsenault8d630032015-02-20 22:10:41 +000078
79 %a = load float addrspace(1)* %gep.0
80 %b = load float addrspace(1)* %gep.1
81 %c = load float addrspace(1)* %gep.2
82 %d = load float addrspace(1)* %gep.3
83
84 %mul = fmul float %a, %b
85 %fma0 = fadd float %mul, %c
86 %fma1 = fadd float %mul, %d
87
88 store float %fma0, float addrspace(1)* %gep.out.0
89 store float %fma1, float addrspace(1)* %gep.out.1
90 ret void
91}
92
93; (fadd x, (fmul y, z)) -> (fma y, z, x)
94; FUNC-LABEL: {{^}}combine_to_mad_f32_1:
95; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
96; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
97; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
98
99; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
100; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
101
102; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
103; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
104
105; SI: buffer_store_dword [[RESULT]]
106define void @combine_to_mad_f32_1(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
107 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000108 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
109 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
110 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
111 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000112
113 %a = load float addrspace(1)* %gep.0
114 %b = load float addrspace(1)* %gep.1
115 %c = load float addrspace(1)* %gep.2
116
117 %mul = fmul float %a, %b
118 %fma = fadd float %c, %mul
119 store float %fma, float addrspace(1)* %gep.out
120 ret void
121}
122
123; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
124; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32:
125; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
126; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
127; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
128
129; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
130; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
131
132; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
133; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
134
135; SI: buffer_store_dword [[RESULT]]
136define void @combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
137 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000138 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
139 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
140 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
141 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000142
143 %a = load float addrspace(1)* %gep.0
144 %b = load float addrspace(1)* %gep.1
145 %c = load float addrspace(1)* %gep.2
146
147 %mul = fmul float %a, %b
148 %fma = fsub float %mul, %c
149 store float %fma, float addrspace(1)* %gep.out
150 ret void
151}
152
153; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
154; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32_2use:
155; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
156; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
157; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
158; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
159
160; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
161; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
162
163; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
164; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
165
166; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
167; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
168; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
169
170; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
171; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
172; SI: s_endpgm
173define void @combine_to_mad_fsub_0_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
174 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000175 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
176 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
177 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
178 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
179 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
180 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
Matt Arsenault8d630032015-02-20 22:10:41 +0000181
182 %a = load float addrspace(1)* %gep.0
183 %b = load float addrspace(1)* %gep.1
184 %c = load float addrspace(1)* %gep.2
185 %d = load float addrspace(1)* %gep.3
186
187 %mul = fmul float %a, %b
188 %fma0 = fsub float %mul, %c
189 %fma1 = fsub float %mul, %d
190 store float %fma0, float addrspace(1)* %gep.out.0
191 store float %fma1, float addrspace(1)* %gep.out.1
192 ret void
193}
194
195; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
196; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32:
197; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
198; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
199; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
200
201; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
202; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
203
204; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
205; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
206
207; SI: buffer_store_dword [[RESULT]]
208define void @combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
209 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000210 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
211 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
212 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
213 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000214
215 %a = load float addrspace(1)* %gep.0
216 %b = load float addrspace(1)* %gep.1
217 %c = load float addrspace(1)* %gep.2
218
219 %mul = fmul float %a, %b
220 %fma = fsub float %c, %mul
221 store float %fma, float addrspace(1)* %gep.out
222 ret void
223}
224
225; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
226; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32_2use:
227; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
228; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
229; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
230
231; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
232; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
233
234; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
235; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
236
237; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
238; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
239; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
240
241; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
242; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
243; SI: s_endpgm
244define void @combine_to_mad_fsub_1_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
245 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000246 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
247 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
248 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
249 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
250 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
251 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
Matt Arsenault8d630032015-02-20 22:10:41 +0000252
253 %a = load float addrspace(1)* %gep.0
254 %b = load float addrspace(1)* %gep.1
255 %c = load float addrspace(1)* %gep.2
256 %d = load float addrspace(1)* %gep.3
257
258 %mul = fmul float %a, %b
259 %fma0 = fsub float %c, %mul
260 %fma1 = fsub float %d, %mul
261 store float %fma0, float addrspace(1)* %gep.out.0
262 store float %fma1, float addrspace(1)* %gep.out.1
263 ret void
264}
265
266; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
267; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32:
268; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
269; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
270; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
271
272; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
273
274; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
275
276; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
277; SI-DENORM-SLOWFMAF: v_sub_f32_e64 [[RESULT:v[0-9]+]], -[[TMP]], [[C]]
278
279; SI: buffer_store_dword [[RESULT]]
280define void @combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
281 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000282 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
283 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
284 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
285 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000286
287 %a = load float addrspace(1)* %gep.0
288 %b = load float addrspace(1)* %gep.1
289 %c = load float addrspace(1)* %gep.2
290
291 %mul = fmul float %a, %b
292 %mul.neg = fsub float -0.0, %mul
293 %fma = fsub float %mul.neg, %c
294
295 store float %fma, float addrspace(1)* %gep.out
296 ret void
297}
298
299; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
300; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_neg:
301; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
302; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
303; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
304
305; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
306; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
307
308; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
309; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
310
311; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
312; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
313; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT1:v[0-9]+]], -[[TMP]], [[D]]
314
315; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
316; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
317; SI: s_endpgm
318define void @combine_to_mad_fsub_2_f32_2uses_neg(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
319 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000320 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
321 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
322 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
323 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
324 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
325 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
Matt Arsenault8d630032015-02-20 22:10:41 +0000326
327 %a = load float addrspace(1)* %gep.0
328 %b = load float addrspace(1)* %gep.1
329 %c = load float addrspace(1)* %gep.2
330 %d = load float addrspace(1)* %gep.3
331
332 %mul = fmul float %a, %b
333 %mul.neg = fsub float -0.0, %mul
334 %fma0 = fsub float %mul.neg, %c
335 %fma1 = fsub float %mul.neg, %d
336
337 store float %fma0, float addrspace(1)* %gep.out.0
338 store float %fma1, float addrspace(1)* %gep.out.1
339 ret void
340}
341
342; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
343; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_mul:
344; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
345; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
346; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
347
348; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
349; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
350
351; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
352; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
353
354; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
355; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
356; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
357
358; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
359; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
360; SI: s_endpgm
361define void @combine_to_mad_fsub_2_f32_2uses_mul(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
362 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000363 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
364 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
365 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
366 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
367 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
368 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
Matt Arsenault8d630032015-02-20 22:10:41 +0000369
370 %a = load float addrspace(1)* %gep.0
371 %b = load float addrspace(1)* %gep.1
372 %c = load float addrspace(1)* %gep.2
373 %d = load float addrspace(1)* %gep.3
374
375 %mul = fmul float %a, %b
376 %mul.neg = fsub float -0.0, %mul
377 %fma0 = fsub float %mul.neg, %c
378 %fma1 = fsub float %mul, %d
379
380 store float %fma0, float addrspace(1)* %gep.out.0
381 store float %fma1, float addrspace(1)* %gep.out.1
382 ret void
383}
384
385; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
386
387; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_0_f32:
388; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
389; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
390; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
391; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
392; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
393
394; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
395; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
396; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP1]]
397
398; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], [[D]], [[E]], -[[C]]
399; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP0]]
400
401; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
402; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
403; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[C]], [[TMP1]]
404
405; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
406define void @aggressive_combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
407 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000408 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
409 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
410 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
411 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
412 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
413 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000414
415 %x = load float addrspace(1)* %gep.0
416 %y = load float addrspace(1)* %gep.1
417 %z = load float addrspace(1)* %gep.2
418 %u = load float addrspace(1)* %gep.3
419 %v = load float addrspace(1)* %gep.4
420
421 %tmp0 = fmul float %u, %v
422 %tmp1 = call float @llvm.fma.f32(float %x, float %y, float %tmp0) #0
423 %tmp2 = fsub float %tmp1, %z
424
425 store float %tmp2, float addrspace(1)* %gep.out
426 ret void
427}
428
429; fold (fsub x, (fma y, z, (fmul u, v)))
430; -> (fma (fneg y), z, (fma (fneg u), v, x))
431
432; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_1_f32:
433; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
434; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
435; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
436; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
437; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
438
439; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
440; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
441; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
442
443; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], -[[D]], [[E]], [[A]]
444; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP0]]
445
446; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
447; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
448; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
449
450; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
451; SI: s_endpgm
452define void @aggressive_combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
453 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000454 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
455 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
456 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
457 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
458 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
459 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000460
461 %x = load float addrspace(1)* %gep.0
462 %y = load float addrspace(1)* %gep.1
463 %z = load float addrspace(1)* %gep.2
464 %u = load float addrspace(1)* %gep.3
465 %v = load float addrspace(1)* %gep.4
466
467 %tmp0 = fmul float %u, %v
468 %tmp1 = call float @llvm.fma.f32(float %y, float %z, float %tmp0) #0
469 %tmp2 = fsub float %x, %tmp1
470
471 store float %tmp2, float addrspace(1)* %gep.out
472 ret void
473}
474
475; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
476
477; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_2_f32:
478; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
479; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
480; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
481; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
482; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
483
484; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
485; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
486
487; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
488; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
489
490; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
491; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[B]], [[A]]
492; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
493; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP2]]
494
495; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
496; SI: s_endpgm
497define void @aggressive_combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
498 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000499 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
500 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
501 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
502 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
503 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
504 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000505
506 %x = load float addrspace(1)* %gep.0
507 %y = load float addrspace(1)* %gep.1
508 %z = load float addrspace(1)* %gep.2
509 %u = load float addrspace(1)* %gep.3
510 %v = load float addrspace(1)* %gep.4
511
512 %tmp0 = fmul float %u, %v
513 %tmp1 = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0) #0
514 %tmp2 = fsub float %tmp1, %z
515
516 store float %tmp2, float addrspace(1)* %gep.out
517 ret void
518}
519
520; fold (fsub x, (fmuladd y, z, (fmul u, v)))
521; -> (fmuladd (fneg y), z, (fmuladd (fneg u), v, x))
522
523; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_3_f32:
524; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
525; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
526; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
527; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
528; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
529
530; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
531; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
532
533; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
534; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
535
536; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
537; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[C]], [[B]]
538; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
539; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP2]], [[A]]
540
541; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
542; SI: s_endpgm
543define void @aggressive_combine_to_mad_fsub_3_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
544 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
David Blaikie79e6c742015-02-27 19:29:02 +0000545 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
546 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
547 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
548 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
549 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
550 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
Matt Arsenault8d630032015-02-20 22:10:41 +0000551
552 %x = load float addrspace(1)* %gep.0
553 %y = load float addrspace(1)* %gep.1
554 %z = load float addrspace(1)* %gep.2
555 %u = load float addrspace(1)* %gep.3
556 %v = load float addrspace(1)* %gep.4
557
558 %tmp0 = fmul float %u, %v
559 %tmp1 = call float @llvm.fmuladd.f32(float %y, float %z, float %tmp0) #0
560 %tmp2 = fsub float %x, %tmp1
561
562 store float %tmp2, float addrspace(1)* %gep.out
563 ret void
564}
565
566attributes #0 = { nounwind readnone }
567attributes #1 = { nounwind }