Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / ff212f25feb44a915ce9c0144faef7fae27a6e61 / . / drivers / gpu / drm / radeon / cik.c
blob: 835dcfb7891625a64a45a9abe9a1a008c4d431ba [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchera1d6f972013-09-06 12:33:04 -0400[diff] [blame]78extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]79static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]80static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]81static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]82static void cik_init_pg(struct radeon_device *rdev);
83static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]84static void cik_fini_pg(struct radeon_device *rdev);
85static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]86static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
87 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]88
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]89/* get temperature in millidegrees */
90int ci_get_temp(struct radeon_device *rdev)
91{
92 u32 temp;
93 int actual_temp = 0;
94
95 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
96 CTF_TEMP_SHIFT;
97
98 if (temp & 0x200)
99 actual_temp = 255;
100 else
101 actual_temp = temp & 0x1ff;
102
103 actual_temp = actual_temp * 1000;
104
105 return actual_temp;
106}
107
108/* get temperature in millidegrees */
109int kv_get_temp(struct radeon_device *rdev)
110{
111 u32 temp;
112 int actual_temp = 0;
113
114 temp = RREG32_SMC(0xC0300E0C);
115
116 if (temp)
117 actual_temp = (temp / 8) - 49;
118 else
119 actual_temp = 0;
120
121 actual_temp = actual_temp * 1000;
122
123 return actual_temp;
124}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]125
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]126/*
127 * Indirect registers accessor
128 */
129u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
130{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]131 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]132 u32 r;
133
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]134 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]135 WREG32(PCIE_INDEX, reg);
136 (void)RREG32(PCIE_INDEX);
137 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]138 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]139 return r;
140}
141
142void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
143{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]144 unsigned long flags;
145
146 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]147 WREG32(PCIE_INDEX, reg);
148 (void)RREG32(PCIE_INDEX);
149 WREG32(PCIE_DATA, v);
150 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]151 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]152}
153
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]154static const u32 spectre_rlc_save_restore_register_list[] =
155{
156 (0x0e00 << 16) | (0xc12c >> 2),
157 0x00000000,
158 (0x0e00 << 16) | (0xc140 >> 2),
159 0x00000000,
160 (0x0e00 << 16) | (0xc150 >> 2),
161 0x00000000,
162 (0x0e00 << 16) | (0xc15c >> 2),
163 0x00000000,
164 (0x0e00 << 16) | (0xc168 >> 2),
165 0x00000000,
166 (0x0e00 << 16) | (0xc170 >> 2),
167 0x00000000,
168 (0x0e00 << 16) | (0xc178 >> 2),
169 0x00000000,
170 (0x0e00 << 16) | (0xc204 >> 2),
171 0x00000000,
172 (0x0e00 << 16) | (0xc2b4 >> 2),
173 0x00000000,
174 (0x0e00 << 16) | (0xc2b8 >> 2),
175 0x00000000,
176 (0x0e00 << 16) | (0xc2bc >> 2),
177 0x00000000,
178 (0x0e00 << 16) | (0xc2c0 >> 2),
179 0x00000000,
180 (0x0e00 << 16) | (0x8228 >> 2),
181 0x00000000,
182 (0x0e00 << 16) | (0x829c >> 2),
183 0x00000000,
184 (0x0e00 << 16) | (0x869c >> 2),
185 0x00000000,
186 (0x0600 << 16) | (0x98f4 >> 2),
187 0x00000000,
188 (0x0e00 << 16) | (0x98f8 >> 2),
189 0x00000000,
190 (0x0e00 << 16) | (0x9900 >> 2),
191 0x00000000,
192 (0x0e00 << 16) | (0xc260 >> 2),
193 0x00000000,
194 (0x0e00 << 16) | (0x90e8 >> 2),
195 0x00000000,
196 (0x0e00 << 16) | (0x3c000 >> 2),
197 0x00000000,
198 (0x0e00 << 16) | (0x3c00c >> 2),
199 0x00000000,
200 (0x0e00 << 16) | (0x8c1c >> 2),
201 0x00000000,
202 (0x0e00 << 16) | (0x9700 >> 2),
203 0x00000000,
204 (0x0e00 << 16) | (0xcd20 >> 2),
205 0x00000000,
206 (0x4e00 << 16) | (0xcd20 >> 2),
207 0x00000000,
208 (0x5e00 << 16) | (0xcd20 >> 2),
209 0x00000000,
210 (0x6e00 << 16) | (0xcd20 >> 2),
211 0x00000000,
212 (0x7e00 << 16) | (0xcd20 >> 2),
213 0x00000000,
214 (0x8e00 << 16) | (0xcd20 >> 2),
215 0x00000000,
216 (0x9e00 << 16) | (0xcd20 >> 2),
217 0x00000000,
218 (0xae00 << 16) | (0xcd20 >> 2),
219 0x00000000,
220 (0xbe00 << 16) | (0xcd20 >> 2),
221 0x00000000,
222 (0x0e00 << 16) | (0x89bc >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0x8900 >> 2),
225 0x00000000,
226 0x3,
227 (0x0e00 << 16) | (0xc130 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc134 >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc1fc >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0xc208 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0xc264 >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0xc268 >> 2),
238 0x00000000,
239 (0x0e00 << 16) | (0xc26c >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0xc270 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0xc274 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc278 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0xc27c >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0xc280 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0xc284 >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0xc288 >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0xc28c >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xc290 >> 2),
258 0x00000000,
259 (0x0e00 << 16) | (0xc294 >> 2),
260 0x00000000,
261 (0x0e00 << 16) | (0xc298 >> 2),
262 0x00000000,
263 (0x0e00 << 16) | (0xc29c >> 2),
264 0x00000000,
265 (0x0e00 << 16) | (0xc2a0 >> 2),
266 0x00000000,
267 (0x0e00 << 16) | (0xc2a4 >> 2),
268 0x00000000,
269 (0x0e00 << 16) | (0xc2a8 >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0xc2ac >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc2b0 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x301d0 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x30238 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0x30250 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0x30254 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0x30258 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0x3025c >> 2),
286 0x00000000,
287 (0x4e00 << 16) | (0xc900 >> 2),
288 0x00000000,
289 (0x5e00 << 16) | (0xc900 >> 2),
290 0x00000000,
291 (0x6e00 << 16) | (0xc900 >> 2),
292 0x00000000,
293 (0x7e00 << 16) | (0xc900 >> 2),
294 0x00000000,
295 (0x8e00 << 16) | (0xc900 >> 2),
296 0x00000000,
297 (0x9e00 << 16) | (0xc900 >> 2),
298 0x00000000,
299 (0xae00 << 16) | (0xc900 >> 2),
300 0x00000000,
301 (0xbe00 << 16) | (0xc900 >> 2),
302 0x00000000,
303 (0x4e00 << 16) | (0xc904 >> 2),
304 0x00000000,
305 (0x5e00 << 16) | (0xc904 >> 2),
306 0x00000000,
307 (0x6e00 << 16) | (0xc904 >> 2),
308 0x00000000,
309 (0x7e00 << 16) | (0xc904 >> 2),
310 0x00000000,
311 (0x8e00 << 16) | (0xc904 >> 2),
312 0x00000000,
313 (0x9e00 << 16) | (0xc904 >> 2),
314 0x00000000,
315 (0xae00 << 16) | (0xc904 >> 2),
316 0x00000000,
317 (0xbe00 << 16) | (0xc904 >> 2),
318 0x00000000,
319 (0x4e00 << 16) | (0xc908 >> 2),
320 0x00000000,
321 (0x5e00 << 16) | (0xc908 >> 2),
322 0x00000000,
323 (0x6e00 << 16) | (0xc908 >> 2),
324 0x00000000,
325 (0x7e00 << 16) | (0xc908 >> 2),
326 0x00000000,
327 (0x8e00 << 16) | (0xc908 >> 2),
328 0x00000000,
329 (0x9e00 << 16) | (0xc908 >> 2),
330 0x00000000,
331 (0xae00 << 16) | (0xc908 >> 2),
332 0x00000000,
333 (0xbe00 << 16) | (0xc908 >> 2),
334 0x00000000,
335 (0x4e00 << 16) | (0xc90c >> 2),
336 0x00000000,
337 (0x5e00 << 16) | (0xc90c >> 2),
338 0x00000000,
339 (0x6e00 << 16) | (0xc90c >> 2),
340 0x00000000,
341 (0x7e00 << 16) | (0xc90c >> 2),
342 0x00000000,
343 (0x8e00 << 16) | (0xc90c >> 2),
344 0x00000000,
345 (0x9e00 << 16) | (0xc90c >> 2),
346 0x00000000,
347 (0xae00 << 16) | (0xc90c >> 2),
348 0x00000000,
349 (0xbe00 << 16) | (0xc90c >> 2),
350 0x00000000,
351 (0x4e00 << 16) | (0xc910 >> 2),
352 0x00000000,
353 (0x5e00 << 16) | (0xc910 >> 2),
354 0x00000000,
355 (0x6e00 << 16) | (0xc910 >> 2),
356 0x00000000,
357 (0x7e00 << 16) | (0xc910 >> 2),
358 0x00000000,
359 (0x8e00 << 16) | (0xc910 >> 2),
360 0x00000000,
361 (0x9e00 << 16) | (0xc910 >> 2),
362 0x00000000,
363 (0xae00 << 16) | (0xc910 >> 2),
364 0x00000000,
365 (0xbe00 << 16) | (0xc910 >> 2),
366 0x00000000,
367 (0x0e00 << 16) | (0xc99c >> 2),
368 0x00000000,
369 (0x0e00 << 16) | (0x9834 >> 2),
370 0x00000000,
371 (0x0000 << 16) | (0x30f00 >> 2),
372 0x00000000,
373 (0x0001 << 16) | (0x30f00 >> 2),
374 0x00000000,
375 (0x0000 << 16) | (0x30f04 >> 2),
376 0x00000000,
377 (0x0001 << 16) | (0x30f04 >> 2),
378 0x00000000,
379 (0x0000 << 16) | (0x30f08 >> 2),
380 0x00000000,
381 (0x0001 << 16) | (0x30f08 >> 2),
382 0x00000000,
383 (0x0000 << 16) | (0x30f0c >> 2),
384 0x00000000,
385 (0x0001 << 16) | (0x30f0c >> 2),
386 0x00000000,
387 (0x0600 << 16) | (0x9b7c >> 2),
388 0x00000000,
389 (0x0e00 << 16) | (0x8a14 >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0x8a18 >> 2),
392 0x00000000,
393 (0x0600 << 16) | (0x30a00 >> 2),
394 0x00000000,
395 (0x0e00 << 16) | (0x8bf0 >> 2),
396 0x00000000,
397 (0x0e00 << 16) | (0x8bcc >> 2),
398 0x00000000,
399 (0x0e00 << 16) | (0x8b24 >> 2),
400 0x00000000,
401 (0x0e00 << 16) | (0x30a04 >> 2),
402 0x00000000,
403 (0x0600 << 16) | (0x30a10 >> 2),
404 0x00000000,
405 (0x0600 << 16) | (0x30a14 >> 2),
406 0x00000000,
407 (0x0600 << 16) | (0x30a18 >> 2),
408 0x00000000,
409 (0x0600 << 16) | (0x30a2c >> 2),
410 0x00000000,
411 (0x0e00 << 16) | (0xc700 >> 2),
412 0x00000000,
413 (0x0e00 << 16) | (0xc704 >> 2),
414 0x00000000,
415 (0x0e00 << 16) | (0xc708 >> 2),
416 0x00000000,
417 (0x0e00 << 16) | (0xc768 >> 2),
418 0x00000000,
419 (0x0400 << 16) | (0xc770 >> 2),
420 0x00000000,
421 (0x0400 << 16) | (0xc774 >> 2),
422 0x00000000,
423 (0x0400 << 16) | (0xc778 >> 2),
424 0x00000000,
425 (0x0400 << 16) | (0xc77c >> 2),
426 0x00000000,
427 (0x0400 << 16) | (0xc780 >> 2),
428 0x00000000,
429 (0x0400 << 16) | (0xc784 >> 2),
430 0x00000000,
431 (0x0400 << 16) | (0xc788 >> 2),
432 0x00000000,
433 (0x0400 << 16) | (0xc78c >> 2),
434 0x00000000,
435 (0x0400 << 16) | (0xc798 >> 2),
436 0x00000000,
437 (0x0400 << 16) | (0xc79c >> 2),
438 0x00000000,
439 (0x0400 << 16) | (0xc7a0 >> 2),
440 0x00000000,
441 (0x0400 << 16) | (0xc7a4 >> 2),
442 0x00000000,
443 (0x0400 << 16) | (0xc7a8 >> 2),
444 0x00000000,
445 (0x0400 << 16) | (0xc7ac >> 2),
446 0x00000000,
447 (0x0400 << 16) | (0xc7b0 >> 2),
448 0x00000000,
449 (0x0400 << 16) | (0xc7b4 >> 2),
450 0x00000000,
451 (0x0e00 << 16) | (0x9100 >> 2),
452 0x00000000,
453 (0x0e00 << 16) | (0x3c010 >> 2),
454 0x00000000,
455 (0x0e00 << 16) | (0x92a8 >> 2),
456 0x00000000,
457 (0x0e00 << 16) | (0x92ac >> 2),
458 0x00000000,
459 (0x0e00 << 16) | (0x92b4 >> 2),
460 0x00000000,
461 (0x0e00 << 16) | (0x92b8 >> 2),
462 0x00000000,
463 (0x0e00 << 16) | (0x92bc >> 2),
464 0x00000000,
465 (0x0e00 << 16) | (0x92c0 >> 2),
466 0x00000000,
467 (0x0e00 << 16) | (0x92c4 >> 2),
468 0x00000000,
469 (0x0e00 << 16) | (0x92c8 >> 2),
470 0x00000000,
471 (0x0e00 << 16) | (0x92cc >> 2),
472 0x00000000,
473 (0x0e00 << 16) | (0x92d0 >> 2),
474 0x00000000,
475 (0x0e00 << 16) | (0x8c00 >> 2),
476 0x00000000,
477 (0x0e00 << 16) | (0x8c04 >> 2),
478 0x00000000,
479 (0x0e00 << 16) | (0x8c20 >> 2),
480 0x00000000,
481 (0x0e00 << 16) | (0x8c38 >> 2),
482 0x00000000,
483 (0x0e00 << 16) | (0x8c3c >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0xae00 >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0x9604 >> 2),
488 0x00000000,
489 (0x0e00 << 16) | (0xac08 >> 2),
490 0x00000000,
491 (0x0e00 << 16) | (0xac0c >> 2),
492 0x00000000,
493 (0x0e00 << 16) | (0xac10 >> 2),
494 0x00000000,
495 (0x0e00 << 16) | (0xac14 >> 2),
496 0x00000000,
497 (0x0e00 << 16) | (0xac58 >> 2),
498 0x00000000,
499 (0x0e00 << 16) | (0xac68 >> 2),
500 0x00000000,
501 (0x0e00 << 16) | (0xac6c >> 2),
502 0x00000000,
503 (0x0e00 << 16) | (0xac70 >> 2),
504 0x00000000,
505 (0x0e00 << 16) | (0xac74 >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0xac78 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0xac7c >> 2),
510 0x00000000,
511 (0x0e00 << 16) | (0xac80 >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0xac84 >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0xac88 >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0xac8c >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0x970c >> 2),
520 0x00000000,
521 (0x0e00 << 16) | (0x9714 >> 2),
522 0x00000000,
523 (0x0e00 << 16) | (0x9718 >> 2),
524 0x00000000,
525 (0x0e00 << 16) | (0x971c >> 2),
526 0x00000000,
527 (0x0e00 << 16) | (0x31068 >> 2),
528 0x00000000,
529 (0x4e00 << 16) | (0x31068 >> 2),
530 0x00000000,
531 (0x5e00 << 16) | (0x31068 >> 2),
532 0x00000000,
533 (0x6e00 << 16) | (0x31068 >> 2),
534 0x00000000,
535 (0x7e00 << 16) | (0x31068 >> 2),
536 0x00000000,
537 (0x8e00 << 16) | (0x31068 >> 2),
538 0x00000000,
539 (0x9e00 << 16) | (0x31068 >> 2),
540 0x00000000,
541 (0xae00 << 16) | (0x31068 >> 2),
542 0x00000000,
543 (0xbe00 << 16) | (0x31068 >> 2),
544 0x00000000,
545 (0x0e00 << 16) | (0xcd10 >> 2),
546 0x00000000,
547 (0x0e00 << 16) | (0xcd14 >> 2),
548 0x00000000,
549 (0x0e00 << 16) | (0x88b0 >> 2),
550 0x00000000,
551 (0x0e00 << 16) | (0x88b4 >> 2),
552 0x00000000,
553 (0x0e00 << 16) | (0x88b8 >> 2),
554 0x00000000,
555 (0x0e00 << 16) | (0x88bc >> 2),
556 0x00000000,
557 (0x0400 << 16) | (0x89c0 >> 2),
558 0x00000000,
559 (0x0e00 << 16) | (0x88c4 >> 2),
560 0x00000000,
561 (0x0e00 << 16) | (0x88c8 >> 2),
562 0x00000000,
563 (0x0e00 << 16) | (0x88d0 >> 2),
564 0x00000000,
565 (0x0e00 << 16) | (0x88d4 >> 2),
566 0x00000000,
567 (0x0e00 << 16) | (0x88d8 >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x8980 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x30938 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x3093c >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x30940 >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x89a0 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x30900 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x30904 >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x89b4 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x3c210 >> 2),
586 0x00000000,
587 (0x0e00 << 16) | (0x3c214 >> 2),
588 0x00000000,
589 (0x0e00 << 16) | (0x3c218 >> 2),
590 0x00000000,
591 (0x0e00 << 16) | (0x8904 >> 2),
592 0x00000000,
593 0x5,
594 (0x0e00 << 16) | (0x8c28 >> 2),
595 (0x0e00 << 16) | (0x8c2c >> 2),
596 (0x0e00 << 16) | (0x8c30 >> 2),
597 (0x0e00 << 16) | (0x8c34 >> 2),
598 (0x0e00 << 16) | (0x9600 >> 2),
599};
600
601static const u32 kalindi_rlc_save_restore_register_list[] =
602{
603 (0x0e00 << 16) | (0xc12c >> 2),
604 0x00000000,
605 (0x0e00 << 16) | (0xc140 >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xc150 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xc15c >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xc168 >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xc170 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xc204 >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xc2b4 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0xc2b8 >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0xc2bc >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0xc2c0 >> 2),
624 0x00000000,
625 (0x0e00 << 16) | (0x8228 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0x829c >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0x869c >> 2),
630 0x00000000,
631 (0x0600 << 16) | (0x98f4 >> 2),
632 0x00000000,
633 (0x0e00 << 16) | (0x98f8 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0x9900 >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0xc260 >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0x90e8 >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x3c000 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0x3c00c >> 2),
644 0x00000000,
645 (0x0e00 << 16) | (0x8c1c >> 2),
646 0x00000000,
647 (0x0e00 << 16) | (0x9700 >> 2),
648 0x00000000,
649 (0x0e00 << 16) | (0xcd20 >> 2),
650 0x00000000,
651 (0x4e00 << 16) | (0xcd20 >> 2),
652 0x00000000,
653 (0x5e00 << 16) | (0xcd20 >> 2),
654 0x00000000,
655 (0x6e00 << 16) | (0xcd20 >> 2),
656 0x00000000,
657 (0x7e00 << 16) | (0xcd20 >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0x89bc >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0x8900 >> 2),
662 0x00000000,
663 0x3,
664 (0x0e00 << 16) | (0xc130 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc134 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc1fc >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc208 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc264 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc268 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc26c >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0xc270 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0xc274 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0xc28c >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0xc290 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0xc294 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0xc298 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc2a0 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0xc2a4 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0xc2a8 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0xc2ac >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x301d0 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x30238 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x30250 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x30254 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x30258 >> 2),
707 0x00000000,
708 (0x0e00 << 16) | (0x3025c >> 2),
709 0x00000000,
710 (0x4e00 << 16) | (0xc900 >> 2),
711 0x00000000,
712 (0x5e00 << 16) | (0xc900 >> 2),
713 0x00000000,
714 (0x6e00 << 16) | (0xc900 >> 2),
715 0x00000000,
716 (0x7e00 << 16) | (0xc900 >> 2),
717 0x00000000,
718 (0x4e00 << 16) | (0xc904 >> 2),
719 0x00000000,
720 (0x5e00 << 16) | (0xc904 >> 2),
721 0x00000000,
722 (0x6e00 << 16) | (0xc904 >> 2),
723 0x00000000,
724 (0x7e00 << 16) | (0xc904 >> 2),
725 0x00000000,
726 (0x4e00 << 16) | (0xc908 >> 2),
727 0x00000000,
728 (0x5e00 << 16) | (0xc908 >> 2),
729 0x00000000,
730 (0x6e00 << 16) | (0xc908 >> 2),
731 0x00000000,
732 (0x7e00 << 16) | (0xc908 >> 2),
733 0x00000000,
734 (0x4e00 << 16) | (0xc90c >> 2),
735 0x00000000,
736 (0x5e00 << 16) | (0xc90c >> 2),
737 0x00000000,
738 (0x6e00 << 16) | (0xc90c >> 2),
739 0x00000000,
740 (0x7e00 << 16) | (0xc90c >> 2),
741 0x00000000,
742 (0x4e00 << 16) | (0xc910 >> 2),
743 0x00000000,
744 (0x5e00 << 16) | (0xc910 >> 2),
745 0x00000000,
746 (0x6e00 << 16) | (0xc910 >> 2),
747 0x00000000,
748 (0x7e00 << 16) | (0xc910 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0xc99c >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x9834 >> 2),
753 0x00000000,
754 (0x0000 << 16) | (0x30f00 >> 2),
755 0x00000000,
756 (0x0000 << 16) | (0x30f04 >> 2),
757 0x00000000,
758 (0x0000 << 16) | (0x30f08 >> 2),
759 0x00000000,
760 (0x0000 << 16) | (0x30f0c >> 2),
761 0x00000000,
762 (0x0600 << 16) | (0x9b7c >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0x8a14 >> 2),
765 0x00000000,
766 (0x0e00 << 16) | (0x8a18 >> 2),
767 0x00000000,
768 (0x0600 << 16) | (0x30a00 >> 2),
769 0x00000000,
770 (0x0e00 << 16) | (0x8bf0 >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0x8bcc >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x8b24 >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x30a04 >> 2),
777 0x00000000,
778 (0x0600 << 16) | (0x30a10 >> 2),
779 0x00000000,
780 (0x0600 << 16) | (0x30a14 >> 2),
781 0x00000000,
782 (0x0600 << 16) | (0x30a18 >> 2),
783 0x00000000,
784 (0x0600 << 16) | (0x30a2c >> 2),
785 0x00000000,
786 (0x0e00 << 16) | (0xc700 >> 2),
787 0x00000000,
788 (0x0e00 << 16) | (0xc704 >> 2),
789 0x00000000,
790 (0x0e00 << 16) | (0xc708 >> 2),
791 0x00000000,
792 (0x0e00 << 16) | (0xc768 >> 2),
793 0x00000000,
794 (0x0400 << 16) | (0xc770 >> 2),
795 0x00000000,
796 (0x0400 << 16) | (0xc774 >> 2),
797 0x00000000,
798 (0x0400 << 16) | (0xc798 >> 2),
799 0x00000000,
800 (0x0400 << 16) | (0xc79c >> 2),
801 0x00000000,
802 (0x0e00 << 16) | (0x9100 >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0x3c010 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0x8c00 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0x8c04 >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0x8c20 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0x8c38 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0x8c3c >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0xae00 >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0x9604 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0xac08 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0xac0c >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0xac10 >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0xac14 >> 2),
827 0x00000000,
828 (0x0e00 << 16) | (0xac58 >> 2),
829 0x00000000,
830 (0x0e00 << 16) | (0xac68 >> 2),
831 0x00000000,
832 (0x0e00 << 16) | (0xac6c >> 2),
833 0x00000000,
834 (0x0e00 << 16) | (0xac70 >> 2),
835 0x00000000,
836 (0x0e00 << 16) | (0xac74 >> 2),
837 0x00000000,
838 (0x0e00 << 16) | (0xac78 >> 2),
839 0x00000000,
840 (0x0e00 << 16) | (0xac7c >> 2),
841 0x00000000,
842 (0x0e00 << 16) | (0xac80 >> 2),
843 0x00000000,
844 (0x0e00 << 16) | (0xac84 >> 2),
845 0x00000000,
846 (0x0e00 << 16) | (0xac88 >> 2),
847 0x00000000,
848 (0x0e00 << 16) | (0xac8c >> 2),
849 0x00000000,
850 (0x0e00 << 16) | (0x970c >> 2),
851 0x00000000,
852 (0x0e00 << 16) | (0x9714 >> 2),
853 0x00000000,
854 (0x0e00 << 16) | (0x9718 >> 2),
855 0x00000000,
856 (0x0e00 << 16) | (0x971c >> 2),
857 0x00000000,
858 (0x0e00 << 16) | (0x31068 >> 2),
859 0x00000000,
860 (0x4e00 << 16) | (0x31068 >> 2),
861 0x00000000,
862 (0x5e00 << 16) | (0x31068 >> 2),
863 0x00000000,
864 (0x6e00 << 16) | (0x31068 >> 2),
865 0x00000000,
866 (0x7e00 << 16) | (0x31068 >> 2),
867 0x00000000,
868 (0x0e00 << 16) | (0xcd10 >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0xcd14 >> 2),
871 0x00000000,
872 (0x0e00 << 16) | (0x88b0 >> 2),
873 0x00000000,
874 (0x0e00 << 16) | (0x88b4 >> 2),
875 0x00000000,
876 (0x0e00 << 16) | (0x88b8 >> 2),
877 0x00000000,
878 (0x0e00 << 16) | (0x88bc >> 2),
879 0x00000000,
880 (0x0400 << 16) | (0x89c0 >> 2),
881 0x00000000,
882 (0x0e00 << 16) | (0x88c4 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x88c8 >> 2),
885 0x00000000,
886 (0x0e00 << 16) | (0x88d0 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x88d4 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x88d8 >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x8980 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x30938 >> 2),
895 0x00000000,
896 (0x0e00 << 16) | (0x3093c >> 2),
897 0x00000000,
898 (0x0e00 << 16) | (0x30940 >> 2),
899 0x00000000,
900 (0x0e00 << 16) | (0x89a0 >> 2),
901 0x00000000,
902 (0x0e00 << 16) | (0x30900 >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0x30904 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0x89b4 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0x3e1fc >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0x3c210 >> 2),
911 0x00000000,
912 (0x0e00 << 16) | (0x3c214 >> 2),
913 0x00000000,
914 (0x0e00 << 16) | (0x3c218 >> 2),
915 0x00000000,
916 (0x0e00 << 16) | (0x8904 >> 2),
917 0x00000000,
918 0x5,
919 (0x0e00 << 16) | (0x8c28 >> 2),
920 (0x0e00 << 16) | (0x8c2c >> 2),
921 (0x0e00 << 16) | (0x8c30 >> 2),
922 (0x0e00 << 16) | (0x8c34 >> 2),
923 (0x0e00 << 16) | (0x9600 >> 2),
924};
925
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]926static const u32 bonaire_golden_spm_registers[] =
927{
928 0x30800, 0xe0ffffff, 0xe0000000
929};
930
931static const u32 bonaire_golden_common_registers[] =
932{
933 0xc770, 0xffffffff, 0x00000800,
934 0xc774, 0xffffffff, 0x00000800,
935 0xc798, 0xffffffff, 0x00007fbf,
936 0xc79c, 0xffffffff, 0x00007faf
937};
938
939static const u32 bonaire_golden_registers[] =
940{
941 0x3354, 0x00000333, 0x00000333,
942 0x3350, 0x000c0fc0, 0x00040200,
943 0x9a10, 0x00010000, 0x00058208,
944 0x3c000, 0xffff1fff, 0x00140000,
945 0x3c200, 0xfdfc0fff, 0x00000100,
946 0x3c234, 0x40000000, 0x40000200,
947 0x9830, 0xffffffff, 0x00000000,
948 0x9834, 0xf00fffff, 0x00000400,
949 0x9838, 0x0002021c, 0x00020200,
950 0xc78, 0x00000080, 0x00000000,
951 0x5bb0, 0x000000f0, 0x00000070,
952 0x5bc0, 0xf0311fff, 0x80300000,
953 0x98f8, 0x73773777, 0x12010001,
954 0x350c, 0x00810000, 0x408af000,
955 0x7030, 0x31000111, 0x00000011,
956 0x2f48, 0x73773777, 0x12010001,
957 0x220c, 0x00007fb6, 0x0021a1b1,
958 0x2210, 0x00007fb6, 0x002021b1,
959 0x2180, 0x00007fb6, 0x00002191,
960 0x2218, 0x00007fb6, 0x002121b1,
961 0x221c, 0x00007fb6, 0x002021b1,
962 0x21dc, 0x00007fb6, 0x00002191,
963 0x21e0, 0x00007fb6, 0x00002191,
964 0x3628, 0x0000003f, 0x0000000a,
965 0x362c, 0x0000003f, 0x0000000a,
966 0x2ae4, 0x00073ffe, 0x000022a2,
967 0x240c, 0x000007ff, 0x00000000,
968 0x8a14, 0xf000003f, 0x00000007,
969 0x8bf0, 0x00002001, 0x00000001,
970 0x8b24, 0xffffffff, 0x00ffffff,
971 0x30a04, 0x0000ff0f, 0x00000000,
972 0x28a4c, 0x07ffffff, 0x06000000,
973 0x4d8, 0x00000fff, 0x00000100,
974 0x3e78, 0x00000001, 0x00000002,
975 0x9100, 0x03000000, 0x0362c688,
976 0x8c00, 0x000000ff, 0x00000001,
977 0xe40, 0x00001fff, 0x00001fff,
978 0x9060, 0x0000007f, 0x00000020,
979 0x9508, 0x00010000, 0x00010000,
980 0xac14, 0x000003ff, 0x000000f3,
981 0xac0c, 0xffffffff, 0x00001032
982};
983
984static const u32 bonaire_mgcg_cgcg_init[] =
985{
986 0xc420, 0xffffffff, 0xfffffffc,
987 0x30800, 0xffffffff, 0xe0000000,
988 0x3c2a0, 0xffffffff, 0x00000100,
989 0x3c208, 0xffffffff, 0x00000100,
990 0x3c2c0, 0xffffffff, 0xc0000100,
991 0x3c2c8, 0xffffffff, 0xc0000100,
992 0x3c2c4, 0xffffffff, 0xc0000100,
993 0x55e4, 0xffffffff, 0x00600100,
994 0x3c280, 0xffffffff, 0x00000100,
995 0x3c214, 0xffffffff, 0x06000100,
996 0x3c220, 0xffffffff, 0x00000100,
997 0x3c218, 0xffffffff, 0x06000100,
998 0x3c204, 0xffffffff, 0x00000100,
999 0x3c2e0, 0xffffffff, 0x00000100,
1000 0x3c224, 0xffffffff, 0x00000100,
1001 0x3c200, 0xffffffff, 0x00000100,
1002 0x3c230, 0xffffffff, 0x00000100,
1003 0x3c234, 0xffffffff, 0x00000100,
1004 0x3c250, 0xffffffff, 0x00000100,
1005 0x3c254, 0xffffffff, 0x00000100,
1006 0x3c258, 0xffffffff, 0x00000100,
1007 0x3c25c, 0xffffffff, 0x00000100,
1008 0x3c260, 0xffffffff, 0x00000100,
1009 0x3c27c, 0xffffffff, 0x00000100,
1010 0x3c278, 0xffffffff, 0x00000100,
1011 0x3c210, 0xffffffff, 0x06000100,
1012 0x3c290, 0xffffffff, 0x00000100,
1013 0x3c274, 0xffffffff, 0x00000100,
1014 0x3c2b4, 0xffffffff, 0x00000100,
1015 0x3c2b0, 0xffffffff, 0x00000100,
1016 0x3c270, 0xffffffff, 0x00000100,
1017 0x30800, 0xffffffff, 0xe0000000,
1018 0x3c020, 0xffffffff, 0x00010000,
1019 0x3c024, 0xffffffff, 0x00030002,
1020 0x3c028, 0xffffffff, 0x00040007,
1021 0x3c02c, 0xffffffff, 0x00060005,
1022 0x3c030, 0xffffffff, 0x00090008,
1023 0x3c034, 0xffffffff, 0x00010000,
1024 0x3c038, 0xffffffff, 0x00030002,
1025 0x3c03c, 0xffffffff, 0x00040007,
1026 0x3c040, 0xffffffff, 0x00060005,
1027 0x3c044, 0xffffffff, 0x00090008,
1028 0x3c048, 0xffffffff, 0x00010000,
1029 0x3c04c, 0xffffffff, 0x00030002,
1030 0x3c050, 0xffffffff, 0x00040007,
1031 0x3c054, 0xffffffff, 0x00060005,
1032 0x3c058, 0xffffffff, 0x00090008,
1033 0x3c05c, 0xffffffff, 0x00010000,
1034 0x3c060, 0xffffffff, 0x00030002,
1035 0x3c064, 0xffffffff, 0x00040007,
1036 0x3c068, 0xffffffff, 0x00060005,
1037 0x3c06c, 0xffffffff, 0x00090008,
1038 0x3c070, 0xffffffff, 0x00010000,
1039 0x3c074, 0xffffffff, 0x00030002,
1040 0x3c078, 0xffffffff, 0x00040007,
1041 0x3c07c, 0xffffffff, 0x00060005,
1042 0x3c080, 0xffffffff, 0x00090008,
1043 0x3c084, 0xffffffff, 0x00010000,
1044 0x3c088, 0xffffffff, 0x00030002,
1045 0x3c08c, 0xffffffff, 0x00040007,
1046 0x3c090, 0xffffffff, 0x00060005,
1047 0x3c094, 0xffffffff, 0x00090008,
1048 0x3c098, 0xffffffff, 0x00010000,
1049 0x3c09c, 0xffffffff, 0x00030002,
1050 0x3c0a0, 0xffffffff, 0x00040007,
1051 0x3c0a4, 0xffffffff, 0x00060005,
1052 0x3c0a8, 0xffffffff, 0x00090008,
1053 0x3c000, 0xffffffff, 0x96e00200,
1054 0x8708, 0xffffffff, 0x00900100,
1055 0xc424, 0xffffffff, 0x0020003f,
1056 0x38, 0xffffffff, 0x0140001c,
1057 0x3c, 0x000f0000, 0x000f0000,
1058 0x220, 0xffffffff, 0xC060000C,
1059 0x224, 0xc0000fff, 0x00000100,
1060 0xf90, 0xffffffff, 0x00000100,
1061 0xf98, 0x00000101, 0x00000000,
1062 0x20a8, 0xffffffff, 0x00000104,
1063 0x55e4, 0xff000fff, 0x00000100,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd00c, 0xff000ff0, 0x00000100,
1067 0xd80c, 0xff000ff0, 0x00000100
1068};
1069
1070static const u32 spectre_golden_spm_registers[] =
1071{
1072 0x30800, 0xe0ffffff, 0xe0000000
1073};
1074
1075static const u32 spectre_golden_common_registers[] =
1076{
1077 0xc770, 0xffffffff, 0x00000800,
1078 0xc774, 0xffffffff, 0x00000800,
1079 0xc798, 0xffffffff, 0x00007fbf,
1080 0xc79c, 0xffffffff, 0x00007faf
1081};
1082
1083static const u32 spectre_golden_registers[] =
1084{
1085 0x3c000, 0xffff1fff, 0x96940200,
1086 0x3c00c, 0xffff0001, 0xff000000,
1087 0x3c200, 0xfffc0fff, 0x00000100,
1088 0x6ed8, 0x00010101, 0x00010000,
1089 0x9834, 0xf00fffff, 0x00000400,
1090 0x9838, 0xfffffffc, 0x00020200,
1091 0x5bb0, 0x000000f0, 0x00000070,
1092 0x5bc0, 0xf0311fff, 0x80300000,
1093 0x98f8, 0x73773777, 0x12010001,
1094 0x9b7c, 0x00ff0000, 0x00fc0000,
1095 0x2f48, 0x73773777, 0x12010001,
1096 0x8a14, 0xf000003f, 0x00000007,
1097 0x8b24, 0xffffffff, 0x00ffffff,
1098 0x28350, 0x3f3f3fff, 0x00000082,
1099 0x28355, 0x0000003f, 0x00000000,
1100 0x3e78, 0x00000001, 0x00000002,
1101 0x913c, 0xffff03df, 0x00000004,
1102 0xc768, 0x00000008, 0x00000008,
1103 0x8c00, 0x000008ff, 0x00000800,
1104 0x9508, 0x00010000, 0x00010000,
1105 0xac0c, 0xffffffff, 0x54763210,
1106 0x214f8, 0x01ff01ff, 0x00000002,
1107 0x21498, 0x007ff800, 0x00200000,
1108 0x2015c, 0xffffffff, 0x00000f40,
1109 0x30934, 0xffffffff, 0x00000001
1110};
1111
1112static const u32 spectre_mgcg_cgcg_init[] =
1113{
1114 0xc420, 0xffffffff, 0xfffffffc,
1115 0x30800, 0xffffffff, 0xe0000000,
1116 0x3c2a0, 0xffffffff, 0x00000100,
1117 0x3c208, 0xffffffff, 0x00000100,
1118 0x3c2c0, 0xffffffff, 0x00000100,
1119 0x3c2c8, 0xffffffff, 0x00000100,
1120 0x3c2c4, 0xffffffff, 0x00000100,
1121 0x55e4, 0xffffffff, 0x00600100,
1122 0x3c280, 0xffffffff, 0x00000100,
1123 0x3c214, 0xffffffff, 0x06000100,
1124 0x3c220, 0xffffffff, 0x00000100,
1125 0x3c218, 0xffffffff, 0x06000100,
1126 0x3c204, 0xffffffff, 0x00000100,
1127 0x3c2e0, 0xffffffff, 0x00000100,
1128 0x3c224, 0xffffffff, 0x00000100,
1129 0x3c200, 0xffffffff, 0x00000100,
1130 0x3c230, 0xffffffff, 0x00000100,
1131 0x3c234, 0xffffffff, 0x00000100,
1132 0x3c250, 0xffffffff, 0x00000100,
1133 0x3c254, 0xffffffff, 0x00000100,
1134 0x3c258, 0xffffffff, 0x00000100,
1135 0x3c25c, 0xffffffff, 0x00000100,
1136 0x3c260, 0xffffffff, 0x00000100,
1137 0x3c27c, 0xffffffff, 0x00000100,
1138 0x3c278, 0xffffffff, 0x00000100,
1139 0x3c210, 0xffffffff, 0x06000100,
1140 0x3c290, 0xffffffff, 0x00000100,
1141 0x3c274, 0xffffffff, 0x00000100,
1142 0x3c2b4, 0xffffffff, 0x00000100,
1143 0x3c2b0, 0xffffffff, 0x00000100,
1144 0x3c270, 0xffffffff, 0x00000100,
1145 0x30800, 0xffffffff, 0xe0000000,
1146 0x3c020, 0xffffffff, 0x00010000,
1147 0x3c024, 0xffffffff, 0x00030002,
1148 0x3c028, 0xffffffff, 0x00040007,
1149 0x3c02c, 0xffffffff, 0x00060005,
1150 0x3c030, 0xffffffff, 0x00090008,
1151 0x3c034, 0xffffffff, 0x00010000,
1152 0x3c038, 0xffffffff, 0x00030002,
1153 0x3c03c, 0xffffffff, 0x00040007,
1154 0x3c040, 0xffffffff, 0x00060005,
1155 0x3c044, 0xffffffff, 0x00090008,
1156 0x3c048, 0xffffffff, 0x00010000,
1157 0x3c04c, 0xffffffff, 0x00030002,
1158 0x3c050, 0xffffffff, 0x00040007,
1159 0x3c054, 0xffffffff, 0x00060005,
1160 0x3c058, 0xffffffff, 0x00090008,
1161 0x3c05c, 0xffffffff, 0x00010000,
1162 0x3c060, 0xffffffff, 0x00030002,
1163 0x3c064, 0xffffffff, 0x00040007,
1164 0x3c068, 0xffffffff, 0x00060005,
1165 0x3c06c, 0xffffffff, 0x00090008,
1166 0x3c070, 0xffffffff, 0x00010000,
1167 0x3c074, 0xffffffff, 0x00030002,
1168 0x3c078, 0xffffffff, 0x00040007,
1169 0x3c07c, 0xffffffff, 0x00060005,
1170 0x3c080, 0xffffffff, 0x00090008,
1171 0x3c084, 0xffffffff, 0x00010000,
1172 0x3c088, 0xffffffff, 0x00030002,
1173 0x3c08c, 0xffffffff, 0x00040007,
1174 0x3c090, 0xffffffff, 0x00060005,
1175 0x3c094, 0xffffffff, 0x00090008,
1176 0x3c098, 0xffffffff, 0x00010000,
1177 0x3c09c, 0xffffffff, 0x00030002,
1178 0x3c0a0, 0xffffffff, 0x00040007,
1179 0x3c0a4, 0xffffffff, 0x00060005,
1180 0x3c0a8, 0xffffffff, 0x00090008,
1181 0x3c0ac, 0xffffffff, 0x00010000,
1182 0x3c0b0, 0xffffffff, 0x00030002,
1183 0x3c0b4, 0xffffffff, 0x00040007,
1184 0x3c0b8, 0xffffffff, 0x00060005,
1185 0x3c0bc, 0xffffffff, 0x00090008,
1186 0x3c000, 0xffffffff, 0x96e00200,
1187 0x8708, 0xffffffff, 0x00900100,
1188 0xc424, 0xffffffff, 0x0020003f,
1189 0x38, 0xffffffff, 0x0140001c,
1190 0x3c, 0x000f0000, 0x000f0000,
1191 0x220, 0xffffffff, 0xC060000C,
1192 0x224, 0xc0000fff, 0x00000100,
1193 0xf90, 0xffffffff, 0x00000100,
1194 0xf98, 0x00000101, 0x00000000,
1195 0x20a8, 0xffffffff, 0x00000104,
1196 0x55e4, 0xff000fff, 0x00000100,
1197 0x30cc, 0xc0000fff, 0x00000104,
1198 0xc1e4, 0x00000001, 0x00000001,
1199 0xd00c, 0xff000ff0, 0x00000100,
1200 0xd80c, 0xff000ff0, 0x00000100
1201};
1202
1203static const u32 kalindi_golden_spm_registers[] =
1204{
1205 0x30800, 0xe0ffffff, 0xe0000000
1206};
1207
1208static const u32 kalindi_golden_common_registers[] =
1209{
1210 0xc770, 0xffffffff, 0x00000800,
1211 0xc774, 0xffffffff, 0x00000800,
1212 0xc798, 0xffffffff, 0x00007fbf,
1213 0xc79c, 0xffffffff, 0x00007faf
1214};
1215
1216static const u32 kalindi_golden_registers[] =
1217{
1218 0x3c000, 0xffffdfff, 0x6e944040,
1219 0x55e4, 0xff607fff, 0xfc000100,
1220 0x3c220, 0xff000fff, 0x00000100,
1221 0x3c224, 0xff000fff, 0x00000100,
1222 0x3c200, 0xfffc0fff, 0x00000100,
1223 0x6ed8, 0x00010101, 0x00010000,
1224 0x9830, 0xffffffff, 0x00000000,
1225 0x9834, 0xf00fffff, 0x00000400,
1226 0x5bb0, 0x000000f0, 0x00000070,
1227 0x5bc0, 0xf0311fff, 0x80300000,
1228 0x98f8, 0x73773777, 0x12010001,
1229 0x98fc, 0xffffffff, 0x00000010,
1230 0x9b7c, 0x00ff0000, 0x00fc0000,
1231 0x8030, 0x00001f0f, 0x0000100a,
1232 0x2f48, 0x73773777, 0x12010001,
1233 0x2408, 0x000fffff, 0x000c007f,
1234 0x8a14, 0xf000003f, 0x00000007,
1235 0x8b24, 0x3fff3fff, 0x00ffcfff,
1236 0x30a04, 0x0000ff0f, 0x00000000,
1237 0x28a4c, 0x07ffffff, 0x06000000,
1238 0x4d8, 0x00000fff, 0x00000100,
1239 0x3e78, 0x00000001, 0x00000002,
1240 0xc768, 0x00000008, 0x00000008,
1241 0x8c00, 0x000000ff, 0x00000003,
1242 0x214f8, 0x01ff01ff, 0x00000002,
1243 0x21498, 0x007ff800, 0x00200000,
1244 0x2015c, 0xffffffff, 0x00000f40,
1245 0x88c4, 0x001f3ae3, 0x00000082,
1246 0x88d4, 0x0000001f, 0x00000010,
1247 0x30934, 0xffffffff, 0x00000000
1248};
1249
1250static const u32 kalindi_mgcg_cgcg_init[] =
1251{
1252 0xc420, 0xffffffff, 0xfffffffc,
1253 0x30800, 0xffffffff, 0xe0000000,
1254 0x3c2a0, 0xffffffff, 0x00000100,
1255 0x3c208, 0xffffffff, 0x00000100,
1256 0x3c2c0, 0xffffffff, 0x00000100,
1257 0x3c2c8, 0xffffffff, 0x00000100,
1258 0x3c2c4, 0xffffffff, 0x00000100,
1259 0x55e4, 0xffffffff, 0x00600100,
1260 0x3c280, 0xffffffff, 0x00000100,
1261 0x3c214, 0xffffffff, 0x06000100,
1262 0x3c220, 0xffffffff, 0x00000100,
1263 0x3c218, 0xffffffff, 0x06000100,
1264 0x3c204, 0xffffffff, 0x00000100,
1265 0x3c2e0, 0xffffffff, 0x00000100,
1266 0x3c224, 0xffffffff, 0x00000100,
1267 0x3c200, 0xffffffff, 0x00000100,
1268 0x3c230, 0xffffffff, 0x00000100,
1269 0x3c234, 0xffffffff, 0x00000100,
1270 0x3c250, 0xffffffff, 0x00000100,
1271 0x3c254, 0xffffffff, 0x00000100,
1272 0x3c258, 0xffffffff, 0x00000100,
1273 0x3c25c, 0xffffffff, 0x00000100,
1274 0x3c260, 0xffffffff, 0x00000100,
1275 0x3c27c, 0xffffffff, 0x00000100,
1276 0x3c278, 0xffffffff, 0x00000100,
1277 0x3c210, 0xffffffff, 0x06000100,
1278 0x3c290, 0xffffffff, 0x00000100,
1279 0x3c274, 0xffffffff, 0x00000100,
1280 0x3c2b4, 0xffffffff, 0x00000100,
1281 0x3c2b0, 0xffffffff, 0x00000100,
1282 0x3c270, 0xffffffff, 0x00000100,
1283 0x30800, 0xffffffff, 0xe0000000,
1284 0x3c020, 0xffffffff, 0x00010000,
1285 0x3c024, 0xffffffff, 0x00030002,
1286 0x3c028, 0xffffffff, 0x00040007,
1287 0x3c02c, 0xffffffff, 0x00060005,
1288 0x3c030, 0xffffffff, 0x00090008,
1289 0x3c034, 0xffffffff, 0x00010000,
1290 0x3c038, 0xffffffff, 0x00030002,
1291 0x3c03c, 0xffffffff, 0x00040007,
1292 0x3c040, 0xffffffff, 0x00060005,
1293 0x3c044, 0xffffffff, 0x00090008,
1294 0x3c000, 0xffffffff, 0x96e00200,
1295 0x8708, 0xffffffff, 0x00900100,
1296 0xc424, 0xffffffff, 0x0020003f,
1297 0x38, 0xffffffff, 0x0140001c,
1298 0x3c, 0x000f0000, 0x000f0000,
1299 0x220, 0xffffffff, 0xC060000C,
1300 0x224, 0xc0000fff, 0x00000100,
1301 0x20a8, 0xffffffff, 0x00000104,
1302 0x55e4, 0xff000fff, 0x00000100,
1303 0x30cc, 0xc0000fff, 0x00000104,
1304 0xc1e4, 0x00000001, 0x00000001,
1305 0xd00c, 0xff000ff0, 0x00000100,
1306 0xd80c, 0xff000ff0, 0x00000100
1307};
1308
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1309static const u32 hawaii_golden_spm_registers[] =
1310{
1311 0x30800, 0xe0ffffff, 0xe0000000
1312};
1313
1314static const u32 hawaii_golden_common_registers[] =
1315{
1316 0x30800, 0xffffffff, 0xe0000000,
1317 0x28350, 0xffffffff, 0x3a00161a,
1318 0x28354, 0xffffffff, 0x0000002e,
1319 0x9a10, 0xffffffff, 0x00018208,
1320 0x98f8, 0xffffffff, 0x12011003
1321};
1322
1323static const u32 hawaii_golden_registers[] =
1324{
1325 0x3354, 0x00000333, 0x00000333,
1326 0x9a10, 0x00010000, 0x00058208,
1327 0x9830, 0xffffffff, 0x00000000,
1328 0x9834, 0xf00fffff, 0x00000400,
1329 0x9838, 0x0002021c, 0x00020200,
1330 0xc78, 0x00000080, 0x00000000,
1331 0x5bb0, 0x000000f0, 0x00000070,
1332 0x5bc0, 0xf0311fff, 0x80300000,
1333 0x350c, 0x00810000, 0x408af000,
1334 0x7030, 0x31000111, 0x00000011,
1335 0x2f48, 0x73773777, 0x12010001,
1336 0x2120, 0x0000007f, 0x0000001b,
1337 0x21dc, 0x00007fb6, 0x00002191,
1338 0x3628, 0x0000003f, 0x0000000a,
1339 0x362c, 0x0000003f, 0x0000000a,
1340 0x2ae4, 0x00073ffe, 0x000022a2,
1341 0x240c, 0x000007ff, 0x00000000,
1342 0x8bf0, 0x00002001, 0x00000001,
1343 0x8b24, 0xffffffff, 0x00ffffff,
1344 0x30a04, 0x0000ff0f, 0x00000000,
1345 0x28a4c, 0x07ffffff, 0x06000000,
1346 0x3e78, 0x00000001, 0x00000002,
1347 0xc768, 0x00000008, 0x00000008,
1348 0xc770, 0x00000f00, 0x00000800,
1349 0xc774, 0x00000f00, 0x00000800,
1350 0xc798, 0x00ffffff, 0x00ff7fbf,
1351 0xc79c, 0x00ffffff, 0x00ff7faf,
1352 0x8c00, 0x000000ff, 0x00000800,
1353 0xe40, 0x00001fff, 0x00001fff,
1354 0x9060, 0x0000007f, 0x00000020,
1355 0x9508, 0x00010000, 0x00010000,
1356 0xae00, 0x00100000, 0x000ff07c,
1357 0xac14, 0x000003ff, 0x0000000f,
1358 0xac10, 0xffffffff, 0x7564fdec,
1359 0xac0c, 0xffffffff, 0x3120b9a8,
1360 0xac08, 0x20000000, 0x0f9c0000
1361};
1362
1363static const u32 hawaii_mgcg_cgcg_init[] =
1364{
1365 0xc420, 0xffffffff, 0xfffffffd,
1366 0x30800, 0xffffffff, 0xe0000000,
1367 0x3c2a0, 0xffffffff, 0x00000100,
1368 0x3c208, 0xffffffff, 0x00000100,
1369 0x3c2c0, 0xffffffff, 0x00000100,
1370 0x3c2c8, 0xffffffff, 0x00000100,
1371 0x3c2c4, 0xffffffff, 0x00000100,
1372 0x55e4, 0xffffffff, 0x00200100,
1373 0x3c280, 0xffffffff, 0x00000100,
1374 0x3c214, 0xffffffff, 0x06000100,
1375 0x3c220, 0xffffffff, 0x00000100,
1376 0x3c218, 0xffffffff, 0x06000100,
1377 0x3c204, 0xffffffff, 0x00000100,
1378 0x3c2e0, 0xffffffff, 0x00000100,
1379 0x3c224, 0xffffffff, 0x00000100,
1380 0x3c200, 0xffffffff, 0x00000100,
1381 0x3c230, 0xffffffff, 0x00000100,
1382 0x3c234, 0xffffffff, 0x00000100,
1383 0x3c250, 0xffffffff, 0x00000100,
1384 0x3c254, 0xffffffff, 0x00000100,
1385 0x3c258, 0xffffffff, 0x00000100,
1386 0x3c25c, 0xffffffff, 0x00000100,
1387 0x3c260, 0xffffffff, 0x00000100,
1388 0x3c27c, 0xffffffff, 0x00000100,
1389 0x3c278, 0xffffffff, 0x00000100,
1390 0x3c210, 0xffffffff, 0x06000100,
1391 0x3c290, 0xffffffff, 0x00000100,
1392 0x3c274, 0xffffffff, 0x00000100,
1393 0x3c2b4, 0xffffffff, 0x00000100,
1394 0x3c2b0, 0xffffffff, 0x00000100,
1395 0x3c270, 0xffffffff, 0x00000100,
1396 0x30800, 0xffffffff, 0xe0000000,
1397 0x3c020, 0xffffffff, 0x00010000,
1398 0x3c024, 0xffffffff, 0x00030002,
1399 0x3c028, 0xffffffff, 0x00040007,
1400 0x3c02c, 0xffffffff, 0x00060005,
1401 0x3c030, 0xffffffff, 0x00090008,
1402 0x3c034, 0xffffffff, 0x00010000,
1403 0x3c038, 0xffffffff, 0x00030002,
1404 0x3c03c, 0xffffffff, 0x00040007,
1405 0x3c040, 0xffffffff, 0x00060005,
1406 0x3c044, 0xffffffff, 0x00090008,
1407 0x3c048, 0xffffffff, 0x00010000,
1408 0x3c04c, 0xffffffff, 0x00030002,
1409 0x3c050, 0xffffffff, 0x00040007,
1410 0x3c054, 0xffffffff, 0x00060005,
1411 0x3c058, 0xffffffff, 0x00090008,
1412 0x3c05c, 0xffffffff, 0x00010000,
1413 0x3c060, 0xffffffff, 0x00030002,
1414 0x3c064, 0xffffffff, 0x00040007,
1415 0x3c068, 0xffffffff, 0x00060005,
1416 0x3c06c, 0xffffffff, 0x00090008,
1417 0x3c070, 0xffffffff, 0x00010000,
1418 0x3c074, 0xffffffff, 0x00030002,
1419 0x3c078, 0xffffffff, 0x00040007,
1420 0x3c07c, 0xffffffff, 0x00060005,
1421 0x3c080, 0xffffffff, 0x00090008,
1422 0x3c084, 0xffffffff, 0x00010000,
1423 0x3c088, 0xffffffff, 0x00030002,
1424 0x3c08c, 0xffffffff, 0x00040007,
1425 0x3c090, 0xffffffff, 0x00060005,
1426 0x3c094, 0xffffffff, 0x00090008,
1427 0x3c098, 0xffffffff, 0x00010000,
1428 0x3c09c, 0xffffffff, 0x00030002,
1429 0x3c0a0, 0xffffffff, 0x00040007,
1430 0x3c0a4, 0xffffffff, 0x00060005,
1431 0x3c0a8, 0xffffffff, 0x00090008,
1432 0x3c0ac, 0xffffffff, 0x00010000,
1433 0x3c0b0, 0xffffffff, 0x00030002,
1434 0x3c0b4, 0xffffffff, 0x00040007,
1435 0x3c0b8, 0xffffffff, 0x00060005,
1436 0x3c0bc, 0xffffffff, 0x00090008,
1437 0x3c0c0, 0xffffffff, 0x00010000,
1438 0x3c0c4, 0xffffffff, 0x00030002,
1439 0x3c0c8, 0xffffffff, 0x00040007,
1440 0x3c0cc, 0xffffffff, 0x00060005,
1441 0x3c0d0, 0xffffffff, 0x00090008,
1442 0x3c0d4, 0xffffffff, 0x00010000,
1443 0x3c0d8, 0xffffffff, 0x00030002,
1444 0x3c0dc, 0xffffffff, 0x00040007,
1445 0x3c0e0, 0xffffffff, 0x00060005,
1446 0x3c0e4, 0xffffffff, 0x00090008,
1447 0x3c0e8, 0xffffffff, 0x00010000,
1448 0x3c0ec, 0xffffffff, 0x00030002,
1449 0x3c0f0, 0xffffffff, 0x00040007,
1450 0x3c0f4, 0xffffffff, 0x00060005,
1451 0x3c0f8, 0xffffffff, 0x00090008,
1452 0xc318, 0xffffffff, 0x00020200,
1453 0x3350, 0xffffffff, 0x00000200,
1454 0x15c0, 0xffffffff, 0x00000400,
1455 0x55e8, 0xffffffff, 0x00000000,
1456 0x2f50, 0xffffffff, 0x00000902,
1457 0x3c000, 0xffffffff, 0x96940200,
1458 0x8708, 0xffffffff, 0x00900100,
1459 0xc424, 0xffffffff, 0x0020003f,
1460 0x38, 0xffffffff, 0x0140001c,
1461 0x3c, 0x000f0000, 0x000f0000,
1462 0x220, 0xffffffff, 0xc060000c,
1463 0x224, 0xc0000fff, 0x00000100,
1464 0xf90, 0xffffffff, 0x00000100,
1465 0xf98, 0x00000101, 0x00000000,
1466 0x20a8, 0xffffffff, 0x00000104,
1467 0x55e4, 0xff000fff, 0x00000100,
1468 0x30cc, 0xc0000fff, 0x00000104,
1469 0xc1e4, 0x00000001, 0x00000001,
1470 0xd00c, 0xff000ff0, 0x00000100,
1471 0xd80c, 0xff000ff0, 0x00000100
1472};
1473
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1474static void cik_init_golden_registers(struct radeon_device *rdev)
1475{
1476 switch (rdev->family) {
1477 case CHIP_BONAIRE:
1478 radeon_program_register_sequence(rdev,
1479 bonaire_mgcg_cgcg_init,
1480 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1481 radeon_program_register_sequence(rdev,
1482 bonaire_golden_registers,
1483 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1484 radeon_program_register_sequence(rdev,
1485 bonaire_golden_common_registers,
1486 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1487 radeon_program_register_sequence(rdev,
1488 bonaire_golden_spm_registers,
1489 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1490 break;
1491 case CHIP_KABINI:
1492 radeon_program_register_sequence(rdev,
1493 kalindi_mgcg_cgcg_init,
1494 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1495 radeon_program_register_sequence(rdev,
1496 kalindi_golden_registers,
1497 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1498 radeon_program_register_sequence(rdev,
1499 kalindi_golden_common_registers,
1500 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1501 radeon_program_register_sequence(rdev,
1502 kalindi_golden_spm_registers,
1503 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1504 break;
1505 case CHIP_KAVERI:
1506 radeon_program_register_sequence(rdev,
1507 spectre_mgcg_cgcg_init,
1508 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1509 radeon_program_register_sequence(rdev,
1510 spectre_golden_registers,
1511 (const u32)ARRAY_SIZE(spectre_golden_registers));
1512 radeon_program_register_sequence(rdev,
1513 spectre_golden_common_registers,
1514 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1515 radeon_program_register_sequence(rdev,
1516 spectre_golden_spm_registers,
1517 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1518 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1519 case CHIP_HAWAII:
1520 radeon_program_register_sequence(rdev,
1521 hawaii_mgcg_cgcg_init,
1522 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1523 radeon_program_register_sequence(rdev,
1524 hawaii_golden_registers,
1525 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1526 radeon_program_register_sequence(rdev,
1527 hawaii_golden_common_registers,
1528 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1529 radeon_program_register_sequence(rdev,
1530 hawaii_golden_spm_registers,
1531 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1532 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1533 default:
1534 break;
1535 }
1536}
1537
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1538/**
1539 * cik_get_xclk - get the xclk
1540 *
1541 * @rdev: radeon_device pointer
1542 *
1543 * Returns the reference clock used by the gfx engine
1544 * (CIK).
1545 */
1546u32 cik_get_xclk(struct radeon_device *rdev)
1547{
1548 u32 reference_clock = rdev->clock.spll.reference_freq;
1549
1550 if (rdev->flags & RADEON_IS_IGP) {
1551 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1552 return reference_clock / 2;
1553 } else {
1554 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1555 return reference_clock / 4;
1556 }
1557 return reference_clock;
1558}
1559
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1560/**
1561 * cik_mm_rdoorbell - read a doorbell dword
1562 *
1563 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1564 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1565 *
1566 * Returns the value in the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1567 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1568 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1569u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1570{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1571 if (index < rdev->doorbell.num_doorbells) {
1572 return readl(rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1573 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1574 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1575 return 0;
1576 }
1577}
1578
1579/**
1580 * cik_mm_wdoorbell - write a doorbell dword
1581 *
1582 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1583 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1584 * @v: value to write
1585 *
1586 * Writes @v to the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1587 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1588 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1589void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1590{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1591 if (index < rdev->doorbell.num_doorbells) {
1592 writel(v, rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1593 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1594 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1595 }
1596}
1597
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1598#define BONAIRE_IO_MC_REGS_SIZE 36
1599
1600static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1601{
1602 {0x00000070, 0x04400000},
1603 {0x00000071, 0x80c01803},
1604 {0x00000072, 0x00004004},
1605 {0x00000073, 0x00000100},
1606 {0x00000074, 0x00ff0000},
1607 {0x00000075, 0x34000000},
1608 {0x00000076, 0x08000014},
1609 {0x00000077, 0x00cc08ec},
1610 {0x00000078, 0x00000400},
1611 {0x00000079, 0x00000000},
1612 {0x0000007a, 0x04090000},
1613 {0x0000007c, 0x00000000},
1614 {0x0000007e, 0x4408a8e8},
1615 {0x0000007f, 0x00000304},
1616 {0x00000080, 0x00000000},
1617 {0x00000082, 0x00000001},
1618 {0x00000083, 0x00000002},
1619 {0x00000084, 0xf3e4f400},
1620 {0x00000085, 0x052024e3},
1621 {0x00000087, 0x00000000},
1622 {0x00000088, 0x01000000},
1623 {0x0000008a, 0x1c0a0000},
1624 {0x0000008b, 0xff010000},
1625 {0x0000008d, 0xffffefff},
1626 {0x0000008e, 0xfff3efff},
1627 {0x0000008f, 0xfff3efbf},
1628 {0x00000092, 0xf7ffffff},
1629 {0x00000093, 0xffffff7f},
1630 {0x00000095, 0x00101101},
1631 {0x00000096, 0x00000fff},
1632 {0x00000097, 0x00116fff},
1633 {0x00000098, 0x60010000},
1634 {0x00000099, 0x10010000},
1635 {0x0000009a, 0x00006000},
1636 {0x0000009b, 0x00001000},
1637 {0x0000009f, 0x00b48000}
1638};
1639
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1640#define HAWAII_IO_MC_REGS_SIZE 22
1641
1642static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1643{
1644 {0x0000007d, 0x40000000},
1645 {0x0000007e, 0x40180304},
1646 {0x0000007f, 0x0000ff00},
1647 {0x00000081, 0x00000000},
1648 {0x00000083, 0x00000800},
1649 {0x00000086, 0x00000000},
1650 {0x00000087, 0x00000100},
1651 {0x00000088, 0x00020100},
1652 {0x00000089, 0x00000000},
1653 {0x0000008b, 0x00040000},
1654 {0x0000008c, 0x00000100},
1655 {0x0000008e, 0xff010000},
1656 {0x00000090, 0xffffefff},
1657 {0x00000091, 0xfff3efff},
1658 {0x00000092, 0xfff3efbf},
1659 {0x00000093, 0xf7ffffff},
1660 {0x00000094, 0xffffff7f},
1661 {0x00000095, 0x00000fff},
1662 {0x00000096, 0x00116fff},
1663 {0x00000097, 0x60010000},
1664 {0x00000098, 0x10010000},
1665 {0x0000009f, 0x00c79000}
1666};
1667
1668
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1669/**
1670 * cik_srbm_select - select specific register instances
1671 *
1672 * @rdev: radeon_device pointer
1673 * @me: selected ME (micro engine)
1674 * @pipe: pipe
1675 * @queue: queue
1676 * @vmid: VMID
1677 *
1678 * Switches the currently active registers instances. Some
1679 * registers are instanced per VMID, others are instanced per
1680 * me/pipe/queue combination.
1681 */
1682static void cik_srbm_select(struct radeon_device *rdev,
1683 u32 me, u32 pipe, u32 queue, u32 vmid)
1684{
1685 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1686 MEID(me & 0x3) |
1687 VMID(vmid & 0xf) |
1688 QUEUEID(queue & 0x7));
1689 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1690}
1691
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1692/* ucode loading */
1693/**
1694 * ci_mc_load_microcode - load MC ucode into the hw
1695 *
1696 * @rdev: radeon_device pointer
1697 *
1698 * Load the GDDR MC ucode into the hw (CIK).
1699 * Returns 0 on success, error on failure.
1700 */
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]1701int ci_mc_load_microcode(struct radeon_device *rdev)
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1702{
1703 const __be32 *fw_data;
1704 u32 running, blackout = 0;
1705 u32 *io_mc_regs;
1706 int i, ucode_size, regs_size;
1707
1708 if (!rdev->mc_fw)
1709 return -EINVAL;
1710
1711 switch (rdev->family) {
1712 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1713 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1714 ucode_size = CIK_MC_UCODE_SIZE;
1715 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1716 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1717 case CHIP_HAWAII:
1718 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1719 ucode_size = HAWAII_MC_UCODE_SIZE;
1720 regs_size = HAWAII_IO_MC_REGS_SIZE;
1721 break;
1722 default:
1723 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1724 }
1725
1726 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1727
1728 if (running == 0) {
1729 if (running) {
1730 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1731 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1732 }
1733
1734 /* reset the engine and set to writable */
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1736 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1737
1738 /* load mc io regs */
1739 for (i = 0; i < regs_size; i++) {
1740 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1741 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1742 }
1743 /* load the MC ucode */
1744 fw_data = (const __be32 *)rdev->mc_fw->data;
1745 for (i = 0; i < ucode_size; i++)
1746 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1747
1748 /* put the engine back into the active state */
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1752
1753 /* wait for training to complete */
1754 for (i = 0; i < rdev->usec_timeout; i++) {
1755 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1756 break;
1757 udelay(1);
1758 }
1759 for (i = 0; i < rdev->usec_timeout; i++) {
1760 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1761 break;
1762 udelay(1);
1763 }
1764
1765 if (running)
1766 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1767 }
1768
1769 return 0;
1770}
1771
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1772/**
1773 * cik_init_microcode - load ucode images from disk
1774 *
1775 * @rdev: radeon_device pointer
1776 *
1777 * Use the firmware interface to load the ucode images into
1778 * the driver (not loaded into hw).
1779 * Returns 0 on success, error on failure.
1780 */
1781static int cik_init_microcode(struct radeon_device *rdev)
1782{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1783 const char *chip_name;
1784 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1785 mec_req_size, rlc_req_size, mc_req_size = 0,
1786 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1787 char fw_name[30];
1788 int err;
1789
1790 DRM_DEBUG("\n");
1791
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1792 switch (rdev->family) {
1793 case CHIP_BONAIRE:
1794 chip_name = "BONAIRE";
1795 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1796 me_req_size = CIK_ME_UCODE_SIZE * 4;
1797 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1798 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1799 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1800 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1801 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1802 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1803 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1804 case CHIP_HAWAII:
1805 chip_name = "HAWAII";
1806 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1807 me_req_size = CIK_ME_UCODE_SIZE * 4;
1808 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1809 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1810 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1811 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1812 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1813 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1814 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1815 case CHIP_KAVERI:
1816 chip_name = "KAVERI";
1817 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1818 me_req_size = CIK_ME_UCODE_SIZE * 4;
1819 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1820 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1821 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1822 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1823 break;
1824 case CHIP_KABINI:
1825 chip_name = "KABINI";
1826 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1827 me_req_size = CIK_ME_UCODE_SIZE * 4;
1828 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1829 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1830 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1831 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1832 break;
1833 default: BUG();
1834 }
1835
1836 DRM_INFO("Loading %s Microcode\n", chip_name);
1837
1838 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1839 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1840 if (err)
1841 goto out;
1842 if (rdev->pfp_fw->size != pfp_req_size) {
1843 printk(KERN_ERR
1844 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1845 rdev->pfp_fw->size, fw_name);
1846 err = -EINVAL;
1847 goto out;
1848 }
1849
1850 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1851 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1852 if (err)
1853 goto out;
1854 if (rdev->me_fw->size != me_req_size) {
1855 printk(KERN_ERR
1856 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1857 rdev->me_fw->size, fw_name);
1858 err = -EINVAL;
1859 }
1860
1861 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1862 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1863 if (err)
1864 goto out;
1865 if (rdev->ce_fw->size != ce_req_size) {
1866 printk(KERN_ERR
1867 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1868 rdev->ce_fw->size, fw_name);
1869 err = -EINVAL;
1870 }
1871
1872 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1873 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1874 if (err)
1875 goto out;
1876 if (rdev->mec_fw->size != mec_req_size) {
1877 printk(KERN_ERR
1878 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1879 rdev->mec_fw->size, fw_name);
1880 err = -EINVAL;
1881 }
1882
1883 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1884 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1885 if (err)
1886 goto out;
1887 if (rdev->rlc_fw->size != rlc_req_size) {
1888 printk(KERN_ERR
1889 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1890 rdev->rlc_fw->size, fw_name);
1891 err = -EINVAL;
1892 }
1893
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1894 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1895 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1896 if (err)
1897 goto out;
1898 if (rdev->sdma_fw->size != sdma_req_size) {
1899 printk(KERN_ERR
1900 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1901 rdev->sdma_fw->size, fw_name);
1902 err = -EINVAL;
1903 }
1904
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1905 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1906 if (!(rdev->flags & RADEON_IS_IGP)) {
1907 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1908 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1909 if (err)
1910 goto out;
1911 if (rdev->mc_fw->size != mc_req_size) {
1912 printk(KERN_ERR
1913 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1914 rdev->mc_fw->size, fw_name);
1915 err = -EINVAL;
1916 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1917
1918 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1919 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1920 if (err) {
1921 printk(KERN_ERR
1922 "smc: error loading firmware \"%s\"\n",
1923 fw_name);
1924 release_firmware(rdev->smc_fw);
1925 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1926 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1927 } else if (rdev->smc_fw->size != smc_req_size) {
1928 printk(KERN_ERR
1929 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1930 rdev->smc_fw->size, fw_name);
1931 err = -EINVAL;
1932 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1933 }
1934
1935out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1936 if (err) {
1937 if (err != -EINVAL)
1938 printk(KERN_ERR
1939 "cik_cp: Failed to load firmware \"%s\"\n",
1940 fw_name);
1941 release_firmware(rdev->pfp_fw);
1942 rdev->pfp_fw = NULL;
1943 release_firmware(rdev->me_fw);
1944 rdev->me_fw = NULL;
1945 release_firmware(rdev->ce_fw);
1946 rdev->ce_fw = NULL;
1947 release_firmware(rdev->rlc_fw);
1948 rdev->rlc_fw = NULL;
1949 release_firmware(rdev->mc_fw);
1950 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1951 release_firmware(rdev->smc_fw);
1952 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1953 }
1954 return err;
1955}
1956
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1957/*
1958 * Core functions
1959 */
1960/**
1961 * cik_tiling_mode_table_init - init the hw tiling table
1962 *
1963 * @rdev: radeon_device pointer
1964 *
1965 * Starting with SI, the tiling setup is done globally in a
1966 * set of 32 tiling modes. Rather than selecting each set of
1967 * parameters per surface as on older asics, we just select
1968 * which index in the tiling table we want to use, and the
1969 * surface uses those parameters (CIK).
1970 */
1971static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1972{
1973 const u32 num_tile_mode_states = 32;
1974 const u32 num_secondary_tile_mode_states = 16;
1975 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1976 u32 num_pipe_configs;
1977 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1978 rdev->config.cik.max_shader_engines;
1979
1980 switch (rdev->config.cik.mem_row_size_in_kb) {
1981 case 1:
1982 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1983 break;
1984 case 2:
1985 default:
1986 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1987 break;
1988 case 4:
1989 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1990 break;
1991 }
1992
1993 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1994 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1995 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1996
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1997 if (num_pipe_configs == 16) {
1998 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1999 switch (reg_offset) {
2000 case 0:
2001 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2002 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2003 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2004 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2005 break;
2006 case 1:
2007 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2009 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2010 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2011 break;
2012 case 2:
2013 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2014 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2015 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2016 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2017 break;
2018 case 3:
2019 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2021 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2022 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2023 break;
2024 case 4:
2025 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2026 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2027 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2028 TILE_SPLIT(split_equal_to_row_size));
2029 break;
2030 case 5:
2031 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2032 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2033 break;
2034 case 6:
2035 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2036 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2037 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2038 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2039 break;
2040 case 7:
2041 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2042 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2043 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2044 TILE_SPLIT(split_equal_to_row_size));
2045 break;
2046 case 8:
2047 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2048 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2049 break;
2050 case 9:
2051 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2052 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2053 break;
2054 case 10:
2055 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2056 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2057 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2058 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2059 break;
2060 case 11:
2061 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2062 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2063 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2064 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2065 break;
2066 case 12:
2067 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2068 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2069 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071 break;
2072 case 13:
2073 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2074 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2075 break;
2076 case 14:
2077 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2078 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2079 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2080 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2081 break;
2082 case 16:
2083 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2084 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2085 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2087 break;
2088 case 17:
2089 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2090 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2091 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2092 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2093 break;
2094 case 27:
2095 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2096 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2097 break;
2098 case 28:
2099 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2100 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2101 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2102 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2103 break;
2104 case 29:
2105 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2106 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2107 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2108 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2109 break;
2110 case 30:
2111 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2112 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2113 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2115 break;
2116 default:
2117 gb_tile_moden = 0;
2118 break;
2119 }
2120 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2121 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2122 }
2123 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2124 switch (reg_offset) {
2125 case 0:
2126 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2127 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2128 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2129 NUM_BANKS(ADDR_SURF_16_BANK));
2130 break;
2131 case 1:
2132 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2133 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2134 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2135 NUM_BANKS(ADDR_SURF_16_BANK));
2136 break;
2137 case 2:
2138 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2139 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2140 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2141 NUM_BANKS(ADDR_SURF_16_BANK));
2142 break;
2143 case 3:
2144 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2147 NUM_BANKS(ADDR_SURF_16_BANK));
2148 break;
2149 case 4:
2150 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2151 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2152 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2153 NUM_BANKS(ADDR_SURF_8_BANK));
2154 break;
2155 case 5:
2156 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2159 NUM_BANKS(ADDR_SURF_4_BANK));
2160 break;
2161 case 6:
2162 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2163 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2164 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2165 NUM_BANKS(ADDR_SURF_2_BANK));
2166 break;
2167 case 8:
2168 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2169 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2170 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2171 NUM_BANKS(ADDR_SURF_16_BANK));
2172 break;
2173 case 9:
2174 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2177 NUM_BANKS(ADDR_SURF_16_BANK));
2178 break;
2179 case 10:
2180 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2183 NUM_BANKS(ADDR_SURF_16_BANK));
2184 break;
2185 case 11:
2186 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2187 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2188 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2189 NUM_BANKS(ADDR_SURF_8_BANK));
2190 break;
2191 case 12:
2192 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2193 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2194 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2195 NUM_BANKS(ADDR_SURF_4_BANK));
2196 break;
2197 case 13:
2198 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2199 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2200 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2201 NUM_BANKS(ADDR_SURF_2_BANK));
2202 break;
2203 case 14:
2204 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2207 NUM_BANKS(ADDR_SURF_2_BANK));
2208 break;
2209 default:
2210 gb_tile_moden = 0;
2211 break;
2212 }
2213 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2214 }
2215 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2216 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2217 switch (reg_offset) {
2218 case 0:
2219 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2220 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2221 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2222 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2223 break;
2224 case 1:
2225 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2226 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2227 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2228 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2229 break;
2230 case 2:
2231 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2232 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2233 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2235 break;
2236 case 3:
2237 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2238 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2239 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2241 break;
2242 case 4:
2243 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2244 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2245 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2246 TILE_SPLIT(split_equal_to_row_size));
2247 break;
2248 case 5:
2249 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2250 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2251 break;
2252 case 6:
2253 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2254 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2255 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2256 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2257 break;
2258 case 7:
2259 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2260 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2261 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2262 TILE_SPLIT(split_equal_to_row_size));
2263 break;
2264 case 8:
2265 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2267 break;
2268 case 9:
2269 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2270 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2271 break;
2272 case 10:
2273 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2274 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2276 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2277 break;
2278 case 11:
2279 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2283 break;
2284 case 12:
2285 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2287 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2289 break;
2290 case 13:
2291 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2292 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2293 break;
2294 case 14:
2295 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2296 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2299 break;
2300 case 16:
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2305 break;
2306 case 17:
2307 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2308 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2309 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2311 break;
2312 case 27:
2313 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2314 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2315 break;
2316 case 28:
2317 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2319 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2320 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2321 break;
2322 case 29:
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2324 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2325 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2327 break;
2328 case 30:
2329 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2333 break;
2334 default:
2335 gb_tile_moden = 0;
2336 break;
2337 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2338 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2339 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2340 }
2341 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2342 switch (reg_offset) {
2343 case 0:
2344 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2346 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2347 NUM_BANKS(ADDR_SURF_16_BANK));
2348 break;
2349 case 1:
2350 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2351 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2352 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2353 NUM_BANKS(ADDR_SURF_16_BANK));
2354 break;
2355 case 2:
2356 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2359 NUM_BANKS(ADDR_SURF_16_BANK));
2360 break;
2361 case 3:
2362 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2365 NUM_BANKS(ADDR_SURF_16_BANK));
2366 break;
2367 case 4:
2368 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2371 NUM_BANKS(ADDR_SURF_8_BANK));
2372 break;
2373 case 5:
2374 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2377 NUM_BANKS(ADDR_SURF_4_BANK));
2378 break;
2379 case 6:
2380 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2383 NUM_BANKS(ADDR_SURF_2_BANK));
2384 break;
2385 case 8:
2386 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2389 NUM_BANKS(ADDR_SURF_16_BANK));
2390 break;
2391 case 9:
2392 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2395 NUM_BANKS(ADDR_SURF_16_BANK));
2396 break;
2397 case 10:
2398 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2401 NUM_BANKS(ADDR_SURF_16_BANK));
2402 break;
2403 case 11:
2404 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2407 NUM_BANKS(ADDR_SURF_16_BANK));
2408 break;
2409 case 12:
2410 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2413 NUM_BANKS(ADDR_SURF_8_BANK));
2414 break;
2415 case 13:
2416 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2419 NUM_BANKS(ADDR_SURF_4_BANK));
2420 break;
2421 case 14:
2422 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2425 NUM_BANKS(ADDR_SURF_2_BANK));
2426 break;
2427 default:
2428 gb_tile_moden = 0;
2429 break;
2430 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2431 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2432 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2433 }
2434 } else if (num_pipe_configs == 4) {
2435 if (num_rbs == 4) {
2436 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2437 switch (reg_offset) {
2438 case 0:
2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2443 break;
2444 case 1:
2445 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2446 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2447 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2448 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2449 break;
2450 case 2:
2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2455 break;
2456 case 3:
2457 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2459 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2461 break;
2462 case 4:
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2466 TILE_SPLIT(split_equal_to_row_size));
2467 break;
2468 case 5:
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2471 break;
2472 case 6:
2473 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2475 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2476 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2477 break;
2478 case 7:
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2480 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2481 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2482 TILE_SPLIT(split_equal_to_row_size));
2483 break;
2484 case 8:
2485 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2486 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2487 break;
2488 case 9:
2489 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2490 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2491 break;
2492 case 10:
2493 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2494 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2495 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2497 break;
2498 case 11:
2499 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2501 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2502 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2503 break;
2504 case 12:
2505 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2508 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2509 break;
2510 case 13:
2511 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2512 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2513 break;
2514 case 14:
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2518 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2519 break;
2520 case 16:
2521 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2525 break;
2526 case 17:
2527 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2528 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2529 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2530 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2531 break;
2532 case 27:
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2535 break;
2536 case 28:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2540 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2541 break;
2542 case 29:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2547 break;
2548 case 30:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553 break;
2554 default:
2555 gb_tile_moden = 0;
2556 break;
2557 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2558 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2559 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2560 }
2561 } else if (num_rbs < 4) {
2562 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2563 switch (reg_offset) {
2564 case 0:
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2569 break;
2570 case 1:
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2574 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2575 break;
2576 case 2:
2577 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2581 break;
2582 case 3:
2583 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2584 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2587 break;
2588 case 4:
2589 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2590 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2591 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2592 TILE_SPLIT(split_equal_to_row_size));
2593 break;
2594 case 5:
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2597 break;
2598 case 6:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2603 break;
2604 case 7:
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(split_equal_to_row_size));
2609 break;
2610 case 8:
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2613 break;
2614 case 9:
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2617 break;
2618 case 10:
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2622 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2623 break;
2624 case 11:
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2626 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2628 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2629 break;
2630 case 12:
2631 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2635 break;
2636 case 13:
2637 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2638 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2639 break;
2640 case 14:
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2645 break;
2646 case 16:
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2649 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2651 break;
2652 case 17:
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 break;
2658 case 27:
2659 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2660 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2661 break;
2662 case 28:
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2666 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2667 break;
2668 case 29:
2669 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2670 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2671 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2672 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2673 break;
2674 case 30:
2675 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2678 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2679 break;
2680 default:
2681 gb_tile_moden = 0;
2682 break;
2683 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2684 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2685 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2686 }
2687 }
2688 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2689 switch (reg_offset) {
2690 case 0:
2691 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2692 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2693 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2694 NUM_BANKS(ADDR_SURF_16_BANK));
2695 break;
2696 case 1:
2697 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2700 NUM_BANKS(ADDR_SURF_16_BANK));
2701 break;
2702 case 2:
2703 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2704 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2705 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2706 NUM_BANKS(ADDR_SURF_16_BANK));
2707 break;
2708 case 3:
2709 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2710 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2711 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2712 NUM_BANKS(ADDR_SURF_16_BANK));
2713 break;
2714 case 4:
2715 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2718 NUM_BANKS(ADDR_SURF_16_BANK));
2719 break;
2720 case 5:
2721 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2722 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2723 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2724 NUM_BANKS(ADDR_SURF_8_BANK));
2725 break;
2726 case 6:
2727 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2730 NUM_BANKS(ADDR_SURF_4_BANK));
2731 break;
2732 case 8:
2733 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2734 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2735 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2736 NUM_BANKS(ADDR_SURF_16_BANK));
2737 break;
2738 case 9:
2739 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2740 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2741 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2742 NUM_BANKS(ADDR_SURF_16_BANK));
2743 break;
2744 case 10:
2745 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2746 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2747 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2748 NUM_BANKS(ADDR_SURF_16_BANK));
2749 break;
2750 case 11:
2751 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2752 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2753 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2754 NUM_BANKS(ADDR_SURF_16_BANK));
2755 break;
2756 case 12:
2757 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2758 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2759 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2760 NUM_BANKS(ADDR_SURF_16_BANK));
2761 break;
2762 case 13:
2763 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2764 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2765 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2766 NUM_BANKS(ADDR_SURF_8_BANK));
2767 break;
2768 case 14:
2769 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2770 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2771 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2772 NUM_BANKS(ADDR_SURF_4_BANK));
2773 break;
2774 default:
2775 gb_tile_moden = 0;
2776 break;
2777 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2778 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2779 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2780 }
2781 } else if (num_pipe_configs == 2) {
2782 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2783 switch (reg_offset) {
2784 case 0:
2785 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2786 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2787 PIPE_CONFIG(ADDR_SURF_P2) |
2788 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2789 break;
2790 case 1:
2791 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2792 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P2) |
2794 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2795 break;
2796 case 2:
2797 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2798 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2799 PIPE_CONFIG(ADDR_SURF_P2) |
2800 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2801 break;
2802 case 3:
2803 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2805 PIPE_CONFIG(ADDR_SURF_P2) |
2806 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2807 break;
2808 case 4:
2809 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2811 PIPE_CONFIG(ADDR_SURF_P2) |
2812 TILE_SPLIT(split_equal_to_row_size));
2813 break;
2814 case 5:
2815 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2817 break;
2818 case 6:
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P2) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2823 break;
2824 case 7:
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P2) |
2828 TILE_SPLIT(split_equal_to_row_size));
2829 break;
2830 case 8:
2831 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2832 break;
2833 case 9:
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2836 break;
2837 case 10:
2838 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2840 PIPE_CONFIG(ADDR_SURF_P2) |
2841 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2842 break;
2843 case 11:
2844 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2845 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2846 PIPE_CONFIG(ADDR_SURF_P2) |
2847 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2848 break;
2849 case 12:
2850 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2851 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2852 PIPE_CONFIG(ADDR_SURF_P2) |
2853 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2854 break;
2855 case 13:
2856 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2857 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2858 break;
2859 case 14:
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2861 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2862 PIPE_CONFIG(ADDR_SURF_P2) |
2863 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2864 break;
2865 case 16:
2866 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2867 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2868 PIPE_CONFIG(ADDR_SURF_P2) |
2869 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2870 break;
2871 case 17:
2872 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2873 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2874 PIPE_CONFIG(ADDR_SURF_P2) |
2875 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2876 break;
2877 case 27:
2878 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2879 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2880 break;
2881 case 28:
2882 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2883 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2884 PIPE_CONFIG(ADDR_SURF_P2) |
2885 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2886 break;
2887 case 29:
2888 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2890 PIPE_CONFIG(ADDR_SURF_P2) |
2891 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2892 break;
2893 case 30:
2894 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2896 PIPE_CONFIG(ADDR_SURF_P2) |
2897 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2898 break;
2899 default:
2900 gb_tile_moden = 0;
2901 break;
2902 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2903 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2904 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2905 }
2906 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2907 switch (reg_offset) {
2908 case 0:
2909 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2910 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2911 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2912 NUM_BANKS(ADDR_SURF_16_BANK));
2913 break;
2914 case 1:
2915 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2916 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2917 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2918 NUM_BANKS(ADDR_SURF_16_BANK));
2919 break;
2920 case 2:
2921 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2922 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2923 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2924 NUM_BANKS(ADDR_SURF_16_BANK));
2925 break;
2926 case 3:
2927 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2928 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2929 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2930 NUM_BANKS(ADDR_SURF_16_BANK));
2931 break;
2932 case 4:
2933 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2934 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2935 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2936 NUM_BANKS(ADDR_SURF_16_BANK));
2937 break;
2938 case 5:
2939 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2940 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2941 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2942 NUM_BANKS(ADDR_SURF_16_BANK));
2943 break;
2944 case 6:
2945 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2946 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2947 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2948 NUM_BANKS(ADDR_SURF_8_BANK));
2949 break;
2950 case 8:
2951 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2952 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2953 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2954 NUM_BANKS(ADDR_SURF_16_BANK));
2955 break;
2956 case 9:
2957 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2958 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2959 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2960 NUM_BANKS(ADDR_SURF_16_BANK));
2961 break;
2962 case 10:
2963 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966 NUM_BANKS(ADDR_SURF_16_BANK));
2967 break;
2968 case 11:
2969 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2970 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2971 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2972 NUM_BANKS(ADDR_SURF_16_BANK));
2973 break;
2974 case 12:
2975 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2976 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2977 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978 NUM_BANKS(ADDR_SURF_16_BANK));
2979 break;
2980 case 13:
2981 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2982 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2983 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2984 NUM_BANKS(ADDR_SURF_16_BANK));
2985 break;
2986 case 14:
2987 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2990 NUM_BANKS(ADDR_SURF_8_BANK));
2991 break;
2992 default:
2993 gb_tile_moden = 0;
2994 break;
2995 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2996 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2997 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2998 }
2999 } else
3000 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3001}
3002
3003/**
3004 * cik_select_se_sh - select which SE, SH to address
3005 *
3006 * @rdev: radeon_device pointer
3007 * @se_num: shader engine to address
3008 * @sh_num: sh block to address
3009 *
3010 * Select which SE, SH combinations to address. Certain
3011 * registers are instanced per SE or SH. 0xffffffff means
3012 * broadcast to all SEs or SHs (CIK).
3013 */
3014static void cik_select_se_sh(struct radeon_device *rdev,
3015 u32 se_num, u32 sh_num)
3016{
3017 u32 data = INSTANCE_BROADCAST_WRITES;
3018
3019 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3020 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3021 else if (se_num == 0xffffffff)
3022 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3023 else if (sh_num == 0xffffffff)
3024 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3025 else
3026 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3027 WREG32(GRBM_GFX_INDEX, data);
3028}
3029
3030/**
3031 * cik_create_bitmask - create a bitmask
3032 *
3033 * @bit_width: length of the mask
3034 *
3035 * create a variable length bit mask (CIK).
3036 * Returns the bitmask.
3037 */
3038static u32 cik_create_bitmask(u32 bit_width)
3039{
3040 u32 i, mask = 0;
3041
3042 for (i = 0; i < bit_width; i++) {
3043 mask <<= 1;
3044 mask |= 1;
3045 }
3046 return mask;
3047}
3048
3049/**
3050 * cik_select_se_sh - select which SE, SH to address
3051 *
3052 * @rdev: radeon_device pointer
3053 * @max_rb_num: max RBs (render backends) for the asic
3054 * @se_num: number of SEs (shader engines) for the asic
3055 * @sh_per_se: number of SH blocks per SE for the asic
3056 *
3057 * Calculates the bitmask of disabled RBs (CIK).
3058 * Returns the disabled RB bitmask.
3059 */
3060static u32 cik_get_rb_disabled(struct radeon_device *rdev,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3061 u32 max_rb_num_per_se,
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3062 u32 sh_per_se)
3063{
3064 u32 data, mask;
3065
3066 data = RREG32(CC_RB_BACKEND_DISABLE);
3067 if (data & 1)
3068 data &= BACKEND_DISABLE_MASK;
3069 else
3070 data = 0;
3071 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3072
3073 data >>= BACKEND_DISABLE_SHIFT;
3074
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3075 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3076
3077 return data & mask;
3078}
3079
3080/**
3081 * cik_setup_rb - setup the RBs on the asic
3082 *
3083 * @rdev: radeon_device pointer
3084 * @se_num: number of SEs (shader engines) for the asic
3085 * @sh_per_se: number of SH blocks per SE for the asic
3086 * @max_rb_num: max RBs (render backends) for the asic
3087 *
3088 * Configures per-SE/SH RB registers (CIK).
3089 */
3090static void cik_setup_rb(struct radeon_device *rdev,
3091 u32 se_num, u32 sh_per_se,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3092 u32 max_rb_num_per_se)
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3093{
3094 int i, j;
3095 u32 data, mask;
3096 u32 disabled_rbs = 0;
3097 u32 enabled_rbs = 0;
3098
3099 for (i = 0; i < se_num; i++) {
3100 for (j = 0; j < sh_per_se; j++) {
3101 cik_select_se_sh(rdev, i, j);
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3102 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3103 if (rdev->family == CHIP_HAWAII)
3104 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3105 else
3106 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3107 }
3108 }
3109 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3110
3111 mask = 1;
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3112 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3113 if (!(disabled_rbs & mask))
3114 enabled_rbs |= mask;
3115 mask <<= 1;
3116 }
3117
Marek Olšák439a1cf2013-12-22 02:18:01 +0100[diff] [blame]3118 rdev->config.cik.backend_enable_mask = enabled_rbs;
3119
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3120 for (i = 0; i < se_num; i++) {
3121 cik_select_se_sh(rdev, i, 0xffffffff);
3122 data = 0;
3123 for (j = 0; j < sh_per_se; j++) {
3124 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3125 case 0:
3126 if (j == 0)
3127 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3128 else
3129 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3130 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3131 case 1:
3132 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3133 break;
3134 case 2:
3135 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3136 break;
3137 case 3:
3138 default:
3139 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3140 break;
3141 }
3142 enabled_rbs >>= 2;
3143 }
3144 WREG32(PA_SC_RASTER_CONFIG, data);
3145 }
3146 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3147}
3148
3149/**
3150 * cik_gpu_init - setup the 3D engine
3151 *
3152 * @rdev: radeon_device pointer
3153 *
3154 * Configures the 3D engine and tiling configuration
3155 * registers so that the 3D engine is usable.
3156 */
3157static void cik_gpu_init(struct radeon_device *rdev)
3158{
3159 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3160 u32 mc_shared_chmap, mc_arb_ramcfg;
3161 u32 hdp_host_path_cntl;
3162 u32 tmp;
3163 int i, j;
3164
3165 switch (rdev->family) {
3166 case CHIP_BONAIRE:
3167 rdev->config.cik.max_shader_engines = 2;
3168 rdev->config.cik.max_tile_pipes = 4;
3169 rdev->config.cik.max_cu_per_sh = 7;
3170 rdev->config.cik.max_sh_per_se = 1;
3171 rdev->config.cik.max_backends_per_se = 2;
3172 rdev->config.cik.max_texture_channel_caches = 4;
3173 rdev->config.cik.max_gprs = 256;
3174 rdev->config.cik.max_gs_threads = 32;
3175 rdev->config.cik.max_hw_contexts = 8;
3176
3177 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3178 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3179 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3180 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3181 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3182 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3183 case CHIP_HAWAII:
3184 rdev->config.cik.max_shader_engines = 4;
3185 rdev->config.cik.max_tile_pipes = 16;
3186 rdev->config.cik.max_cu_per_sh = 11;
3187 rdev->config.cik.max_sh_per_se = 1;
3188 rdev->config.cik.max_backends_per_se = 4;
3189 rdev->config.cik.max_texture_channel_caches = 16;
3190 rdev->config.cik.max_gprs = 256;
3191 rdev->config.cik.max_gs_threads = 32;
3192 rdev->config.cik.max_hw_contexts = 8;
3193
3194 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3195 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3196 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3197 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3198 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3199 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3200 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3201 rdev->config.cik.max_shader_engines = 1;
3202 rdev->config.cik.max_tile_pipes = 4;
3203 if ((rdev->pdev->device == 0x1304) ||
3204 (rdev->pdev->device == 0x1305) ||
3205 (rdev->pdev->device == 0x130C) ||
3206 (rdev->pdev->device == 0x130F) ||
3207 (rdev->pdev->device == 0x1310) ||
3208 (rdev->pdev->device == 0x1311) ||
3209 (rdev->pdev->device == 0x131C)) {
3210 rdev->config.cik.max_cu_per_sh = 8;
3211 rdev->config.cik.max_backends_per_se = 2;
3212 } else if ((rdev->pdev->device == 0x1309) ||
3213 (rdev->pdev->device == 0x130A) ||
3214 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3215 (rdev->pdev->device == 0x1313) ||
3216 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3217 rdev->config.cik.max_cu_per_sh = 6;
3218 rdev->config.cik.max_backends_per_se = 2;
3219 } else if ((rdev->pdev->device == 0x1306) ||
3220 (rdev->pdev->device == 0x1307) ||
3221 (rdev->pdev->device == 0x130B) ||
3222 (rdev->pdev->device == 0x130E) ||
3223 (rdev->pdev->device == 0x1315) ||
3224 (rdev->pdev->device == 0x131B)) {
3225 rdev->config.cik.max_cu_per_sh = 4;
3226 rdev->config.cik.max_backends_per_se = 1;
3227 } else {
3228 rdev->config.cik.max_cu_per_sh = 3;
3229 rdev->config.cik.max_backends_per_se = 1;
3230 }
3231 rdev->config.cik.max_sh_per_se = 1;
3232 rdev->config.cik.max_texture_channel_caches = 4;
3233 rdev->config.cik.max_gprs = 256;
3234 rdev->config.cik.max_gs_threads = 16;
3235 rdev->config.cik.max_hw_contexts = 8;
3236
3237 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3238 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3239 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3240 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3241 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3242 break;
3243 case CHIP_KABINI:
3244 default:
3245 rdev->config.cik.max_shader_engines = 1;
3246 rdev->config.cik.max_tile_pipes = 2;
3247 rdev->config.cik.max_cu_per_sh = 2;
3248 rdev->config.cik.max_sh_per_se = 1;
3249 rdev->config.cik.max_backends_per_se = 1;
3250 rdev->config.cik.max_texture_channel_caches = 2;
3251 rdev->config.cik.max_gprs = 256;
3252 rdev->config.cik.max_gs_threads = 16;
3253 rdev->config.cik.max_hw_contexts = 8;
3254
3255 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3256 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3257 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3258 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3259 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3260 break;
3261 }
3262
3263 /* Initialize HDP */
3264 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3265 WREG32((0x2c14 + j), 0x00000000);
3266 WREG32((0x2c18 + j), 0x00000000);
3267 WREG32((0x2c1c + j), 0x00000000);
3268 WREG32((0x2c20 + j), 0x00000000);
3269 WREG32((0x2c24 + j), 0x00000000);
3270 }
3271
3272 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3273
3274 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3275
3276 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3277 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3278
3279 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3280 rdev->config.cik.mem_max_burst_length_bytes = 256;
3281 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3282 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3283 if (rdev->config.cik.mem_row_size_in_kb > 4)
3284 rdev->config.cik.mem_row_size_in_kb = 4;
3285 /* XXX use MC settings? */
3286 rdev->config.cik.shader_engine_tile_size = 32;
3287 rdev->config.cik.num_gpus = 1;
3288 rdev->config.cik.multi_gpu_tile_size = 64;
3289
3290 /* fix up row size */
3291 gb_addr_config &= ~ROW_SIZE_MASK;
3292 switch (rdev->config.cik.mem_row_size_in_kb) {
3293 case 1:
3294 default:
3295 gb_addr_config |= ROW_SIZE(0);
3296 break;
3297 case 2:
3298 gb_addr_config |= ROW_SIZE(1);
3299 break;
3300 case 4:
3301 gb_addr_config |= ROW_SIZE(2);
3302 break;
3303 }
3304
3305 /* setup tiling info dword. gb_addr_config is not adequate since it does
3306 * not have bank info, so create a custom tiling dword.
3307 * bits 3:0 num_pipes
3308 * bits 7:4 num_banks
3309 * bits 11:8 group_size
3310 * bits 15:12 row_size
3311 */
3312 rdev->config.cik.tile_config = 0;
3313 switch (rdev->config.cik.num_tile_pipes) {
3314 case 1:
3315 rdev->config.cik.tile_config |= (0 << 0);
3316 break;
3317 case 2:
3318 rdev->config.cik.tile_config |= (1 << 0);
3319 break;
3320 case 4:
3321 rdev->config.cik.tile_config |= (2 << 0);
3322 break;
3323 case 8:
3324 default:
3325 /* XXX what about 12? */
3326 rdev->config.cik.tile_config |= (3 << 0);
3327 break;
3328 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3329 rdev->config.cik.tile_config |=
3330 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3331 rdev->config.cik.tile_config |=
3332 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3333 rdev->config.cik.tile_config |=
3334 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3335
3336 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3337 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3338 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3339 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3340 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3341 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3342 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3343 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3344
3345 cik_tiling_mode_table_init(rdev);
3346
3347 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3348 rdev->config.cik.max_sh_per_se,
3349 rdev->config.cik.max_backends_per_se);
3350
3351 /* set HW defaults for 3D engine */
3352 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3353
3354 WREG32(SX_DEBUG_1, 0x20);
3355
3356 WREG32(TA_CNTL_AUX, 0x00010000);
3357
3358 tmp = RREG32(SPI_CONFIG_CNTL);
3359 tmp |= 0x03000000;
3360 WREG32(SPI_CONFIG_CNTL, tmp);
3361
3362 WREG32(SQ_CONFIG, 1);
3363
3364 WREG32(DB_DEBUG, 0);
3365
3366 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3367 tmp |= 0x00000400;
3368 WREG32(DB_DEBUG2, tmp);
3369
3370 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3371 tmp |= 0x00020200;
3372 WREG32(DB_DEBUG3, tmp);
3373
3374 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3375 tmp |= 0x00018208;
3376 WREG32(CB_HW_CONTROL, tmp);
3377
3378 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3379
3380 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3381 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3382 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3383 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3384
3385 WREG32(VGT_NUM_INSTANCES, 1);
3386
3387 WREG32(CP_PERFMON_CNTL, 0);
3388
3389 WREG32(SQ_CONFIG, 0);
3390
3391 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3392 FORCE_EOV_MAX_REZ_CNT(255)));
3393
3394 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3395 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3396
3397 WREG32(VGT_GS_VERTEX_REUSE, 16);
3398 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3399
3400 tmp = RREG32(HDP_MISC_CNTL);
3401 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3402 WREG32(HDP_MISC_CNTL, tmp);
3403
3404 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3405 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3406
3407 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3408 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3409
3410 udelay(50);
3411}
3412
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3413/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3414 * GPU scratch registers helpers function.
3415 */
3416/**
3417 * cik_scratch_init - setup driver info for CP scratch regs
3418 *
3419 * @rdev: radeon_device pointer
3420 *
3421 * Set up the number and offset of the CP scratch registers.
3422 * NOTE: use of CP scratch registers is a legacy inferface and
3423 * is not used by default on newer asics (r6xx+). On newer asics,
3424 * memory buffers are used for fences rather than scratch regs.
3425 */
3426static void cik_scratch_init(struct radeon_device *rdev)
3427{
3428 int i;
3429
3430 rdev->scratch.num_reg = 7;
3431 rdev->scratch.reg_base = SCRATCH_REG0;
3432 for (i = 0; i < rdev->scratch.num_reg; i++) {
3433 rdev->scratch.free[i] = true;
3434 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3435 }
3436}
3437
3438/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3439 * cik_ring_test - basic gfx ring test
3440 *
3441 * @rdev: radeon_device pointer
3442 * @ring: radeon_ring structure holding ring information
3443 *
3444 * Allocate a scratch register and write to it using the gfx ring (CIK).
3445 * Provides a basic gfx ring test to verify that the ring is working.
3446 * Used by cik_cp_gfx_resume();
3447 * Returns 0 on success, error on failure.
3448 */
3449int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3450{
3451 uint32_t scratch;
3452 uint32_t tmp = 0;
3453 unsigned i;
3454 int r;
3455
3456 r = radeon_scratch_get(rdev, &scratch);
3457 if (r) {
3458 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3459 return r;
3460 }
3461 WREG32(scratch, 0xCAFEDEAD);
3462 r = radeon_ring_lock(rdev, ring, 3);
3463 if (r) {
3464 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3465 radeon_scratch_free(rdev, scratch);
3466 return r;
3467 }
3468 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3469 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3470 radeon_ring_write(ring, 0xDEADBEEF);
3471 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3472
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3473 for (i = 0; i < rdev->usec_timeout; i++) {
3474 tmp = RREG32(scratch);
3475 if (tmp == 0xDEADBEEF)
3476 break;
3477 DRM_UDELAY(1);
3478 }
3479 if (i < rdev->usec_timeout) {
3480 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3481 } else {
3482 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3483 ring->idx, scratch, tmp);
3484 r = -EINVAL;
3485 }
3486 radeon_scratch_free(rdev, scratch);
3487 return r;
3488}
3489
3490/**
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3491 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3492 *
3493 * @rdev: radeon_device pointer
3494 * @ridx: radeon ring index
3495 *
3496 * Emits an hdp flush on the cp.
3497 */
3498static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3499 int ridx)
3500{
3501 struct radeon_ring *ring = &rdev->ring[ridx];
Alex Deucher5d259062014-01-09 16:51:56 -0500[diff] [blame]3502 u32 ref_and_mask;
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3503
Alex Deucher5d259062014-01-09 16:51:56 -0500[diff] [blame]3504 switch (ring->idx) {
3505 case CAYMAN_RING_TYPE_CP1_INDEX:
3506 case CAYMAN_RING_TYPE_CP2_INDEX:
3507 default:
3508 switch (ring->me) {
3509 case 0:
3510 ref_and_mask = CP2 << ring->pipe;
3511 break;
3512 case 1:
3513 ref_and_mask = CP6 << ring->pipe;
3514 break;
3515 default:
3516 return;
3517 }
3518 break;
3519 case RADEON_RING_TYPE_GFX_INDEX:
3520 ref_and_mask = CP0;
3521 break;
3522 }
3523
3524 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3525 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3526 WAIT_REG_MEM_FUNCTION(3) | /* == */
3527 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3528 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3529 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3530 radeon_ring_write(ring, ref_and_mask);
3531 radeon_ring_write(ring, ref_and_mask);
3532 radeon_ring_write(ring, 0x20); /* poll interval */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3533}
3534
3535/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3536 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3537 *
3538 * @rdev: radeon_device pointer
3539 * @fence: radeon fence object
3540 *
3541 * Emits a fence sequnce number on the gfx ring and flushes
3542 * GPU caches.
3543 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3544void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3545 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3546{
3547 struct radeon_ring *ring = &rdev->ring[fence->ring];
3548 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3549
3550 /* EVENT_WRITE_EOP - flush caches, send int */
3551 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3552 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3553 EOP_TC_ACTION_EN |
3554 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3555 EVENT_INDEX(5)));
3556 radeon_ring_write(ring, addr & 0xfffffffc);
3557 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3558 radeon_ring_write(ring, fence->seq);
3559 radeon_ring_write(ring, 0);
3560 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3561 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3562}
3563
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3564/**
3565 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3566 *
3567 * @rdev: radeon_device pointer
3568 * @fence: radeon fence object
3569 *
3570 * Emits a fence sequnce number on the compute ring and flushes
3571 * GPU caches.
3572 */
3573void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3574 struct radeon_fence *fence)
3575{
3576 struct radeon_ring *ring = &rdev->ring[fence->ring];
3577 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3578
3579 /* RELEASE_MEM - flush caches, send int */
3580 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3581 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3582 EOP_TC_ACTION_EN |
3583 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3584 EVENT_INDEX(5)));
3585 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3586 radeon_ring_write(ring, addr & 0xfffffffc);
3587 radeon_ring_write(ring, upper_32_bits(addr));
3588 radeon_ring_write(ring, fence->seq);
3589 radeon_ring_write(ring, 0);
3590 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3591 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3592}
3593
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3594bool cik_semaphore_ring_emit(struct radeon_device *rdev,
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3595 struct radeon_ring *ring,
3596 struct radeon_semaphore *semaphore,
3597 bool emit_wait)
3598{
3599 uint64_t addr = semaphore->gpu_addr;
3600 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3601
3602 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3603 radeon_ring_write(ring, addr & 0xffffffff);
3604 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3605
3606 return true;
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3607}
3608
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3609/**
3610 * cik_copy_cpdma - copy pages using the CP DMA engine
3611 *
3612 * @rdev: radeon_device pointer
3613 * @src_offset: src GPU address
3614 * @dst_offset: dst GPU address
3615 * @num_gpu_pages: number of GPU pages to xfer
3616 * @fence: radeon fence object
3617 *
3618 * Copy GPU paging using the CP DMA engine (CIK+).
3619 * Used by the radeon ttm implementation to move pages if
3620 * registered as the asic copy callback.
3621 */
3622int cik_copy_cpdma(struct radeon_device *rdev,
3623 uint64_t src_offset, uint64_t dst_offset,
3624 unsigned num_gpu_pages,
3625 struct radeon_fence **fence)
3626{
3627 struct radeon_semaphore *sem = NULL;
3628 int ring_index = rdev->asic->copy.blit_ring_index;
3629 struct radeon_ring *ring = &rdev->ring[ring_index];
3630 u32 size_in_bytes, cur_size_in_bytes, control;
3631 int i, num_loops;
3632 int r = 0;
3633
3634 r = radeon_semaphore_create(rdev, &sem);
3635 if (r) {
3636 DRM_ERROR("radeon: moving bo (%d).\n", r);
3637 return r;
3638 }
3639
3640 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3641 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3642 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3643 if (r) {
3644 DRM_ERROR("radeon: moving bo (%d).\n", r);
3645 radeon_semaphore_free(rdev, &sem, NULL);
3646 return r;
3647 }
3648
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3649 radeon_semaphore_sync_to(sem, *fence);
3650 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3651
3652 for (i = 0; i < num_loops; i++) {
3653 cur_size_in_bytes = size_in_bytes;
3654 if (cur_size_in_bytes > 0x1fffff)
3655 cur_size_in_bytes = 0x1fffff;
3656 size_in_bytes -= cur_size_in_bytes;
3657 control = 0;
3658 if (size_in_bytes == 0)
3659 control |= PACKET3_DMA_DATA_CP_SYNC;
3660 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3661 radeon_ring_write(ring, control);
3662 radeon_ring_write(ring, lower_32_bits(src_offset));
3663 radeon_ring_write(ring, upper_32_bits(src_offset));
3664 radeon_ring_write(ring, lower_32_bits(dst_offset));
3665 radeon_ring_write(ring, upper_32_bits(dst_offset));
3666 radeon_ring_write(ring, cur_size_in_bytes);
3667 src_offset += cur_size_in_bytes;
3668 dst_offset += cur_size_in_bytes;
3669 }
3670
3671 r = radeon_fence_emit(rdev, fence, ring->idx);
3672 if (r) {
3673 radeon_ring_unlock_undo(rdev, ring);
3674 return r;
3675 }
3676
3677 radeon_ring_unlock_commit(rdev, ring);
3678 radeon_semaphore_free(rdev, &sem, *fence);
3679
3680 return r;
3681}
3682
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3683/*
3684 * IB stuff
3685 */
3686/**
3687 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3688 *
3689 * @rdev: radeon_device pointer
3690 * @ib: radeon indirect buffer object
3691 *
3692 * Emits an DE (drawing engine) or CE (constant engine) IB
3693 * on the gfx ring. IBs are usually generated by userspace
3694 * acceleration drivers and submitted to the kernel for
3695 * sheduling on the ring. This function schedules the IB
3696 * on the gfx ring for execution by the GPU.
3697 */
3698void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3699{
3700 struct radeon_ring *ring = &rdev->ring[ib->ring];
3701 u32 header, control = INDIRECT_BUFFER_VALID;
3702
3703 if (ib->is_const_ib) {
3704 /* set switch buffer packet before const IB */
3705 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3706 radeon_ring_write(ring, 0);
3707
3708 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3709 } else {
3710 u32 next_rptr;
3711 if (ring->rptr_save_reg) {
3712 next_rptr = ring->wptr + 3 + 4;
3713 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3714 radeon_ring_write(ring, ((ring->rptr_save_reg -
3715 PACKET3_SET_UCONFIG_REG_START) >> 2));
3716 radeon_ring_write(ring, next_rptr);
3717 } else if (rdev->wb.enabled) {
3718 next_rptr = ring->wptr + 5 + 4;
3719 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3720 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3721 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3722 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3723 radeon_ring_write(ring, next_rptr);
3724 }
3725
3726 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3727 }
3728
3729 control |= ib->length_dw |
3730 (ib->vm ? (ib->vm->id << 24) : 0);
3731
3732 radeon_ring_write(ring, header);
3733 radeon_ring_write(ring,
3734#ifdef __BIG_ENDIAN
3735 (2 << 0) |
3736#endif
3737 (ib->gpu_addr & 0xFFFFFFFC));
3738 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3739 radeon_ring_write(ring, control);
3740}
3741
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3742/**
3743 * cik_ib_test - basic gfx ring IB test
3744 *
3745 * @rdev: radeon_device pointer
3746 * @ring: radeon_ring structure holding ring information
3747 *
3748 * Allocate an IB and execute it on the gfx ring (CIK).
3749 * Provides a basic gfx ring test to verify that IBs are working.
3750 * Returns 0 on success, error on failure.
3751 */
3752int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3753{
3754 struct radeon_ib ib;
3755 uint32_t scratch;
3756 uint32_t tmp = 0;
3757 unsigned i;
3758 int r;
3759
3760 r = radeon_scratch_get(rdev, &scratch);
3761 if (r) {
3762 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3763 return r;
3764 }
3765 WREG32(scratch, 0xCAFEDEAD);
3766 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3767 if (r) {
3768 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3769 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3770 return r;
3771 }
3772 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3773 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3774 ib.ptr[2] = 0xDEADBEEF;
3775 ib.length_dw = 3;
3776 r = radeon_ib_schedule(rdev, &ib, NULL);
3777 if (r) {
3778 radeon_scratch_free(rdev, scratch);
3779 radeon_ib_free(rdev, &ib);
3780 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3781 return r;
3782 }
3783 r = radeon_fence_wait(ib.fence, false);
3784 if (r) {
3785 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3786 radeon_scratch_free(rdev, scratch);
3787 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3788 return r;
3789 }
3790 for (i = 0; i < rdev->usec_timeout; i++) {
3791 tmp = RREG32(scratch);
3792 if (tmp == 0xDEADBEEF)
3793 break;
3794 DRM_UDELAY(1);
3795 }
3796 if (i < rdev->usec_timeout) {
3797 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3798 } else {
3799 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3800 scratch, tmp);
3801 r = -EINVAL;
3802 }
3803 radeon_scratch_free(rdev, scratch);
3804 radeon_ib_free(rdev, &ib);
3805 return r;
3806}
3807
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3808/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3809 * CP.
3810 * On CIK, gfx and compute now have independant command processors.
3811 *
3812 * GFX
3813 * Gfx consists of a single ring and can process both gfx jobs and
3814 * compute jobs. The gfx CP consists of three microengines (ME):
3815 * PFP - Pre-Fetch Parser
3816 * ME - Micro Engine
3817 * CE - Constant Engine
3818 * The PFP and ME make up what is considered the Drawing Engine (DE).
3819 * The CE is an asynchronous engine used for updating buffer desciptors
3820 * used by the DE so that they can be loaded into cache in parallel
3821 * while the DE is processing state update packets.
3822 *
3823 * Compute
3824 * The compute CP consists of two microengines (ME):
3825 * MEC1 - Compute MicroEngine 1
3826 * MEC2 - Compute MicroEngine 2
3827 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3828 * The queues are exposed to userspace and are programmed directly
3829 * by the compute runtime.
3830 */
3831/**
3832 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3833 *
3834 * @rdev: radeon_device pointer
3835 * @enable: enable or disable the MEs
3836 *
3837 * Halts or unhalts the gfx MEs.
3838 */
3839static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3840{
3841 if (enable)
3842 WREG32(CP_ME_CNTL, 0);
3843 else {
Alex Deucher50efa512014-01-27 11:26:33 -0500[diff] [blame]3844 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3845 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3846 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3847 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3848 }
3849 udelay(50);
3850}
3851
3852/**
3853 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3854 *
3855 * @rdev: radeon_device pointer
3856 *
3857 * Loads the gfx PFP, ME, and CE ucode.
3858 * Returns 0 for success, -EINVAL if the ucode is not available.
3859 */
3860static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3861{
3862 const __be32 *fw_data;
3863 int i;
3864
3865 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3866 return -EINVAL;
3867
3868 cik_cp_gfx_enable(rdev, false);
3869
3870 /* PFP */
3871 fw_data = (const __be32 *)rdev->pfp_fw->data;
3872 WREG32(CP_PFP_UCODE_ADDR, 0);
3873 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3874 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3875 WREG32(CP_PFP_UCODE_ADDR, 0);
3876
3877 /* CE */
3878 fw_data = (const __be32 *)rdev->ce_fw->data;
3879 WREG32(CP_CE_UCODE_ADDR, 0);
3880 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3881 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3882 WREG32(CP_CE_UCODE_ADDR, 0);
3883
3884 /* ME */
3885 fw_data = (const __be32 *)rdev->me_fw->data;
3886 WREG32(CP_ME_RAM_WADDR, 0);
3887 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3888 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3889 WREG32(CP_ME_RAM_WADDR, 0);
3890
3891 WREG32(CP_PFP_UCODE_ADDR, 0);
3892 WREG32(CP_CE_UCODE_ADDR, 0);
3893 WREG32(CP_ME_RAM_WADDR, 0);
3894 WREG32(CP_ME_RAM_RADDR, 0);
3895 return 0;
3896}
3897
3898/**
3899 * cik_cp_gfx_start - start the gfx ring
3900 *
3901 * @rdev: radeon_device pointer
3902 *
3903 * Enables the ring and loads the clear state context and other
3904 * packets required to init the ring.
3905 * Returns 0 for success, error for failure.
3906 */
3907static int cik_cp_gfx_start(struct radeon_device *rdev)
3908{
3909 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3910 int r, i;
3911
3912 /* init the CP */
3913 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3914 WREG32(CP_ENDIAN_SWAP, 0);
3915 WREG32(CP_DEVICE_ID, 1);
3916
3917 cik_cp_gfx_enable(rdev, true);
3918
3919 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3920 if (r) {
3921 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3922 return r;
3923 }
3924
3925 /* init the CE partitions. CE only used for gfx on CIK */
3926 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3927 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3928 radeon_ring_write(ring, 0xc000);
3929 radeon_ring_write(ring, 0xc000);
3930
3931 /* setup clear context state */
3932 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3933 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3934
3935 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3936 radeon_ring_write(ring, 0x80000000);
3937 radeon_ring_write(ring, 0x80000000);
3938
3939 for (i = 0; i < cik_default_size; i++)
3940 radeon_ring_write(ring, cik_default_state[i]);
3941
3942 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3943 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3944
3945 /* set clear context state */
3946 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3947 radeon_ring_write(ring, 0);
3948
3949 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3950 radeon_ring_write(ring, 0x00000316);
3951 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3952 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3953
3954 radeon_ring_unlock_commit(rdev, ring);
3955
3956 return 0;
3957}
3958
3959/**
3960 * cik_cp_gfx_fini - stop the gfx ring
3961 *
3962 * @rdev: radeon_device pointer
3963 *
3964 * Stop the gfx ring and tear down the driver ring
3965 * info.
3966 */
3967static void cik_cp_gfx_fini(struct radeon_device *rdev)
3968{
3969 cik_cp_gfx_enable(rdev, false);
3970 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3971}
3972
3973/**
3974 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3975 *
3976 * @rdev: radeon_device pointer
3977 *
3978 * Program the location and size of the gfx ring buffer
3979 * and test it to make sure it's working.
3980 * Returns 0 for success, error for failure.
3981 */
3982static int cik_cp_gfx_resume(struct radeon_device *rdev)
3983{
3984 struct radeon_ring *ring;
3985 u32 tmp;
3986 u32 rb_bufsz;
3987 u64 rb_addr;
3988 int r;
3989
3990 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3991 if (rdev->family != CHIP_HAWAII)
3992 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3993
3994 /* Set the write pointer delay */
3995 WREG32(CP_RB_WPTR_DELAY, 0);
3996
3997 /* set the RB to use vmid 0 */
3998 WREG32(CP_RB_VMID, 0);
3999
4000 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4001
4002 /* ring 0 - compute and gfx */
4003 /* Set ring buffer size */
4004 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4005 rb_bufsz = order_base_2(ring->ring_size / 8);
4006 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4007#ifdef __BIG_ENDIAN
4008 tmp |= BUF_SWAP_32BIT;
4009#endif
4010 WREG32(CP_RB0_CNTL, tmp);
4011
4012 /* Initialize the ring buffer's read and write pointers */
4013 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4014 ring->wptr = 0;
4015 WREG32(CP_RB0_WPTR, ring->wptr);
4016
4017 /* set the wb address wether it's enabled or not */
4018 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4019 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4020
4021 /* scratch register shadowing is no longer supported */
4022 WREG32(SCRATCH_UMSK, 0);
4023
4024 if (!rdev->wb.enabled)
4025 tmp |= RB_NO_UPDATE;
4026
4027 mdelay(1);
4028 WREG32(CP_RB0_CNTL, tmp);
4029
4030 rb_addr = ring->gpu_addr >> 8;
4031 WREG32(CP_RB0_BASE, rb_addr);
4032 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4033
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4034 /* start the ring */
4035 cik_cp_gfx_start(rdev);
4036 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4037 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4038 if (r) {
4039 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4040 return r;
4041 }
Alex Deucher50efa512014-01-27 11:26:33 -0500[diff] [blame]4042
4043 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4044 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4045
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4046 return 0;
4047}
4048
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4049u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4050 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4051{
4052 u32 rptr;
4053
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4054 if (rdev->wb.enabled)
4055 rptr = rdev->wb.wb[ring->rptr_offs/4];
4056 else
4057 rptr = RREG32(CP_RB0_RPTR);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4058
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4059 return rptr;
4060}
4061
4062u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4063 struct radeon_ring *ring)
4064{
4065 u32 wptr;
4066
4067 wptr = RREG32(CP_RB0_WPTR);
4068
4069 return wptr;
4070}
4071
4072void cik_gfx_set_wptr(struct radeon_device *rdev,
4073 struct radeon_ring *ring)
4074{
4075 WREG32(CP_RB0_WPTR, ring->wptr);
4076 (void)RREG32(CP_RB0_WPTR);
4077}
4078
4079u32 cik_compute_get_rptr(struct radeon_device *rdev,
4080 struct radeon_ring *ring)
4081{
4082 u32 rptr;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4083
4084 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4085 rptr = rdev->wb.wb[ring->rptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4086 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4087 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4088 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4089 rptr = RREG32(CP_HQD_PQ_RPTR);
4090 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4091 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4092 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4093
4094 return rptr;
4095}
4096
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4097u32 cik_compute_get_wptr(struct radeon_device *rdev,
4098 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4099{
4100 u32 wptr;
4101
4102 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4103 /* XXX check if swapping is necessary on BE */
4104 wptr = rdev->wb.wb[ring->wptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4105 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4106 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4107 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4108 wptr = RREG32(CP_HQD_PQ_WPTR);
4109 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4110 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4111 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4112
4113 return wptr;
4114}
4115
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4116void cik_compute_set_wptr(struct radeon_device *rdev,
4117 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4118{
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4119 /* XXX check if swapping is necessary on BE */
4120 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4121 WDOORBELL32(ring->doorbell_index, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4122}
4123
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4124/**
4125 * cik_cp_compute_enable - enable/disable the compute CP MEs
4126 *
4127 * @rdev: radeon_device pointer
4128 * @enable: enable or disable the MEs
4129 *
4130 * Halts or unhalts the compute MEs.
4131 */
4132static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4133{
4134 if (enable)
4135 WREG32(CP_MEC_CNTL, 0);
4136 else
4137 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4138 udelay(50);
4139}
4140
4141/**
4142 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4143 *
4144 * @rdev: radeon_device pointer
4145 *
4146 * Loads the compute MEC1&2 ucode.
4147 * Returns 0 for success, -EINVAL if the ucode is not available.
4148 */
4149static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4150{
4151 const __be32 *fw_data;
4152 int i;
4153
4154 if (!rdev->mec_fw)
4155 return -EINVAL;
4156
4157 cik_cp_compute_enable(rdev, false);
4158
4159 /* MEC1 */
4160 fw_data = (const __be32 *)rdev->mec_fw->data;
4161 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4162 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4163 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4164 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4165
4166 if (rdev->family == CHIP_KAVERI) {
4167 /* MEC2 */
4168 fw_data = (const __be32 *)rdev->mec_fw->data;
4169 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4170 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4171 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4172 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4173 }
4174
4175 return 0;
4176}
4177
4178/**
4179 * cik_cp_compute_start - start the compute queues
4180 *
4181 * @rdev: radeon_device pointer
4182 *
4183 * Enable the compute queues.
4184 * Returns 0 for success, error for failure.
4185 */
4186static int cik_cp_compute_start(struct radeon_device *rdev)
4187{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4188 cik_cp_compute_enable(rdev, true);
4189
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4190 return 0;
4191}
4192
4193/**
4194 * cik_cp_compute_fini - stop the compute queues
4195 *
4196 * @rdev: radeon_device pointer
4197 *
4198 * Stop the compute queues and tear down the driver queue
4199 * info.
4200 */
4201static void cik_cp_compute_fini(struct radeon_device *rdev)
4202{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4203 int i, idx, r;
4204
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4205 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4206
4207 for (i = 0; i < 2; i++) {
4208 if (i == 0)
4209 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4210 else
4211 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4212
4213 if (rdev->ring[idx].mqd_obj) {
4214 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4215 if (unlikely(r != 0))
4216 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4217
4218 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4219 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4220
4221 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4222 rdev->ring[idx].mqd_obj = NULL;
4223 }
4224 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4225}
4226
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4227static void cik_mec_fini(struct radeon_device *rdev)
4228{
4229 int r;
4230
4231 if (rdev->mec.hpd_eop_obj) {
4232 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4233 if (unlikely(r != 0))
4234 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4235 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4236 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4237
4238 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4239 rdev->mec.hpd_eop_obj = NULL;
4240 }
4241}
4242
4243#define MEC_HPD_SIZE 2048
4244
4245static int cik_mec_init(struct radeon_device *rdev)
4246{
4247 int r;
4248 u32 *hpd;
4249
4250 /*
4251 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4252 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4253 */
4254 if (rdev->family == CHIP_KAVERI)
4255 rdev->mec.num_mec = 2;
4256 else
4257 rdev->mec.num_mec = 1;
4258 rdev->mec.num_pipe = 4;
4259 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4260
4261 if (rdev->mec.hpd_eop_obj == NULL) {
4262 r = radeon_bo_create(rdev,
4263 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4264 PAGE_SIZE, true,
4265 RADEON_GEM_DOMAIN_GTT, NULL,
4266 &rdev->mec.hpd_eop_obj);
4267 if (r) {
4268 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4269 return r;
4270 }
4271 }
4272
4273 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4274 if (unlikely(r != 0)) {
4275 cik_mec_fini(rdev);
4276 return r;
4277 }
4278 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4279 &rdev->mec.hpd_eop_gpu_addr);
4280 if (r) {
4281 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4282 cik_mec_fini(rdev);
4283 return r;
4284 }
4285 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4286 if (r) {
4287 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4288 cik_mec_fini(rdev);
4289 return r;
4290 }
4291
4292 /* clear memory. Not sure if this is required or not */
4293 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4294
4295 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4296 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4297
4298 return 0;
4299}
4300
4301struct hqd_registers
4302{
4303 u32 cp_mqd_base_addr;
4304 u32 cp_mqd_base_addr_hi;
4305 u32 cp_hqd_active;
4306 u32 cp_hqd_vmid;
4307 u32 cp_hqd_persistent_state;
4308 u32 cp_hqd_pipe_priority;
4309 u32 cp_hqd_queue_priority;
4310 u32 cp_hqd_quantum;
4311 u32 cp_hqd_pq_base;
4312 u32 cp_hqd_pq_base_hi;
4313 u32 cp_hqd_pq_rptr;
4314 u32 cp_hqd_pq_rptr_report_addr;
4315 u32 cp_hqd_pq_rptr_report_addr_hi;
4316 u32 cp_hqd_pq_wptr_poll_addr;
4317 u32 cp_hqd_pq_wptr_poll_addr_hi;
4318 u32 cp_hqd_pq_doorbell_control;
4319 u32 cp_hqd_pq_wptr;
4320 u32 cp_hqd_pq_control;
4321 u32 cp_hqd_ib_base_addr;
4322 u32 cp_hqd_ib_base_addr_hi;
4323 u32 cp_hqd_ib_rptr;
4324 u32 cp_hqd_ib_control;
4325 u32 cp_hqd_iq_timer;
4326 u32 cp_hqd_iq_rptr;
4327 u32 cp_hqd_dequeue_request;
4328 u32 cp_hqd_dma_offload;
4329 u32 cp_hqd_sema_cmd;
4330 u32 cp_hqd_msg_type;
4331 u32 cp_hqd_atomic0_preop_lo;
4332 u32 cp_hqd_atomic0_preop_hi;
4333 u32 cp_hqd_atomic1_preop_lo;
4334 u32 cp_hqd_atomic1_preop_hi;
4335 u32 cp_hqd_hq_scheduler0;
4336 u32 cp_hqd_hq_scheduler1;
4337 u32 cp_mqd_control;
4338};
4339
4340struct bonaire_mqd
4341{
4342 u32 header;
4343 u32 dispatch_initiator;
4344 u32 dimensions[3];
4345 u32 start_idx[3];
4346 u32 num_threads[3];
4347 u32 pipeline_stat_enable;
4348 u32 perf_counter_enable;
4349 u32 pgm[2];
4350 u32 tba[2];
4351 u32 tma[2];
4352 u32 pgm_rsrc[2];
4353 u32 vmid;
4354 u32 resource_limits;
4355 u32 static_thread_mgmt01[2];
4356 u32 tmp_ring_size;
4357 u32 static_thread_mgmt23[2];
4358 u32 restart[3];
4359 u32 thread_trace_enable;
4360 u32 reserved1;
4361 u32 user_data[16];
4362 u32 vgtcs_invoke_count[2];
4363 struct hqd_registers queue_state;
4364 u32 dequeue_cntr;
4365 u32 interrupt_queue[64];
4366};
4367
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4368/**
4369 * cik_cp_compute_resume - setup the compute queue registers
4370 *
4371 * @rdev: radeon_device pointer
4372 *
4373 * Program the compute queues and test them to make sure they
4374 * are working.
4375 * Returns 0 for success, error for failure.
4376 */
4377static int cik_cp_compute_resume(struct radeon_device *rdev)
4378{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4379 int r, i, idx;
4380 u32 tmp;
4381 bool use_doorbell = true;
4382 u64 hqd_gpu_addr;
4383 u64 mqd_gpu_addr;
4384 u64 eop_gpu_addr;
4385 u64 wb_gpu_addr;
4386 u32 *buf;
4387 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4388
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4389 r = cik_cp_compute_start(rdev);
4390 if (r)
4391 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4392
4393 /* fix up chicken bits */
4394 tmp = RREG32(CP_CPF_DEBUG);
4395 tmp |= (1 << 23);
4396 WREG32(CP_CPF_DEBUG, tmp);
4397
4398 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4399 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4400 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4401 int me = (i < 4) ? 1 : 2;
4402 int pipe = (i < 4) ? i : (i - 4);
4403
4404 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4405
4406 cik_srbm_select(rdev, me, pipe, 0, 0);
4407
4408 /* write the EOP addr */
4409 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4410 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4411
4412 /* set the VMID assigned */
4413 WREG32(CP_HPD_EOP_VMID, 0);
4414
4415 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4416 tmp = RREG32(CP_HPD_EOP_CONTROL);
4417 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4418 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4419 WREG32(CP_HPD_EOP_CONTROL, tmp);
4420 }
4421 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4422 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4423
4424 /* init the queues. Just two for now. */
4425 for (i = 0; i < 2; i++) {
4426 if (i == 0)
4427 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4428 else
4429 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4430
4431 if (rdev->ring[idx].mqd_obj == NULL) {
4432 r = radeon_bo_create(rdev,
4433 sizeof(struct bonaire_mqd),
4434 PAGE_SIZE, true,
4435 RADEON_GEM_DOMAIN_GTT, NULL,
4436 &rdev->ring[idx].mqd_obj);
4437 if (r) {
4438 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4439 return r;
4440 }
4441 }
4442
4443 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4444 if (unlikely(r != 0)) {
4445 cik_cp_compute_fini(rdev);
4446 return r;
4447 }
4448 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4449 &mqd_gpu_addr);
4450 if (r) {
4451 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4452 cik_cp_compute_fini(rdev);
4453 return r;
4454 }
4455 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4456 if (r) {
4457 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4458 cik_cp_compute_fini(rdev);
4459 return r;
4460 }
4461
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4462 /* init the mqd struct */
4463 memset(buf, 0, sizeof(struct bonaire_mqd));
4464
4465 mqd = (struct bonaire_mqd *)buf;
4466 mqd->header = 0xC0310800;
4467 mqd->static_thread_mgmt01[0] = 0xffffffff;
4468 mqd->static_thread_mgmt01[1] = 0xffffffff;
4469 mqd->static_thread_mgmt23[0] = 0xffffffff;
4470 mqd->static_thread_mgmt23[1] = 0xffffffff;
4471
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4472 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4473 cik_srbm_select(rdev, rdev->ring[idx].me,
4474 rdev->ring[idx].pipe,
4475 rdev->ring[idx].queue, 0);
4476
4477 /* disable wptr polling */
4478 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4479 tmp &= ~WPTR_POLL_EN;
4480 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4481
4482 /* enable doorbell? */
4483 mqd->queue_state.cp_hqd_pq_doorbell_control =
4484 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4485 if (use_doorbell)
4486 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4487 else
4488 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4489 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4490 mqd->queue_state.cp_hqd_pq_doorbell_control);
4491
4492 /* disable the queue if it's active */
4493 mqd->queue_state.cp_hqd_dequeue_request = 0;
4494 mqd->queue_state.cp_hqd_pq_rptr = 0;
4495 mqd->queue_state.cp_hqd_pq_wptr= 0;
4496 if (RREG32(CP_HQD_ACTIVE) & 1) {
4497 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4498 for (i = 0; i < rdev->usec_timeout; i++) {
4499 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4500 break;
4501 udelay(1);
4502 }
4503 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4504 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4505 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4506 }
4507
4508 /* set the pointer to the MQD */
4509 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4510 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4511 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4512 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4513 /* set MQD vmid to 0 */
4514 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4515 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4516 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4517
4518 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4519 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4520 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4521 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4522 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4523 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4524
4525 /* set up the HQD, this is similar to CP_RB0_CNTL */
4526 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4527 mqd->queue_state.cp_hqd_pq_control &=
4528 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4529
4530 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4531 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4532 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4533 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4534#ifdef __BIG_ENDIAN
4535 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4536#endif
4537 mqd->queue_state.cp_hqd_pq_control &=
4538 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4539 mqd->queue_state.cp_hqd_pq_control |=
4540 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4541 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4542
4543 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4544 if (i == 0)
4545 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4546 else
4547 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4548 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4549 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4550 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4551 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4552 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4553
4554 /* set the wb address wether it's enabled or not */
4555 if (i == 0)
4556 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4557 else
4558 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4559 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4560 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4561 upper_32_bits(wb_gpu_addr) & 0xffff;
4562 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4563 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4564 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4565 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4566
4567 /* enable the doorbell if requested */
4568 if (use_doorbell) {
4569 mqd->queue_state.cp_hqd_pq_doorbell_control =
4570 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4571 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4572 mqd->queue_state.cp_hqd_pq_doorbell_control |=
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4573 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4574 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4575 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4576 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4577
4578 } else {
4579 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4580 }
4581 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4582 mqd->queue_state.cp_hqd_pq_doorbell_control);
4583
4584 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4585 rdev->ring[idx].wptr = 0;
4586 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4587 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
Christian Königff212f22014-02-18 14:52:33 +0100[diff] [blame^]4588 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4589
4590 /* set the vmid for the queue */
4591 mqd->queue_state.cp_hqd_vmid = 0;
4592 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4593
4594 /* activate the queue */
4595 mqd->queue_state.cp_hqd_active = 1;
4596 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4597
4598 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4599 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4600
4601 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4602 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4603
4604 rdev->ring[idx].ready = true;
4605 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4606 if (r)
4607 rdev->ring[idx].ready = false;
4608 }
4609
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4610 return 0;
4611}
4612
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4613static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4614{
4615 cik_cp_gfx_enable(rdev, enable);
4616 cik_cp_compute_enable(rdev, enable);
4617}
4618
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4619static int cik_cp_load_microcode(struct radeon_device *rdev)
4620{
4621 int r;
4622
4623 r = cik_cp_gfx_load_microcode(rdev);
4624 if (r)
4625 return r;
4626 r = cik_cp_compute_load_microcode(rdev);
4627 if (r)
4628 return r;
4629
4630 return 0;
4631}
4632
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4633static void cik_cp_fini(struct radeon_device *rdev)
4634{
4635 cik_cp_gfx_fini(rdev);
4636 cik_cp_compute_fini(rdev);
4637}
4638
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4639static int cik_cp_resume(struct radeon_device *rdev)
4640{
4641 int r;
4642
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4643 cik_enable_gui_idle_interrupt(rdev, false);
4644
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4645 r = cik_cp_load_microcode(rdev);
4646 if (r)
4647 return r;
4648
4649 r = cik_cp_gfx_resume(rdev);
4650 if (r)
4651 return r;
4652 r = cik_cp_compute_resume(rdev);
4653 if (r)
4654 return r;
4655
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4656 cik_enable_gui_idle_interrupt(rdev, true);
4657
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4658 return 0;
4659}
4660
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4661static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4662{
4663 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4664 RREG32(GRBM_STATUS));
4665 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4666 RREG32(GRBM_STATUS2));
4667 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4668 RREG32(GRBM_STATUS_SE0));
4669 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4670 RREG32(GRBM_STATUS_SE1));
4671 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4672 RREG32(GRBM_STATUS_SE2));
4673 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4674 RREG32(GRBM_STATUS_SE3));
4675 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4676 RREG32(SRBM_STATUS));
4677 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4678 RREG32(SRBM_STATUS2));
4679 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4680 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4681 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4682 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4683 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4684 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4685 RREG32(CP_STALLED_STAT1));
4686 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4687 RREG32(CP_STALLED_STAT2));
4688 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4689 RREG32(CP_STALLED_STAT3));
4690 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4691 RREG32(CP_CPF_BUSY_STAT));
4692 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4693 RREG32(CP_CPF_STALLED_STAT1));
4694 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4695 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4696 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4697 RREG32(CP_CPC_STALLED_STAT1));
4698 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4699}
4700
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4701/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4702 * cik_gpu_check_soft_reset - check which blocks are busy
4703 *
4704 * @rdev: radeon_device pointer
4705 *
4706 * Check which blocks are busy and return the relevant reset
4707 * mask to be used by cik_gpu_soft_reset().
4708 * Returns a mask of the blocks to be reset.
4709 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4710u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4711{
4712 u32 reset_mask = 0;
4713 u32 tmp;
4714
4715 /* GRBM_STATUS */
4716 tmp = RREG32(GRBM_STATUS);
4717 if (tmp & (PA_BUSY | SC_BUSY |
4718 BCI_BUSY | SX_BUSY |
4719 TA_BUSY | VGT_BUSY |
4720 DB_BUSY | CB_BUSY |
4721 GDS_BUSY | SPI_BUSY |
4722 IA_BUSY | IA_BUSY_NO_DMA))
4723 reset_mask |= RADEON_RESET_GFX;
4724
4725 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4726 reset_mask |= RADEON_RESET_CP;
4727
4728 /* GRBM_STATUS2 */
4729 tmp = RREG32(GRBM_STATUS2);
4730 if (tmp & RLC_BUSY)
4731 reset_mask |= RADEON_RESET_RLC;
4732
4733 /* SDMA0_STATUS_REG */
4734 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4735 if (!(tmp & SDMA_IDLE))
4736 reset_mask |= RADEON_RESET_DMA;
4737
4738 /* SDMA1_STATUS_REG */
4739 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4740 if (!(tmp & SDMA_IDLE))
4741 reset_mask |= RADEON_RESET_DMA1;
4742
4743 /* SRBM_STATUS2 */
4744 tmp = RREG32(SRBM_STATUS2);
4745 if (tmp & SDMA_BUSY)
4746 reset_mask |= RADEON_RESET_DMA;
4747
4748 if (tmp & SDMA1_BUSY)
4749 reset_mask |= RADEON_RESET_DMA1;
4750
4751 /* SRBM_STATUS */
4752 tmp = RREG32(SRBM_STATUS);
4753
4754 if (tmp & IH_BUSY)
4755 reset_mask |= RADEON_RESET_IH;
4756
4757 if (tmp & SEM_BUSY)
4758 reset_mask |= RADEON_RESET_SEM;
4759
4760 if (tmp & GRBM_RQ_PENDING)
4761 reset_mask |= RADEON_RESET_GRBM;
4762
4763 if (tmp & VMC_BUSY)
4764 reset_mask |= RADEON_RESET_VMC;
4765
4766 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4767 MCC_BUSY | MCD_BUSY))
4768 reset_mask |= RADEON_RESET_MC;
4769
4770 if (evergreen_is_display_hung(rdev))
4771 reset_mask |= RADEON_RESET_DISPLAY;
4772
4773 /* Skip MC reset as it's mostly likely not hung, just busy */
4774 if (reset_mask & RADEON_RESET_MC) {
4775 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4776 reset_mask &= ~RADEON_RESET_MC;
4777 }
4778
4779 return reset_mask;
4780}
4781
4782/**
4783 * cik_gpu_soft_reset - soft reset GPU
4784 *
4785 * @rdev: radeon_device pointer
4786 * @reset_mask: mask of which blocks to reset
4787 *
4788 * Soft reset the blocks specified in @reset_mask.
4789 */
4790static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4791{
4792 struct evergreen_mc_save save;
4793 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4794 u32 tmp;
4795
4796 if (reset_mask == 0)
4797 return;
4798
4799 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4800
4801 cik_print_gpu_status_regs(rdev);
4802 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4803 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4804 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4805 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4806
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4807 /* disable CG/PG */
4808 cik_fini_pg(rdev);
4809 cik_fini_cg(rdev);
4810
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4811 /* stop the rlc */
4812 cik_rlc_stop(rdev);
4813
4814 /* Disable GFX parsing/prefetching */
4815 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4816
4817 /* Disable MEC parsing/prefetching */
4818 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4819
4820 if (reset_mask & RADEON_RESET_DMA) {
4821 /* sdma0 */
4822 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4823 tmp |= SDMA_HALT;
4824 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4825 }
4826 if (reset_mask & RADEON_RESET_DMA1) {
4827 /* sdma1 */
4828 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4829 tmp |= SDMA_HALT;
4830 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4831 }
4832
4833 evergreen_mc_stop(rdev, &save);
4834 if (evergreen_mc_wait_for_idle(rdev)) {
4835 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4836 }
4837
4838 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4839 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4840
4841 if (reset_mask & RADEON_RESET_CP) {
4842 grbm_soft_reset |= SOFT_RESET_CP;
4843
4844 srbm_soft_reset |= SOFT_RESET_GRBM;
4845 }
4846
4847 if (reset_mask & RADEON_RESET_DMA)
4848 srbm_soft_reset |= SOFT_RESET_SDMA;
4849
4850 if (reset_mask & RADEON_RESET_DMA1)
4851 srbm_soft_reset |= SOFT_RESET_SDMA1;
4852
4853 if (reset_mask & RADEON_RESET_DISPLAY)
4854 srbm_soft_reset |= SOFT_RESET_DC;
4855
4856 if (reset_mask & RADEON_RESET_RLC)
4857 grbm_soft_reset |= SOFT_RESET_RLC;
4858
4859 if (reset_mask & RADEON_RESET_SEM)
4860 srbm_soft_reset |= SOFT_RESET_SEM;
4861
4862 if (reset_mask & RADEON_RESET_IH)
4863 srbm_soft_reset |= SOFT_RESET_IH;
4864
4865 if (reset_mask & RADEON_RESET_GRBM)
4866 srbm_soft_reset |= SOFT_RESET_GRBM;
4867
4868 if (reset_mask & RADEON_RESET_VMC)
4869 srbm_soft_reset |= SOFT_RESET_VMC;
4870
4871 if (!(rdev->flags & RADEON_IS_IGP)) {
4872 if (reset_mask & RADEON_RESET_MC)
4873 srbm_soft_reset |= SOFT_RESET_MC;
4874 }
4875
4876 if (grbm_soft_reset) {
4877 tmp = RREG32(GRBM_SOFT_RESET);
4878 tmp |= grbm_soft_reset;
4879 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4880 WREG32(GRBM_SOFT_RESET, tmp);
4881 tmp = RREG32(GRBM_SOFT_RESET);
4882
4883 udelay(50);
4884
4885 tmp &= ~grbm_soft_reset;
4886 WREG32(GRBM_SOFT_RESET, tmp);
4887 tmp = RREG32(GRBM_SOFT_RESET);
4888 }
4889
4890 if (srbm_soft_reset) {
4891 tmp = RREG32(SRBM_SOFT_RESET);
4892 tmp |= srbm_soft_reset;
4893 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4894 WREG32(SRBM_SOFT_RESET, tmp);
4895 tmp = RREG32(SRBM_SOFT_RESET);
4896
4897 udelay(50);
4898
4899 tmp &= ~srbm_soft_reset;
4900 WREG32(SRBM_SOFT_RESET, tmp);
4901 tmp = RREG32(SRBM_SOFT_RESET);
4902 }
4903
4904 /* Wait a little for things to settle down */
4905 udelay(50);
4906
4907 evergreen_mc_resume(rdev, &save);
4908 udelay(50);
4909
4910 cik_print_gpu_status_regs(rdev);
4911}
4912
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]4913struct kv_reset_save_regs {
4914 u32 gmcon_reng_execute;
4915 u32 gmcon_misc;
4916 u32 gmcon_misc3;
4917};
4918
4919static void kv_save_regs_for_reset(struct radeon_device *rdev,
4920 struct kv_reset_save_regs *save)
4921{
4922 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4923 save->gmcon_misc = RREG32(GMCON_MISC);
4924 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4925
4926 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4927 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4928 STCTRL_STUTTER_EN));
4929}
4930
4931static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4932 struct kv_reset_save_regs *save)
4933{
4934 int i;
4935
4936 WREG32(GMCON_PGFSM_WRITE, 0);
4937 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4938
4939 for (i = 0; i < 5; i++)
4940 WREG32(GMCON_PGFSM_WRITE, 0);
4941
4942 WREG32(GMCON_PGFSM_WRITE, 0);
4943 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4944
4945 for (i = 0; i < 5; i++)
4946 WREG32(GMCON_PGFSM_WRITE, 0);
4947
4948 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4949 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4950
4951 for (i = 0; i < 5; i++)
4952 WREG32(GMCON_PGFSM_WRITE, 0);
4953
4954 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4955 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4956
4957 for (i = 0; i < 5; i++)
4958 WREG32(GMCON_PGFSM_WRITE, 0);
4959
4960 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4961 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4962
4963 for (i = 0; i < 5; i++)
4964 WREG32(GMCON_PGFSM_WRITE, 0);
4965
4966 WREG32(GMCON_PGFSM_WRITE, 0);
4967 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
4968
4969 for (i = 0; i < 5; i++)
4970 WREG32(GMCON_PGFSM_WRITE, 0);
4971
4972 WREG32(GMCON_PGFSM_WRITE, 0x420000);
4973 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
4974
4975 for (i = 0; i < 5; i++)
4976 WREG32(GMCON_PGFSM_WRITE, 0);
4977
4978 WREG32(GMCON_PGFSM_WRITE, 0x120202);
4979 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
4980
4981 for (i = 0; i < 5; i++)
4982 WREG32(GMCON_PGFSM_WRITE, 0);
4983
4984 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
4985 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
4986
4987 for (i = 0; i < 5; i++)
4988 WREG32(GMCON_PGFSM_WRITE, 0);
4989
4990 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
4991 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
4992
4993 for (i = 0; i < 5; i++)
4994 WREG32(GMCON_PGFSM_WRITE, 0);
4995
4996 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
4997 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
4998
4999 WREG32(GMCON_MISC3, save->gmcon_misc3);
5000 WREG32(GMCON_MISC, save->gmcon_misc);
5001 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5002}
5003
5004static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5005{
5006 struct evergreen_mc_save save;
5007 struct kv_reset_save_regs kv_save = { 0 };
5008 u32 tmp, i;
5009
5010 dev_info(rdev->dev, "GPU pci config reset\n");
5011
5012 /* disable dpm? */
5013
5014 /* disable cg/pg */
5015 cik_fini_pg(rdev);
5016 cik_fini_cg(rdev);
5017
5018 /* Disable GFX parsing/prefetching */
5019 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5020
5021 /* Disable MEC parsing/prefetching */
5022 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5023
5024 /* sdma0 */
5025 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5026 tmp |= SDMA_HALT;
5027 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5028 /* sdma1 */
5029 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5030 tmp |= SDMA_HALT;
5031 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5032 /* XXX other engines? */
5033
5034 /* halt the rlc, disable cp internal ints */
5035 cik_rlc_stop(rdev);
5036
5037 udelay(50);
5038
5039 /* disable mem access */
5040 evergreen_mc_stop(rdev, &save);
5041 if (evergreen_mc_wait_for_idle(rdev)) {
5042 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5043 }
5044
5045 if (rdev->flags & RADEON_IS_IGP)
5046 kv_save_regs_for_reset(rdev, &kv_save);
5047
5048 /* disable BM */
5049 pci_clear_master(rdev->pdev);
5050 /* reset */
5051 radeon_pci_config_reset(rdev);
5052
5053 udelay(100);
5054
5055 /* wait for asic to come out of reset */
5056 for (i = 0; i < rdev->usec_timeout; i++) {
5057 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5058 break;
5059 udelay(1);
5060 }
5061
5062 /* does asic init need to be run first??? */
5063 if (rdev->flags & RADEON_IS_IGP)
5064 kv_restore_regs_for_reset(rdev, &kv_save);
5065}
5066
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5067/**
5068 * cik_asic_reset - soft reset GPU
5069 *
5070 * @rdev: radeon_device pointer
5071 *
5072 * Look up which blocks are hung and attempt
5073 * to reset them.
5074 * Returns 0 for success.
5075 */
5076int cik_asic_reset(struct radeon_device *rdev)
5077{
5078 u32 reset_mask;
5079
5080 reset_mask = cik_gpu_check_soft_reset(rdev);
5081
5082 if (reset_mask)
5083 r600_set_bios_scratch_engine_hung(rdev, true);
5084
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]5085 /* try soft reset */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5086 cik_gpu_soft_reset(rdev, reset_mask);
5087
5088 reset_mask = cik_gpu_check_soft_reset(rdev);
5089
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]5090 /* try pci config reset */
5091 if (reset_mask && radeon_hard_reset)
5092 cik_gpu_pci_config_reset(rdev);
5093
5094 reset_mask = cik_gpu_check_soft_reset(rdev);
5095
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5096 if (!reset_mask)
5097 r600_set_bios_scratch_engine_hung(rdev, false);
5098
5099 return 0;
5100}
5101
5102/**
5103 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5104 *
5105 * @rdev: radeon_device pointer
5106 * @ring: radeon_ring structure holding ring information
5107 *
5108 * Check if the 3D engine is locked up (CIK).
5109 * Returns true if the engine is locked, false if not.
5110 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5111bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5112{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5113 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5114
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5115 if (!(reset_mask & (RADEON_RESET_GFX |
5116 RADEON_RESET_COMPUTE |
5117 RADEON_RESET_CP))) {
Christian Königff212f22014-02-18 14:52:33 +0100[diff] [blame^]5118 radeon_ring_lockup_update(rdev, ring);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5119 return false;
5120 }
5121 /* force CP activities */
5122 radeon_ring_force_activity(rdev, ring);
5123 return radeon_ring_test_lockup(rdev, ring);
5124}
5125
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5126/* MC */
5127/**
5128 * cik_mc_program - program the GPU memory controller
5129 *
5130 * @rdev: radeon_device pointer
5131 *
5132 * Set the location of vram, gart, and AGP in the GPU's
5133 * physical address space (CIK).
5134 */
5135static void cik_mc_program(struct radeon_device *rdev)
5136{
5137 struct evergreen_mc_save save;
5138 u32 tmp;
5139 int i, j;
5140
5141 /* Initialize HDP */
5142 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5143 WREG32((0x2c14 + j), 0x00000000);
5144 WREG32((0x2c18 + j), 0x00000000);
5145 WREG32((0x2c1c + j), 0x00000000);
5146 WREG32((0x2c20 + j), 0x00000000);
5147 WREG32((0x2c24 + j), 0x00000000);
5148 }
5149 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5150
5151 evergreen_mc_stop(rdev, &save);
5152 if (radeon_mc_wait_for_idle(rdev)) {
5153 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5154 }
5155 /* Lockout access through VGA aperture*/
5156 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5157 /* Update configuration */
5158 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5159 rdev->mc.vram_start >> 12);
5160 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5161 rdev->mc.vram_end >> 12);
5162 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5163 rdev->vram_scratch.gpu_addr >> 12);
5164 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5165 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5166 WREG32(MC_VM_FB_LOCATION, tmp);
5167 /* XXX double check these! */
5168 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5169 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5170 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5171 WREG32(MC_VM_AGP_BASE, 0);
5172 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5173 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5174 if (radeon_mc_wait_for_idle(rdev)) {
5175 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5176 }
5177 evergreen_mc_resume(rdev, &save);
5178 /* we need to own VRAM, so turn off the VGA renderer here
5179 * to stop it overwriting our objects */
5180 rv515_vga_render_disable(rdev);
5181}
5182
5183/**
5184 * cik_mc_init - initialize the memory controller driver params
5185 *
5186 * @rdev: radeon_device pointer
5187 *
5188 * Look up the amount of vram, vram width, and decide how to place
5189 * vram and gart within the GPU's physical address space (CIK).
5190 * Returns 0 for success.
5191 */
5192static int cik_mc_init(struct radeon_device *rdev)
5193{
5194 u32 tmp;
5195 int chansize, numchan;
5196
5197 /* Get VRAM informations */
5198 rdev->mc.vram_is_ddr = true;
5199 tmp = RREG32(MC_ARB_RAMCFG);
5200 if (tmp & CHANSIZE_MASK) {
5201 chansize = 64;
5202 } else {
5203 chansize = 32;
5204 }
5205 tmp = RREG32(MC_SHARED_CHMAP);
5206 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5207 case 0:
5208 default:
5209 numchan = 1;
5210 break;
5211 case 1:
5212 numchan = 2;
5213 break;
5214 case 2:
5215 numchan = 4;
5216 break;
5217 case 3:
5218 numchan = 8;
5219 break;
5220 case 4:
5221 numchan = 3;
5222 break;
5223 case 5:
5224 numchan = 6;
5225 break;
5226 case 6:
5227 numchan = 10;
5228 break;
5229 case 7:
5230 numchan = 12;
5231 break;
5232 case 8:
5233 numchan = 16;
5234 break;
5235 }
5236 rdev->mc.vram_width = numchan * chansize;
5237 /* Could aper size report 0 ? */
5238 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5239 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5240 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5241 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5242 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5243 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5244 si_vram_gtt_location(rdev, &rdev->mc);
5245 radeon_update_bandwidth_info(rdev);
5246
5247 return 0;
5248}
5249
5250/*
5251 * GART
5252 * VMID 0 is the physical GPU addresses as used by the kernel.
5253 * VMIDs 1-15 are used for userspace clients and are handled
5254 * by the radeon vm/hsa code.
5255 */
5256/**
5257 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5258 *
5259 * @rdev: radeon_device pointer
5260 *
5261 * Flush the TLB for the VMID 0 page table (CIK).
5262 */
5263void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5264{
5265 /* flush hdp cache */
5266 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5267
5268 /* bits 0-15 are the VM contexts0-15 */
5269 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5270}
5271
5272/**
5273 * cik_pcie_gart_enable - gart enable
5274 *
5275 * @rdev: radeon_device pointer
5276 *
5277 * This sets up the TLBs, programs the page tables for VMID0,
5278 * sets up the hw for VMIDs 1-15 which are allocated on
5279 * demand, and sets up the global locations for the LDS, GDS,
5280 * and GPUVM for FSA64 clients (CIK).
5281 * Returns 0 for success, errors for failure.
5282 */
5283static int cik_pcie_gart_enable(struct radeon_device *rdev)
5284{
5285 int r, i;
5286
5287 if (rdev->gart.robj == NULL) {
5288 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5289 return -EINVAL;
5290 }
5291 r = radeon_gart_table_vram_pin(rdev);
5292 if (r)
5293 return r;
5294 radeon_gart_restore(rdev);
5295 /* Setup TLB control */
5296 WREG32(MC_VM_MX_L1_TLB_CNTL,
5297 (0xA << 7) |
5298 ENABLE_L1_TLB |
5299 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5300 ENABLE_ADVANCED_DRIVER_MODEL |
5301 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5302 /* Setup L2 cache */
5303 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5304 ENABLE_L2_FRAGMENT_PROCESSING |
5305 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5306 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5307 EFFECTIVE_L2_QUEUE_SIZE(7) |
5308 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5309 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5310 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5311 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5312 /* setup context0 */
5313 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5314 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5315 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5316 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5317 (u32)(rdev->dummy_page.addr >> 12));
5318 WREG32(VM_CONTEXT0_CNTL2, 0);
5319 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5320 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5321
5322 WREG32(0x15D4, 0);
5323 WREG32(0x15D8, 0);
5324 WREG32(0x15DC, 0);
5325
5326 /* empty context1-15 */
5327 /* FIXME start with 4G, once using 2 level pt switch to full
5328 * vm size space
5329 */
5330 /* set vm size, must be a multiple of 4 */
5331 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5332 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5333 for (i = 1; i < 16; i++) {
5334 if (i < 8)
5335 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5336 rdev->gart.table_addr >> 12);
5337 else
5338 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5339 rdev->gart.table_addr >> 12);
5340 }
5341
5342 /* enable context1-15 */
5343 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5344 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5345 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5346 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5347 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5348 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5349 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5350 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5351 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5352 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5353 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5354 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5355 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5356 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5357 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5358 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5359
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5360 if (rdev->family == CHIP_KAVERI) {
5361 u32 tmp = RREG32(CHUB_CONTROL);
5362 tmp &= ~BYPASS_VM;
5363 WREG32(CHUB_CONTROL, tmp);
5364 }
5365
5366 /* XXX SH_MEM regs */
5367 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5368 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5369 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5370 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5371 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5372 WREG32(SH_MEM_CONFIG, 0);
5373 WREG32(SH_MEM_APE1_BASE, 1);
5374 WREG32(SH_MEM_APE1_LIMIT, 0);
5375 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5376 /* SDMA GFX */
5377 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5378 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5379 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5380 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5381 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5382 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5383 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5384 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5385
5386 cik_pcie_gart_tlb_flush(rdev);
5387 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5388 (unsigned)(rdev->mc.gtt_size >> 20),
5389 (unsigned long long)rdev->gart.table_addr);
5390 rdev->gart.ready = true;
5391 return 0;
5392}
5393
5394/**
5395 * cik_pcie_gart_disable - gart disable
5396 *
5397 * @rdev: radeon_device pointer
5398 *
5399 * This disables all VM page table (CIK).
5400 */
5401static void cik_pcie_gart_disable(struct radeon_device *rdev)
5402{
5403 /* Disable all tables */
5404 WREG32(VM_CONTEXT0_CNTL, 0);
5405 WREG32(VM_CONTEXT1_CNTL, 0);
5406 /* Setup TLB control */
5407 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5408 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5409 /* Setup L2 cache */
5410 WREG32(VM_L2_CNTL,
5411 ENABLE_L2_FRAGMENT_PROCESSING |
5412 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5413 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5414 EFFECTIVE_L2_QUEUE_SIZE(7) |
5415 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5416 WREG32(VM_L2_CNTL2, 0);
5417 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5418 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5419 radeon_gart_table_vram_unpin(rdev);
5420}
5421
5422/**
5423 * cik_pcie_gart_fini - vm fini callback
5424 *
5425 * @rdev: radeon_device pointer
5426 *
5427 * Tears down the driver GART/VM setup (CIK).
5428 */
5429static void cik_pcie_gart_fini(struct radeon_device *rdev)
5430{
5431 cik_pcie_gart_disable(rdev);
5432 radeon_gart_table_vram_free(rdev);
5433 radeon_gart_fini(rdev);
5434}
5435
5436/* vm parser */
5437/**
5438 * cik_ib_parse - vm ib_parse callback
5439 *
5440 * @rdev: radeon_device pointer
5441 * @ib: indirect buffer pointer
5442 *
5443 * CIK uses hw IB checking so this is a nop (CIK).
5444 */
5445int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5446{
5447 return 0;
5448}
5449
5450/*
5451 * vm
5452 * VMID 0 is the physical GPU addresses as used by the kernel.
5453 * VMIDs 1-15 are used for userspace clients and are handled
5454 * by the radeon vm/hsa code.
5455 */
5456/**
5457 * cik_vm_init - cik vm init callback
5458 *
5459 * @rdev: radeon_device pointer
5460 *
5461 * Inits cik specific vm parameters (number of VMs, base of vram for
5462 * VMIDs 1-15) (CIK).
5463 * Returns 0 for success.
5464 */
5465int cik_vm_init(struct radeon_device *rdev)
5466{
5467 /* number of VMs */
5468 rdev->vm_manager.nvm = 16;
5469 /* base offset of vram pages */
5470 if (rdev->flags & RADEON_IS_IGP) {
5471 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5472 tmp <<= 22;
5473 rdev->vm_manager.vram_base_offset = tmp;
5474 } else
5475 rdev->vm_manager.vram_base_offset = 0;
5476
5477 return 0;
5478}
5479
5480/**
5481 * cik_vm_fini - cik vm fini callback
5482 *
5483 * @rdev: radeon_device pointer
5484 *
5485 * Tear down any asic specific VM setup (CIK).
5486 */
5487void cik_vm_fini(struct radeon_device *rdev)
5488{
5489}
5490
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5491/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5492 * cik_vm_decode_fault - print human readable fault info
5493 *
5494 * @rdev: radeon_device pointer
5495 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5496 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5497 *
5498 * Print human readable fault information (CIK).
5499 */
5500static void cik_vm_decode_fault(struct radeon_device *rdev,
5501 u32 status, u32 addr, u32 mc_client)
5502{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5503 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5504 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5505 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5506 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5507 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5508
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5509 if (rdev->family == CHIP_HAWAII)
5510 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5511 else
5512 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5513
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5514 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5515 protections, vmid, addr,
5516 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5517 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5518}
5519
5520/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5521 * cik_vm_flush - cik vm flush using the CP
5522 *
5523 * @rdev: radeon_device pointer
5524 *
5525 * Update the page table base and flush the VM TLB
5526 * using the CP (CIK).
5527 */
5528void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5529{
5530 struct radeon_ring *ring = &rdev->ring[ridx];
5531
5532 if (vm == NULL)
5533 return;
5534
5535 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5536 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5537 WRITE_DATA_DST_SEL(0)));
5538 if (vm->id < 8) {
5539 radeon_ring_write(ring,
5540 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5541 } else {
5542 radeon_ring_write(ring,
5543 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5544 }
5545 radeon_ring_write(ring, 0);
5546 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5547
5548 /* update SH_MEM_* regs */
5549 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5550 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5551 WRITE_DATA_DST_SEL(0)));
5552 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5553 radeon_ring_write(ring, 0);
5554 radeon_ring_write(ring, VMID(vm->id));
5555
5556 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5557 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5558 WRITE_DATA_DST_SEL(0)));
5559 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5560 radeon_ring_write(ring, 0);
5561
5562 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5563 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5564 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5565 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5566
5567 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5568 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5569 WRITE_DATA_DST_SEL(0)));
5570 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5571 radeon_ring_write(ring, 0);
5572 radeon_ring_write(ring, VMID(0));
5573
5574 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]5575 cik_hdp_flush_cp_ring_emit(rdev, ridx);
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5576
5577 /* bits 0-15 are the VM contexts0-15 */
5578 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5579 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5580 WRITE_DATA_DST_SEL(0)));
5581 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5582 radeon_ring_write(ring, 0);
5583 radeon_ring_write(ring, 1 << vm->id);
5584
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5585 /* compute doesn't have PFP */
5586 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5587 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5588 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5589 radeon_ring_write(ring, 0x0);
5590 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5591}
5592
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5593/*
5594 * RLC
5595 * The RLC is a multi-purpose microengine that handles a
5596 * variety of functions, the most important of which is
5597 * the interrupt controller.
5598 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5599static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5600 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5601{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5602 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5603
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5604 if (enable)
5605 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5606 else
5607 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5608 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5609}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5610
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5611static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5612{
5613 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5614
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5615 tmp = RREG32(RLC_LB_CNTL);
5616 if (enable)
5617 tmp |= LOAD_BALANCE_ENABLE;
5618 else
5619 tmp &= ~LOAD_BALANCE_ENABLE;
5620 WREG32(RLC_LB_CNTL, tmp);
5621}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5622
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5623static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5624{
5625 u32 i, j, k;
5626 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5627
5628 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5629 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5630 cik_select_se_sh(rdev, i, j);
5631 for (k = 0; k < rdev->usec_timeout; k++) {
5632 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5633 break;
5634 udelay(1);
5635 }
5636 }
5637 }
5638 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5639
5640 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5641 for (k = 0; k < rdev->usec_timeout; k++) {
5642 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5643 break;
5644 udelay(1);
5645 }
5646}
5647
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5648static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5649{
5650 u32 tmp;
5651
5652 tmp = RREG32(RLC_CNTL);
5653 if (tmp != rlc)
5654 WREG32(RLC_CNTL, rlc);
5655}
5656
5657static u32 cik_halt_rlc(struct radeon_device *rdev)
5658{
5659 u32 data, orig;
5660
5661 orig = data = RREG32(RLC_CNTL);
5662
5663 if (data & RLC_ENABLE) {
5664 u32 i;
5665
5666 data &= ~RLC_ENABLE;
5667 WREG32(RLC_CNTL, data);
5668
5669 for (i = 0; i < rdev->usec_timeout; i++) {
5670 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5671 break;
5672 udelay(1);
5673 }
5674
5675 cik_wait_for_rlc_serdes(rdev);
5676 }
5677
5678 return orig;
5679}
5680
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5681void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5682{
5683 u32 tmp, i, mask;
5684
5685 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5686 WREG32(RLC_GPR_REG2, tmp);
5687
5688 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5689 for (i = 0; i < rdev->usec_timeout; i++) {
5690 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5691 break;
5692 udelay(1);
5693 }
5694
5695 for (i = 0; i < rdev->usec_timeout; i++) {
5696 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5697 break;
5698 udelay(1);
5699 }
5700}
5701
5702void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5703{
5704 u32 tmp;
5705
5706 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5707 WREG32(RLC_GPR_REG2, tmp);
5708}
5709
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5710/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5711 * cik_rlc_stop - stop the RLC ME
5712 *
5713 * @rdev: radeon_device pointer
5714 *
5715 * Halt the RLC ME (MicroEngine) (CIK).
5716 */
5717static void cik_rlc_stop(struct radeon_device *rdev)
5718{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5719 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5720
5721 cik_enable_gui_idle_interrupt(rdev, false);
5722
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5723 cik_wait_for_rlc_serdes(rdev);
5724}
5725
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5726/**
5727 * cik_rlc_start - start the RLC ME
5728 *
5729 * @rdev: radeon_device pointer
5730 *
5731 * Unhalt the RLC ME (MicroEngine) (CIK).
5732 */
5733static void cik_rlc_start(struct radeon_device *rdev)
5734{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5735 WREG32(RLC_CNTL, RLC_ENABLE);
5736
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5737 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5738
5739 udelay(50);
5740}
5741
5742/**
5743 * cik_rlc_resume - setup the RLC hw
5744 *
5745 * @rdev: radeon_device pointer
5746 *
5747 * Initialize the RLC registers, load the ucode,
5748 * and start the RLC (CIK).
5749 * Returns 0 for success, -EINVAL if the ucode is not available.
5750 */
5751static int cik_rlc_resume(struct radeon_device *rdev)
5752{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5753 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5754 const __be32 *fw_data;
5755
5756 if (!rdev->rlc_fw)
5757 return -EINVAL;
5758
5759 switch (rdev->family) {
5760 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5761 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5762 default:
5763 size = BONAIRE_RLC_UCODE_SIZE;
5764 break;
5765 case CHIP_KAVERI:
5766 size = KV_RLC_UCODE_SIZE;
5767 break;
5768 case CHIP_KABINI:
5769 size = KB_RLC_UCODE_SIZE;
5770 break;
5771 }
5772
5773 cik_rlc_stop(rdev);
5774
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5775 /* disable CG */
5776 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5777 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5778
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5779 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5780
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5781 cik_init_pg(rdev);
5782
5783 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5784
5785 WREG32(RLC_LB_CNTR_INIT, 0);
5786 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5787
5788 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5789 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5790 WREG32(RLC_LB_PARAMS, 0x00600408);
5791 WREG32(RLC_LB_CNTL, 0x80000004);
5792
5793 WREG32(RLC_MC_CNTL, 0);
5794 WREG32(RLC_UCODE_CNTL, 0);
5795
5796 fw_data = (const __be32 *)rdev->rlc_fw->data;
5797 WREG32(RLC_GPM_UCODE_ADDR, 0);
5798 for (i = 0; i < size; i++)
5799 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5800 WREG32(RLC_GPM_UCODE_ADDR, 0);
5801
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5802 /* XXX - find out what chips support lbpw */
5803 cik_enable_lbpw(rdev, false);
5804
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5805 if (rdev->family == CHIP_BONAIRE)
5806 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5807
5808 cik_rlc_start(rdev);
5809
5810 return 0;
5811}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5812
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5813static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5814{
5815 u32 data, orig, tmp, tmp2;
5816
5817 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5818
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5819 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5820 cik_enable_gui_idle_interrupt(rdev, true);
5821
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5822 tmp = cik_halt_rlc(rdev);
5823
5824 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5825 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5826 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5827 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5828 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5829
5830 cik_update_rlc(rdev, tmp);
5831
5832 data |= CGCG_EN | CGLS_EN;
5833 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5834 cik_enable_gui_idle_interrupt(rdev, false);
5835
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5836 RREG32(CB_CGTT_SCLK_CTRL);
5837 RREG32(CB_CGTT_SCLK_CTRL);
5838 RREG32(CB_CGTT_SCLK_CTRL);
5839 RREG32(CB_CGTT_SCLK_CTRL);
5840
5841 data &= ~(CGCG_EN | CGLS_EN);
5842 }
5843
5844 if (orig != data)
5845 WREG32(RLC_CGCG_CGLS_CTRL, data);
5846
5847}
5848
5849static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5850{
5851 u32 data, orig, tmp = 0;
5852
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5853 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5854 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5855 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5856 orig = data = RREG32(CP_MEM_SLP_CNTL);
5857 data |= CP_MEM_LS_EN;
5858 if (orig != data)
5859 WREG32(CP_MEM_SLP_CNTL, data);
5860 }
5861 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5862
5863 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5864 data &= 0xfffffffd;
5865 if (orig != data)
5866 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5867
5868 tmp = cik_halt_rlc(rdev);
5869
5870 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5871 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5872 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5873 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5874 WREG32(RLC_SERDES_WR_CTRL, data);
5875
5876 cik_update_rlc(rdev, tmp);
5877
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5878 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5879 orig = data = RREG32(CGTS_SM_CTRL_REG);
5880 data &= ~SM_MODE_MASK;
5881 data |= SM_MODE(0x2);
5882 data |= SM_MODE_ENABLE;
5883 data &= ~CGTS_OVERRIDE;
5884 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5885 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5886 data &= ~CGTS_LS_OVERRIDE;
5887 data &= ~ON_MONITOR_ADD_MASK;
5888 data |= ON_MONITOR_ADD_EN;
5889 data |= ON_MONITOR_ADD(0x96);
5890 if (orig != data)
5891 WREG32(CGTS_SM_CTRL_REG, data);
5892 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5893 } else {
5894 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5895 data |= 0x00000002;
5896 if (orig != data)
5897 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5898
5899 data = RREG32(RLC_MEM_SLP_CNTL);
5900 if (data & RLC_MEM_LS_EN) {
5901 data &= ~RLC_MEM_LS_EN;
5902 WREG32(RLC_MEM_SLP_CNTL, data);
5903 }
5904
5905 data = RREG32(CP_MEM_SLP_CNTL);
5906 if (data & CP_MEM_LS_EN) {
5907 data &= ~CP_MEM_LS_EN;
5908 WREG32(CP_MEM_SLP_CNTL, data);
5909 }
5910
5911 orig = data = RREG32(CGTS_SM_CTRL_REG);
5912 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5913 if (orig != data)
5914 WREG32(CGTS_SM_CTRL_REG, data);
5915
5916 tmp = cik_halt_rlc(rdev);
5917
5918 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5919 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5920 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5921 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5922 WREG32(RLC_SERDES_WR_CTRL, data);
5923
5924 cik_update_rlc(rdev, tmp);
5925 }
5926}
5927
5928static const u32 mc_cg_registers[] =
5929{
5930 MC_HUB_MISC_HUB_CG,
5931 MC_HUB_MISC_SIP_CG,
5932 MC_HUB_MISC_VM_CG,
5933 MC_XPB_CLK_GAT,
5934 ATC_MISC_CG,
5935 MC_CITF_MISC_WR_CG,
5936 MC_CITF_MISC_RD_CG,
5937 MC_CITF_MISC_VM_CG,
5938 VM_L2_CG,
5939};
5940
5941static void cik_enable_mc_ls(struct radeon_device *rdev,
5942 bool enable)
5943{
5944 int i;
5945 u32 orig, data;
5946
5947 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5948 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5949 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5950 data |= MC_LS_ENABLE;
5951 else
5952 data &= ~MC_LS_ENABLE;
5953 if (data != orig)
5954 WREG32(mc_cg_registers[i], data);
5955 }
5956}
5957
5958static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5959 bool enable)
5960{
5961 int i;
5962 u32 orig, data;
5963
5964 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5965 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5966 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5967 data |= MC_CG_ENABLE;
5968 else
5969 data &= ~MC_CG_ENABLE;
5970 if (data != orig)
5971 WREG32(mc_cg_registers[i], data);
5972 }
5973}
5974
5975static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5976 bool enable)
5977{
5978 u32 orig, data;
5979
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5980 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5981 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5982 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5983 } else {
5984 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5985 data |= 0xff000000;
5986 if (data != orig)
5987 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5988
5989 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5990 data |= 0xff000000;
5991 if (data != orig)
5992 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5993 }
5994}
5995
5996static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5997 bool enable)
5998{
5999 u32 orig, data;
6000
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6001 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6002 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6003 data |= 0x100;
6004 if (orig != data)
6005 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6006
6007 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6008 data |= 0x100;
6009 if (orig != data)
6010 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6011 } else {
6012 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6013 data &= ~0x100;
6014 if (orig != data)
6015 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6016
6017 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6018 data &= ~0x100;
6019 if (orig != data)
6020 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6021 }
6022}
6023
6024static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6025 bool enable)
6026{
6027 u32 orig, data;
6028
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6029 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6030 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6031 data = 0xfff;
6032 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6033
6034 orig = data = RREG32(UVD_CGC_CTRL);
6035 data |= DCM;
6036 if (orig != data)
6037 WREG32(UVD_CGC_CTRL, data);
6038 } else {
6039 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6040 data &= ~0xfff;
6041 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6042
6043 orig = data = RREG32(UVD_CGC_CTRL);
6044 data &= ~DCM;
6045 if (orig != data)
6046 WREG32(UVD_CGC_CTRL, data);
6047 }
6048}
6049
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6050static void cik_enable_bif_mgls(struct radeon_device *rdev,
6051 bool enable)
6052{
6053 u32 orig, data;
6054
6055 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6056
6057 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6058 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6059 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6060 else
6061 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6062 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6063
6064 if (orig != data)
6065 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6066}
6067
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6068static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6069 bool enable)
6070{
6071 u32 orig, data;
6072
6073 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6074
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6075 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6076 data &= ~CLOCK_GATING_DIS;
6077 else
6078 data |= CLOCK_GATING_DIS;
6079
6080 if (orig != data)
6081 WREG32(HDP_HOST_PATH_CNTL, data);
6082}
6083
6084static void cik_enable_hdp_ls(struct radeon_device *rdev,
6085 bool enable)
6086{
6087 u32 orig, data;
6088
6089 orig = data = RREG32(HDP_MEM_POWER_LS);
6090
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6091 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6092 data |= HDP_LS_ENABLE;
6093 else
6094 data &= ~HDP_LS_ENABLE;
6095
6096 if (orig != data)
6097 WREG32(HDP_MEM_POWER_LS, data);
6098}
6099
6100void cik_update_cg(struct radeon_device *rdev,
6101 u32 block, bool enable)
6102{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6103
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6104 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6105 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6106 /* order matters! */
6107 if (enable) {
6108 cik_enable_mgcg(rdev, true);
6109 cik_enable_cgcg(rdev, true);
6110 } else {
6111 cik_enable_cgcg(rdev, false);
6112 cik_enable_mgcg(rdev, false);
6113 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6114 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6115 }
6116
6117 if (block & RADEON_CG_BLOCK_MC) {
6118 if (!(rdev->flags & RADEON_IS_IGP)) {
6119 cik_enable_mc_mgcg(rdev, enable);
6120 cik_enable_mc_ls(rdev, enable);
6121 }
6122 }
6123
6124 if (block & RADEON_CG_BLOCK_SDMA) {
6125 cik_enable_sdma_mgcg(rdev, enable);
6126 cik_enable_sdma_mgls(rdev, enable);
6127 }
6128
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6129 if (block & RADEON_CG_BLOCK_BIF) {
6130 cik_enable_bif_mgls(rdev, enable);
6131 }
6132
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6133 if (block & RADEON_CG_BLOCK_UVD) {
6134 if (rdev->has_uvd)
6135 cik_enable_uvd_mgcg(rdev, enable);
6136 }
6137
6138 if (block & RADEON_CG_BLOCK_HDP) {
6139 cik_enable_hdp_mgcg(rdev, enable);
6140 cik_enable_hdp_ls(rdev, enable);
6141 }
Alex Deuchera1d6f972013-09-06 12:33:04 -0400[diff] [blame]6142
6143 if (block & RADEON_CG_BLOCK_VCE) {
6144 vce_v2_0_enable_mgcg(rdev, enable);
6145 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6146}
6147
6148static void cik_init_cg(struct radeon_device *rdev)
6149{
6150
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]6151 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6152
6153 if (rdev->has_uvd)
6154 si_init_uvd_internal_cg(rdev);
6155
6156 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6157 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6158 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6159 RADEON_CG_BLOCK_UVD |
6160 RADEON_CG_BLOCK_HDP), true);
6161}
6162
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6163static void cik_fini_cg(struct radeon_device *rdev)
6164{
6165 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6166 RADEON_CG_BLOCK_SDMA |
6167 RADEON_CG_BLOCK_BIF |
6168 RADEON_CG_BLOCK_UVD |
6169 RADEON_CG_BLOCK_HDP), false);
6170
6171 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6172}
6173
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6174static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6175 bool enable)
6176{
6177 u32 data, orig;
6178
6179 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6180 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6181 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6182 else
6183 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6184 if (orig != data)
6185 WREG32(RLC_PG_CNTL, data);
6186}
6187
6188static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6189 bool enable)
6190{
6191 u32 data, orig;
6192
6193 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6194 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6195 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6196 else
6197 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6198 if (orig != data)
6199 WREG32(RLC_PG_CNTL, data);
6200}
6201
6202static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6203{
6204 u32 data, orig;
6205
6206 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6207 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6208 data &= ~DISABLE_CP_PG;
6209 else
6210 data |= DISABLE_CP_PG;
6211 if (orig != data)
6212 WREG32(RLC_PG_CNTL, data);
6213}
6214
6215static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6216{
6217 u32 data, orig;
6218
6219 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6220 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6221 data &= ~DISABLE_GDS_PG;
6222 else
6223 data |= DISABLE_GDS_PG;
6224 if (orig != data)
6225 WREG32(RLC_PG_CNTL, data);
6226}
6227
6228#define CP_ME_TABLE_SIZE 96
6229#define CP_ME_TABLE_OFFSET 2048
6230#define CP_MEC_TABLE_OFFSET 4096
6231
6232void cik_init_cp_pg_table(struct radeon_device *rdev)
6233{
6234 const __be32 *fw_data;
6235 volatile u32 *dst_ptr;
6236 int me, i, max_me = 4;
6237 u32 bo_offset = 0;
6238 u32 table_offset;
6239
6240 if (rdev->family == CHIP_KAVERI)
6241 max_me = 5;
6242
6243 if (rdev->rlc.cp_table_ptr == NULL)
6244 return;
6245
6246 /* write the cp table buffer */
6247 dst_ptr = rdev->rlc.cp_table_ptr;
6248 for (me = 0; me < max_me; me++) {
6249 if (me == 0) {
6250 fw_data = (const __be32 *)rdev->ce_fw->data;
6251 table_offset = CP_ME_TABLE_OFFSET;
6252 } else if (me == 1) {
6253 fw_data = (const __be32 *)rdev->pfp_fw->data;
6254 table_offset = CP_ME_TABLE_OFFSET;
6255 } else if (me == 2) {
6256 fw_data = (const __be32 *)rdev->me_fw->data;
6257 table_offset = CP_ME_TABLE_OFFSET;
6258 } else {
6259 fw_data = (const __be32 *)rdev->mec_fw->data;
6260 table_offset = CP_MEC_TABLE_OFFSET;
6261 }
6262
6263 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6264 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6265 }
6266 bo_offset += CP_ME_TABLE_SIZE;
6267 }
6268}
6269
6270static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6271 bool enable)
6272{
6273 u32 data, orig;
6274
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6275 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6276 orig = data = RREG32(RLC_PG_CNTL);
6277 data |= GFX_PG_ENABLE;
6278 if (orig != data)
6279 WREG32(RLC_PG_CNTL, data);
6280
6281 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6282 data |= AUTO_PG_EN;
6283 if (orig != data)
6284 WREG32(RLC_AUTO_PG_CTRL, data);
6285 } else {
6286 orig = data = RREG32(RLC_PG_CNTL);
6287 data &= ~GFX_PG_ENABLE;
6288 if (orig != data)
6289 WREG32(RLC_PG_CNTL, data);
6290
6291 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6292 data &= ~AUTO_PG_EN;
6293 if (orig != data)
6294 WREG32(RLC_AUTO_PG_CTRL, data);
6295
6296 data = RREG32(DB_RENDER_CONTROL);
6297 }
6298}
6299
6300static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6301{
6302 u32 mask = 0, tmp, tmp1;
6303 int i;
6304
6305 cik_select_se_sh(rdev, se, sh);
6306 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6307 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6308 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6309
6310 tmp &= 0xffff0000;
6311
6312 tmp |= tmp1;
6313 tmp >>= 16;
6314
6315 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6316 mask <<= 1;
6317 mask |= 1;
6318 }
6319
6320 return (~tmp) & mask;
6321}
6322
6323static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6324{
6325 u32 i, j, k, active_cu_number = 0;
6326 u32 mask, counter, cu_bitmap;
6327 u32 tmp = 0;
6328
6329 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6330 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6331 mask = 1;
6332 cu_bitmap = 0;
6333 counter = 0;
6334 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6335 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6336 if (counter < 2)
6337 cu_bitmap |= mask;
6338 counter ++;
6339 }
6340 mask <<= 1;
6341 }
6342
6343 active_cu_number += counter;
6344 tmp |= (cu_bitmap << (i * 16 + j * 8));
6345 }
6346 }
6347
6348 WREG32(RLC_PG_AO_CU_MASK, tmp);
6349
6350 tmp = RREG32(RLC_MAX_PG_CU);
6351 tmp &= ~MAX_PU_CU_MASK;
6352 tmp |= MAX_PU_CU(active_cu_number);
6353 WREG32(RLC_MAX_PG_CU, tmp);
6354}
6355
6356static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6357 bool enable)
6358{
6359 u32 data, orig;
6360
6361 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6362 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6363 data |= STATIC_PER_CU_PG_ENABLE;
6364 else
6365 data &= ~STATIC_PER_CU_PG_ENABLE;
6366 if (orig != data)
6367 WREG32(RLC_PG_CNTL, data);
6368}
6369
6370static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6371 bool enable)
6372{
6373 u32 data, orig;
6374
6375 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6376 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6377 data |= DYN_PER_CU_PG_ENABLE;
6378 else
6379 data &= ~DYN_PER_CU_PG_ENABLE;
6380 if (orig != data)
6381 WREG32(RLC_PG_CNTL, data);
6382}
6383
6384#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6385#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6386
6387static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6388{
6389 u32 data, orig;
6390 u32 i;
6391
6392 if (rdev->rlc.cs_data) {
6393 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6394 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6395 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6396 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6397 } else {
6398 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6399 for (i = 0; i < 3; i++)
6400 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6401 }
6402 if (rdev->rlc.reg_list) {
6403 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6404 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6405 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6406 }
6407
6408 orig = data = RREG32(RLC_PG_CNTL);
6409 data |= GFX_PG_SRC;
6410 if (orig != data)
6411 WREG32(RLC_PG_CNTL, data);
6412
6413 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6414 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6415
6416 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6417 data &= ~IDLE_POLL_COUNT_MASK;
6418 data |= IDLE_POLL_COUNT(0x60);
6419 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6420
6421 data = 0x10101010;
6422 WREG32(RLC_PG_DELAY, data);
6423
6424 data = RREG32(RLC_PG_DELAY_2);
6425 data &= ~0xff;
6426 data |= 0x3;
6427 WREG32(RLC_PG_DELAY_2, data);
6428
6429 data = RREG32(RLC_AUTO_PG_CTRL);
6430 data &= ~GRBM_REG_SGIT_MASK;
6431 data |= GRBM_REG_SGIT(0x700);
6432 WREG32(RLC_AUTO_PG_CTRL, data);
6433
6434}
6435
6436static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6437{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6438 cik_enable_gfx_cgpg(rdev, enable);
6439 cik_enable_gfx_static_mgpg(rdev, enable);
6440 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6441}
6442
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6443u32 cik_get_csb_size(struct radeon_device *rdev)
6444{
6445 u32 count = 0;
6446 const struct cs_section_def *sect = NULL;
6447 const struct cs_extent_def *ext = NULL;
6448
6449 if (rdev->rlc.cs_data == NULL)
6450 return 0;
6451
6452 /* begin clear state */
6453 count += 2;
6454 /* context control state */
6455 count += 3;
6456
6457 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6458 for (ext = sect->section; ext->extent != NULL; ++ext) {
6459 if (sect->id == SECT_CONTEXT)
6460 count += 2 + ext->reg_count;
6461 else
6462 return 0;
6463 }
6464 }
6465 /* pa_sc_raster_config/pa_sc_raster_config1 */
6466 count += 4;
6467 /* end clear state */
6468 count += 2;
6469 /* clear state */
6470 count += 2;
6471
6472 return count;
6473}
6474
6475void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6476{
6477 u32 count = 0, i;
6478 const struct cs_section_def *sect = NULL;
6479 const struct cs_extent_def *ext = NULL;
6480
6481 if (rdev->rlc.cs_data == NULL)
6482 return;
6483 if (buffer == NULL)
6484 return;
6485
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6486 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6487 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6488
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6489 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6490 buffer[count++] = cpu_to_le32(0x80000000);
6491 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6492
6493 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6494 for (ext = sect->section; ext->extent != NULL; ++ext) {
6495 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6496 buffer[count++] =
6497 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6498 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6499 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6500 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6501 } else {
6502 return;
6503 }
6504 }
6505 }
6506
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6507 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6508 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6509 switch (rdev->family) {
6510 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6511 buffer[count++] = cpu_to_le32(0x16000012);
6512 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6513 break;
6514 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6515 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6516 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6517 break;
6518 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6519 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6520 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6521 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame]6522 case CHIP_HAWAII:
6523 buffer[count++] = 0x3a00161a;
6524 buffer[count++] = 0x0000002e;
6525 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6526 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6527 buffer[count++] = cpu_to_le32(0x00000000);
6528 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6529 break;
6530 }
6531
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6532 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6533 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6534
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6535 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6536 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6537}
6538
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6539static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6540{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6541 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6542 cik_enable_sck_slowdown_on_pu(rdev, true);
6543 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6544 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6545 cik_init_gfx_cgpg(rdev);
6546 cik_enable_cp_pg(rdev, true);
6547 cik_enable_gds_pg(rdev, true);
6548 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6549 cik_init_ao_cu_mask(rdev);
6550 cik_update_gfx_pg(rdev, true);
6551 }
6552}
6553
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6554static void cik_fini_pg(struct radeon_device *rdev)
6555{
6556 if (rdev->pg_flags) {
6557 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6558 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6559 cik_enable_cp_pg(rdev, false);
6560 cik_enable_gds_pg(rdev, false);
6561 }
6562 }
6563}
6564
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6565/*
6566 * Interrupts
6567 * Starting with r6xx, interrupts are handled via a ring buffer.
6568 * Ring buffers are areas of GPU accessible memory that the GPU
6569 * writes interrupt vectors into and the host reads vectors out of.
6570 * There is a rptr (read pointer) that determines where the
6571 * host is currently reading, and a wptr (write pointer)
6572 * which determines where the GPU has written. When the
6573 * pointers are equal, the ring is idle. When the GPU
6574 * writes vectors to the ring buffer, it increments the
6575 * wptr. When there is an interrupt, the host then starts
6576 * fetching commands and processing them until the pointers are
6577 * equal again at which point it updates the rptr.
6578 */
6579
6580/**
6581 * cik_enable_interrupts - Enable the interrupt ring buffer
6582 *
6583 * @rdev: radeon_device pointer
6584 *
6585 * Enable the interrupt ring buffer (CIK).
6586 */
6587static void cik_enable_interrupts(struct radeon_device *rdev)
6588{
6589 u32 ih_cntl = RREG32(IH_CNTL);
6590 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6591
6592 ih_cntl |= ENABLE_INTR;
6593 ih_rb_cntl |= IH_RB_ENABLE;
6594 WREG32(IH_CNTL, ih_cntl);
6595 WREG32(IH_RB_CNTL, ih_rb_cntl);
6596 rdev->ih.enabled = true;
6597}
6598
6599/**
6600 * cik_disable_interrupts - Disable the interrupt ring buffer
6601 *
6602 * @rdev: radeon_device pointer
6603 *
6604 * Disable the interrupt ring buffer (CIK).
6605 */
6606static void cik_disable_interrupts(struct radeon_device *rdev)
6607{
6608 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6609 u32 ih_cntl = RREG32(IH_CNTL);
6610
6611 ih_rb_cntl &= ~IH_RB_ENABLE;
6612 ih_cntl &= ~ENABLE_INTR;
6613 WREG32(IH_RB_CNTL, ih_rb_cntl);
6614 WREG32(IH_CNTL, ih_cntl);
6615 /* set rptr, wptr to 0 */
6616 WREG32(IH_RB_RPTR, 0);
6617 WREG32(IH_RB_WPTR, 0);
6618 rdev->ih.enabled = false;
6619 rdev->ih.rptr = 0;
6620}
6621
6622/**
6623 * cik_disable_interrupt_state - Disable all interrupt sources
6624 *
6625 * @rdev: radeon_device pointer
6626 *
6627 * Clear all interrupt enable bits used by the driver (CIK).
6628 */
6629static void cik_disable_interrupt_state(struct radeon_device *rdev)
6630{
6631 u32 tmp;
6632
6633 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6634 tmp = RREG32(CP_INT_CNTL_RING0) &
6635 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6636 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6637 /* sdma */
6638 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6639 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6640 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6641 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6642 /* compute queues */
6643 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6644 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6645 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6646 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6647 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6648 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6649 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6650 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6651 /* grbm */
6652 WREG32(GRBM_INT_CNTL, 0);
6653 /* vline/vblank, etc. */
6654 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6655 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6656 if (rdev->num_crtc >= 4) {
6657 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6658 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6659 }
6660 if (rdev->num_crtc >= 6) {
6661 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6662 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6663 }
6664
6665 /* dac hotplug */
6666 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6667
6668 /* digital hotplug */
6669 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6670 WREG32(DC_HPD1_INT_CONTROL, tmp);
6671 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6672 WREG32(DC_HPD2_INT_CONTROL, tmp);
6673 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6674 WREG32(DC_HPD3_INT_CONTROL, tmp);
6675 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6676 WREG32(DC_HPD4_INT_CONTROL, tmp);
6677 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6678 WREG32(DC_HPD5_INT_CONTROL, tmp);
6679 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6680 WREG32(DC_HPD6_INT_CONTROL, tmp);
6681
6682}
6683
6684/**
6685 * cik_irq_init - init and enable the interrupt ring
6686 *
6687 * @rdev: radeon_device pointer
6688 *
6689 * Allocate a ring buffer for the interrupt controller,
6690 * enable the RLC, disable interrupts, enable the IH
6691 * ring buffer and enable it (CIK).
6692 * Called at device load and reume.
6693 * Returns 0 for success, errors for failure.
6694 */
6695static int cik_irq_init(struct radeon_device *rdev)
6696{
6697 int ret = 0;
6698 int rb_bufsz;
6699 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6700
6701 /* allocate ring */
6702 ret = r600_ih_ring_alloc(rdev);
6703 if (ret)
6704 return ret;
6705
6706 /* disable irqs */
6707 cik_disable_interrupts(rdev);
6708
6709 /* init rlc */
6710 ret = cik_rlc_resume(rdev);
6711 if (ret) {
6712 r600_ih_ring_fini(rdev);
6713 return ret;
6714 }
6715
6716 /* setup interrupt control */
6717 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6718 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6719 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6720 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6721 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6722 */
6723 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6724 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6725 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6726 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6727
6728 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6729 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6730
6731 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6732 IH_WPTR_OVERFLOW_CLEAR |
6733 (rb_bufsz << 1));
6734
6735 if (rdev->wb.enabled)
6736 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6737
6738 /* set the writeback address whether it's enabled or not */
6739 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6740 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6741
6742 WREG32(IH_RB_CNTL, ih_rb_cntl);
6743
6744 /* set rptr, wptr to 0 */
6745 WREG32(IH_RB_RPTR, 0);
6746 WREG32(IH_RB_WPTR, 0);
6747
6748 /* Default settings for IH_CNTL (disabled at first) */
6749 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6750 /* RPTR_REARM only works if msi's are enabled */
6751 if (rdev->msi_enabled)
6752 ih_cntl |= RPTR_REARM;
6753 WREG32(IH_CNTL, ih_cntl);
6754
6755 /* force the active interrupt state to all disabled */
6756 cik_disable_interrupt_state(rdev);
6757
6758 pci_set_master(rdev->pdev);
6759
6760 /* enable irqs */
6761 cik_enable_interrupts(rdev);
6762
6763 return ret;
6764}
6765
6766/**
6767 * cik_irq_set - enable/disable interrupt sources
6768 *
6769 * @rdev: radeon_device pointer
6770 *
6771 * Enable interrupt sources on the GPU (vblanks, hpd,
6772 * etc.) (CIK).
6773 * Returns 0 for success, errors for failure.
6774 */
6775int cik_irq_set(struct radeon_device *rdev)
6776{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6777 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6778 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6779 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6780 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6781 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6782 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6783 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6784 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6785
6786 if (!rdev->irq.installed) {
6787 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6788 return -EINVAL;
6789 }
6790 /* don't enable anything if the ih is disabled */
6791 if (!rdev->ih.enabled) {
6792 cik_disable_interrupts(rdev);
6793 /* force the active interrupt state to all disabled */
6794 cik_disable_interrupt_state(rdev);
6795 return 0;
6796 }
6797
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6798 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6799 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6800 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6801
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6802 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6803 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6804 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6805 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6806 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6807 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6808
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6809 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6810 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6811
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6812 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6813 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6814 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6815 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6816 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6817 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6818 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6819 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6820
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6821 if (rdev->flags & RADEON_IS_IGP)
6822 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6823 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6824 else
6825 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6826 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6827
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6828 /* enable CP interrupts on all rings */
6829 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6830 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6831 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6832 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6833 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6834 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6835 DRM_DEBUG("si_irq_set: sw int cp1\n");
6836 if (ring->me == 1) {
6837 switch (ring->pipe) {
6838 case 0:
6839 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6840 break;
6841 case 1:
6842 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6843 break;
6844 case 2:
6845 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6846 break;
6847 case 3:
6848 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6849 break;
6850 default:
6851 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6852 break;
6853 }
6854 } else if (ring->me == 2) {
6855 switch (ring->pipe) {
6856 case 0:
6857 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6858 break;
6859 case 1:
6860 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6861 break;
6862 case 2:
6863 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6864 break;
6865 case 3:
6866 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6867 break;
6868 default:
6869 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6870 break;
6871 }
6872 } else {
6873 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6874 }
6875 }
6876 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6877 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6878 DRM_DEBUG("si_irq_set: sw int cp2\n");
6879 if (ring->me == 1) {
6880 switch (ring->pipe) {
6881 case 0:
6882 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6883 break;
6884 case 1:
6885 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6886 break;
6887 case 2:
6888 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6889 break;
6890 case 3:
6891 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6892 break;
6893 default:
6894 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6895 break;
6896 }
6897 } else if (ring->me == 2) {
6898 switch (ring->pipe) {
6899 case 0:
6900 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6901 break;
6902 case 1:
6903 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6904 break;
6905 case 2:
6906 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6907 break;
6908 case 3:
6909 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6910 break;
6911 default:
6912 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6913 break;
6914 }
6915 } else {
6916 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6917 }
6918 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6919
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6920 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6921 DRM_DEBUG("cik_irq_set: sw int dma\n");
6922 dma_cntl |= TRAP_ENABLE;
6923 }
6924
6925 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6926 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6927 dma_cntl1 |= TRAP_ENABLE;
6928 }
6929
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6930 if (rdev->irq.crtc_vblank_int[0] ||
6931 atomic_read(&rdev->irq.pflip[0])) {
6932 DRM_DEBUG("cik_irq_set: vblank 0\n");
6933 crtc1 |= VBLANK_INTERRUPT_MASK;
6934 }
6935 if (rdev->irq.crtc_vblank_int[1] ||
6936 atomic_read(&rdev->irq.pflip[1])) {
6937 DRM_DEBUG("cik_irq_set: vblank 1\n");
6938 crtc2 |= VBLANK_INTERRUPT_MASK;
6939 }
6940 if (rdev->irq.crtc_vblank_int[2] ||
6941 atomic_read(&rdev->irq.pflip[2])) {
6942 DRM_DEBUG("cik_irq_set: vblank 2\n");
6943 crtc3 |= VBLANK_INTERRUPT_MASK;
6944 }
6945 if (rdev->irq.crtc_vblank_int[3] ||
6946 atomic_read(&rdev->irq.pflip[3])) {
6947 DRM_DEBUG("cik_irq_set: vblank 3\n");
6948 crtc4 |= VBLANK_INTERRUPT_MASK;
6949 }
6950 if (rdev->irq.crtc_vblank_int[4] ||
6951 atomic_read(&rdev->irq.pflip[4])) {
6952 DRM_DEBUG("cik_irq_set: vblank 4\n");
6953 crtc5 |= VBLANK_INTERRUPT_MASK;
6954 }
6955 if (rdev->irq.crtc_vblank_int[5] ||
6956 atomic_read(&rdev->irq.pflip[5])) {
6957 DRM_DEBUG("cik_irq_set: vblank 5\n");
6958 crtc6 |= VBLANK_INTERRUPT_MASK;
6959 }
6960 if (rdev->irq.hpd[0]) {
6961 DRM_DEBUG("cik_irq_set: hpd 1\n");
6962 hpd1 |= DC_HPDx_INT_EN;
6963 }
6964 if (rdev->irq.hpd[1]) {
6965 DRM_DEBUG("cik_irq_set: hpd 2\n");
6966 hpd2 |= DC_HPDx_INT_EN;
6967 }
6968 if (rdev->irq.hpd[2]) {
6969 DRM_DEBUG("cik_irq_set: hpd 3\n");
6970 hpd3 |= DC_HPDx_INT_EN;
6971 }
6972 if (rdev->irq.hpd[3]) {
6973 DRM_DEBUG("cik_irq_set: hpd 4\n");
6974 hpd4 |= DC_HPDx_INT_EN;
6975 }
6976 if (rdev->irq.hpd[4]) {
6977 DRM_DEBUG("cik_irq_set: hpd 5\n");
6978 hpd5 |= DC_HPDx_INT_EN;
6979 }
6980 if (rdev->irq.hpd[5]) {
6981 DRM_DEBUG("cik_irq_set: hpd 6\n");
6982 hpd6 |= DC_HPDx_INT_EN;
6983 }
6984
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6985 if (rdev->irq.dpm_thermal) {
6986 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6987 if (rdev->flags & RADEON_IS_IGP)
6988 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6989 else
6990 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6991 }
6992
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6993 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6994
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6995 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6996 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6997
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6998 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6999 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7000 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7001 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7002 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7003 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7004 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7005 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7006
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7007 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7008
7009 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7010 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7011 if (rdev->num_crtc >= 4) {
7012 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7013 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7014 }
7015 if (rdev->num_crtc >= 6) {
7016 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7017 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7018 }
7019
7020 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7021 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7022 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7023 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7024 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7025 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7026
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]7027 if (rdev->flags & RADEON_IS_IGP)
7028 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7029 else
7030 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7031
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7032 return 0;
7033}
7034
7035/**
7036 * cik_irq_ack - ack interrupt sources
7037 *
7038 * @rdev: radeon_device pointer
7039 *
7040 * Ack interrupt sources on the GPU (vblanks, hpd,
7041 * etc.) (CIK). Certain interrupts sources are sw
7042 * generated and do not require an explicit ack.
7043 */
7044static inline void cik_irq_ack(struct radeon_device *rdev)
7045{
7046 u32 tmp;
7047
7048 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7049 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7050 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7051 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7052 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7053 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7054 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7055
7056 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7057 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7058 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7059 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7060 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7061 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7062 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7063 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7064
7065 if (rdev->num_crtc >= 4) {
7066 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7067 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7068 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7069 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7070 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7071 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7072 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7073 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7074 }
7075
7076 if (rdev->num_crtc >= 6) {
7077 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7078 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7079 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7080 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7081 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7082 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7083 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7084 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7085 }
7086
7087 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7088 tmp = RREG32(DC_HPD1_INT_CONTROL);
7089 tmp |= DC_HPDx_INT_ACK;
7090 WREG32(DC_HPD1_INT_CONTROL, tmp);
7091 }
7092 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7093 tmp = RREG32(DC_HPD2_INT_CONTROL);
7094 tmp |= DC_HPDx_INT_ACK;
7095 WREG32(DC_HPD2_INT_CONTROL, tmp);
7096 }
7097 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7098 tmp = RREG32(DC_HPD3_INT_CONTROL);
7099 tmp |= DC_HPDx_INT_ACK;
7100 WREG32(DC_HPD3_INT_CONTROL, tmp);
7101 }
7102 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7103 tmp = RREG32(DC_HPD4_INT_CONTROL);
7104 tmp |= DC_HPDx_INT_ACK;
7105 WREG32(DC_HPD4_INT_CONTROL, tmp);
7106 }
7107 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7108 tmp = RREG32(DC_HPD5_INT_CONTROL);
7109 tmp |= DC_HPDx_INT_ACK;
7110 WREG32(DC_HPD5_INT_CONTROL, tmp);
7111 }
7112 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7113 tmp = RREG32(DC_HPD5_INT_CONTROL);
7114 tmp |= DC_HPDx_INT_ACK;
7115 WREG32(DC_HPD6_INT_CONTROL, tmp);
7116 }
7117}
7118
7119/**
7120 * cik_irq_disable - disable interrupts
7121 *
7122 * @rdev: radeon_device pointer
7123 *
7124 * Disable interrupts on the hw (CIK).
7125 */
7126static void cik_irq_disable(struct radeon_device *rdev)
7127{
7128 cik_disable_interrupts(rdev);
7129 /* Wait and acknowledge irq */
7130 mdelay(1);
7131 cik_irq_ack(rdev);
7132 cik_disable_interrupt_state(rdev);
7133}
7134
7135/**
7136 * cik_irq_disable - disable interrupts for suspend
7137 *
7138 * @rdev: radeon_device pointer
7139 *
7140 * Disable interrupts and stop the RLC (CIK).
7141 * Used for suspend.
7142 */
7143static void cik_irq_suspend(struct radeon_device *rdev)
7144{
7145 cik_irq_disable(rdev);
7146 cik_rlc_stop(rdev);
7147}
7148
7149/**
7150 * cik_irq_fini - tear down interrupt support
7151 *
7152 * @rdev: radeon_device pointer
7153 *
7154 * Disable interrupts on the hw and free the IH ring
7155 * buffer (CIK).
7156 * Used for driver unload.
7157 */
7158static void cik_irq_fini(struct radeon_device *rdev)
7159{
7160 cik_irq_suspend(rdev);
7161 r600_ih_ring_fini(rdev);
7162}
7163
7164/**
7165 * cik_get_ih_wptr - get the IH ring buffer wptr
7166 *
7167 * @rdev: radeon_device pointer
7168 *
7169 * Get the IH ring buffer wptr from either the register
7170 * or the writeback memory buffer (CIK). Also check for
7171 * ring buffer overflow and deal with it.
7172 * Used by cik_irq_process().
7173 * Returns the value of the wptr.
7174 */
7175static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7176{
7177 u32 wptr, tmp;
7178
7179 if (rdev->wb.enabled)
7180 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7181 else
7182 wptr = RREG32(IH_RB_WPTR);
7183
7184 if (wptr & RB_OVERFLOW) {
7185 /* When a ring buffer overflow happen start parsing interrupt
7186 * from the last not overwritten vector (wptr + 16). Hopefully
7187 * this should allow us to catchup.
7188 */
7189 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7190 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7191 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7192 tmp = RREG32(IH_RB_CNTL);
7193 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7194 WREG32(IH_RB_CNTL, tmp);
7195 }
7196 return (wptr & rdev->ih.ptr_mask);
7197}
7198
7199/* CIK IV Ring
7200 * Each IV ring entry is 128 bits:
7201 * [7:0] - interrupt source id
7202 * [31:8] - reserved
7203 * [59:32] - interrupt source data
7204 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7205 * [71:64] - RINGID
7206 * CP:
7207 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7208 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7209 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7210 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7211 * PIPE_ID - ME0 0=3D
7212 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7213 * SDMA:
7214 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7215 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7216 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7217 * [79:72] - VMID
7218 * [95:80] - PASID
7219 * [127:96] - reserved
7220 */
7221/**
7222 * cik_irq_process - interrupt handler
7223 *
7224 * @rdev: radeon_device pointer
7225 *
7226 * Interrupt hander (CIK). Walk the IH ring,
7227 * ack interrupts and schedule work to handle
7228 * interrupt events.
7229 * Returns irq process return code.
7230 */
7231int cik_irq_process(struct radeon_device *rdev)
7232{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7233 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7234 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7235 u32 wptr;
7236 u32 rptr;
7237 u32 src_id, src_data, ring_id;
7238 u8 me_id, pipe_id, queue_id;
7239 u32 ring_index;
7240 bool queue_hotplug = false;
7241 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7242 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7243 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7244
7245 if (!rdev->ih.enabled || rdev->shutdown)
7246 return IRQ_NONE;
7247
7248 wptr = cik_get_ih_wptr(rdev);
7249
7250restart_ih:
7251 /* is somebody else already processing irqs? */
7252 if (atomic_xchg(&rdev->ih.lock, 1))
7253 return IRQ_NONE;
7254
7255 rptr = rdev->ih.rptr;
7256 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7257
7258 /* Order reading of wptr vs. reading of IH ring data */
7259 rmb();
7260
7261 /* display interrupts */
7262 cik_irq_ack(rdev);
7263
7264 while (rptr != wptr) {
7265 /* wptr/rptr are in bytes! */
7266 ring_index = rptr / 4;
7267 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7268 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7269 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7270
7271 switch (src_id) {
7272 case 1: /* D1 vblank/vline */
7273 switch (src_data) {
7274 case 0: /* D1 vblank */
7275 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7276 if (rdev->irq.crtc_vblank_int[0]) {
7277 drm_handle_vblank(rdev->ddev, 0);
7278 rdev->pm.vblank_sync = true;
7279 wake_up(&rdev->irq.vblank_queue);
7280 }
7281 if (atomic_read(&rdev->irq.pflip[0]))
7282 radeon_crtc_handle_flip(rdev, 0);
7283 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7284 DRM_DEBUG("IH: D1 vblank\n");
7285 }
7286 break;
7287 case 1: /* D1 vline */
7288 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7289 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7290 DRM_DEBUG("IH: D1 vline\n");
7291 }
7292 break;
7293 default:
7294 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7295 break;
7296 }
7297 break;
7298 case 2: /* D2 vblank/vline */
7299 switch (src_data) {
7300 case 0: /* D2 vblank */
7301 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7302 if (rdev->irq.crtc_vblank_int[1]) {
7303 drm_handle_vblank(rdev->ddev, 1);
7304 rdev->pm.vblank_sync = true;
7305 wake_up(&rdev->irq.vblank_queue);
7306 }
7307 if (atomic_read(&rdev->irq.pflip[1]))
7308 radeon_crtc_handle_flip(rdev, 1);
7309 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7310 DRM_DEBUG("IH: D2 vblank\n");
7311 }
7312 break;
7313 case 1: /* D2 vline */
7314 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7315 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7316 DRM_DEBUG("IH: D2 vline\n");
7317 }
7318 break;
7319 default:
7320 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7321 break;
7322 }
7323 break;
7324 case 3: /* D3 vblank/vline */
7325 switch (src_data) {
7326 case 0: /* D3 vblank */
7327 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7328 if (rdev->irq.crtc_vblank_int[2]) {
7329 drm_handle_vblank(rdev->ddev, 2);
7330 rdev->pm.vblank_sync = true;
7331 wake_up(&rdev->irq.vblank_queue);
7332 }
7333 if (atomic_read(&rdev->irq.pflip[2]))
7334 radeon_crtc_handle_flip(rdev, 2);
7335 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7336 DRM_DEBUG("IH: D3 vblank\n");
7337 }
7338 break;
7339 case 1: /* D3 vline */
7340 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7341 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7342 DRM_DEBUG("IH: D3 vline\n");
7343 }
7344 break;
7345 default:
7346 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7347 break;
7348 }
7349 break;
7350 case 4: /* D4 vblank/vline */
7351 switch (src_data) {
7352 case 0: /* D4 vblank */
7353 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7354 if (rdev->irq.crtc_vblank_int[3]) {
7355 drm_handle_vblank(rdev->ddev, 3);
7356 rdev->pm.vblank_sync = true;
7357 wake_up(&rdev->irq.vblank_queue);
7358 }
7359 if (atomic_read(&rdev->irq.pflip[3]))
7360 radeon_crtc_handle_flip(rdev, 3);
7361 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7362 DRM_DEBUG("IH: D4 vblank\n");
7363 }
7364 break;
7365 case 1: /* D4 vline */
7366 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7367 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7368 DRM_DEBUG("IH: D4 vline\n");
7369 }
7370 break;
7371 default:
7372 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7373 break;
7374 }
7375 break;
7376 case 5: /* D5 vblank/vline */
7377 switch (src_data) {
7378 case 0: /* D5 vblank */
7379 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7380 if (rdev->irq.crtc_vblank_int[4]) {
7381 drm_handle_vblank(rdev->ddev, 4);
7382 rdev->pm.vblank_sync = true;
7383 wake_up(&rdev->irq.vblank_queue);
7384 }
7385 if (atomic_read(&rdev->irq.pflip[4]))
7386 radeon_crtc_handle_flip(rdev, 4);
7387 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7388 DRM_DEBUG("IH: D5 vblank\n");
7389 }
7390 break;
7391 case 1: /* D5 vline */
7392 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7393 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7394 DRM_DEBUG("IH: D5 vline\n");
7395 }
7396 break;
7397 default:
7398 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7399 break;
7400 }
7401 break;
7402 case 6: /* D6 vblank/vline */
7403 switch (src_data) {
7404 case 0: /* D6 vblank */
7405 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7406 if (rdev->irq.crtc_vblank_int[5]) {
7407 drm_handle_vblank(rdev->ddev, 5);
7408 rdev->pm.vblank_sync = true;
7409 wake_up(&rdev->irq.vblank_queue);
7410 }
7411 if (atomic_read(&rdev->irq.pflip[5]))
7412 radeon_crtc_handle_flip(rdev, 5);
7413 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7414 DRM_DEBUG("IH: D6 vblank\n");
7415 }
7416 break;
7417 case 1: /* D6 vline */
7418 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7419 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7420 DRM_DEBUG("IH: D6 vline\n");
7421 }
7422 break;
7423 default:
7424 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7425 break;
7426 }
7427 break;
7428 case 42: /* HPD hotplug */
7429 switch (src_data) {
7430 case 0:
7431 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7432 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7433 queue_hotplug = true;
7434 DRM_DEBUG("IH: HPD1\n");
7435 }
7436 break;
7437 case 1:
7438 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7439 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7440 queue_hotplug = true;
7441 DRM_DEBUG("IH: HPD2\n");
7442 }
7443 break;
7444 case 2:
7445 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7446 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7447 queue_hotplug = true;
7448 DRM_DEBUG("IH: HPD3\n");
7449 }
7450 break;
7451 case 3:
7452 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7453 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7454 queue_hotplug = true;
7455 DRM_DEBUG("IH: HPD4\n");
7456 }
7457 break;
7458 case 4:
7459 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7460 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7461 queue_hotplug = true;
7462 DRM_DEBUG("IH: HPD5\n");
7463 }
7464 break;
7465 case 5:
7466 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7467 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7468 queue_hotplug = true;
7469 DRM_DEBUG("IH: HPD6\n");
7470 }
7471 break;
7472 default:
7473 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7474 break;
7475 }
7476 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7477 case 124: /* UVD */
7478 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7479 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7480 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7481 case 146:
7482 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7483 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7484 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7485 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7486 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7487 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7488 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7489 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7490 status);
7491 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7492 /* reset addr and status */
7493 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7494 break;
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]7495 case 167: /* VCE */
7496 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7497 switch (src_data) {
7498 case 0:
7499 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7500 break;
7501 case 1:
7502 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7503 break;
7504 default:
7505 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7506 break;
7507 }
7508 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7509 case 176: /* GFX RB CP_INT */
7510 case 177: /* GFX IB CP_INT */
7511 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7512 break;
7513 case 181: /* CP EOP event */
7514 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7515 /* XXX check the bitfield order! */
7516 me_id = (ring_id & 0x60) >> 5;
7517 pipe_id = (ring_id & 0x18) >> 3;
7518 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7519 switch (me_id) {
7520 case 0:
7521 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7522 break;
7523 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7524 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7525 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7526 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7527 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7528 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7529 break;
7530 }
7531 break;
7532 case 184: /* CP Privileged reg access */
7533 DRM_ERROR("Illegal register access in command stream\n");
7534 /* XXX check the bitfield order! */
7535 me_id = (ring_id & 0x60) >> 5;
7536 pipe_id = (ring_id & 0x18) >> 3;
7537 queue_id = (ring_id & 0x7) >> 0;
7538 switch (me_id) {
7539 case 0:
7540 /* This results in a full GPU reset, but all we need to do is soft
7541 * reset the CP for gfx
7542 */
7543 queue_reset = true;
7544 break;
7545 case 1:
7546 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7547 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7548 break;
7549 case 2:
7550 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7551 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7552 break;
7553 }
7554 break;
7555 case 185: /* CP Privileged inst */
7556 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7557 /* XXX check the bitfield order! */
7558 me_id = (ring_id & 0x60) >> 5;
7559 pipe_id = (ring_id & 0x18) >> 3;
7560 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7561 switch (me_id) {
7562 case 0:
7563 /* This results in a full GPU reset, but all we need to do is soft
7564 * reset the CP for gfx
7565 */
7566 queue_reset = true;
7567 break;
7568 case 1:
7569 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7570 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7571 break;
7572 case 2:
7573 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7574 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7575 break;
7576 }
7577 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7578 case 224: /* SDMA trap event */
7579 /* XXX check the bitfield order! */
7580 me_id = (ring_id & 0x3) >> 0;
7581 queue_id = (ring_id & 0xc) >> 2;
7582 DRM_DEBUG("IH: SDMA trap\n");
7583 switch (me_id) {
7584 case 0:
7585 switch (queue_id) {
7586 case 0:
7587 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7588 break;
7589 case 1:
7590 /* XXX compute */
7591 break;
7592 case 2:
7593 /* XXX compute */
7594 break;
7595 }
7596 break;
7597 case 1:
7598 switch (queue_id) {
7599 case 0:
7600 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7601 break;
7602 case 1:
7603 /* XXX compute */
7604 break;
7605 case 2:
7606 /* XXX compute */
7607 break;
7608 }
7609 break;
7610 }
7611 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7612 case 230: /* thermal low to high */
7613 DRM_DEBUG("IH: thermal low to high\n");
7614 rdev->pm.dpm.thermal.high_to_low = false;
7615 queue_thermal = true;
7616 break;
7617 case 231: /* thermal high to low */
7618 DRM_DEBUG("IH: thermal high to low\n");
7619 rdev->pm.dpm.thermal.high_to_low = true;
7620 queue_thermal = true;
7621 break;
7622 case 233: /* GUI IDLE */
7623 DRM_DEBUG("IH: GUI idle\n");
7624 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7625 case 241: /* SDMA Privileged inst */
7626 case 247: /* SDMA Privileged inst */
7627 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7628 /* XXX check the bitfield order! */
7629 me_id = (ring_id & 0x3) >> 0;
7630 queue_id = (ring_id & 0xc) >> 2;
7631 switch (me_id) {
7632 case 0:
7633 switch (queue_id) {
7634 case 0:
7635 queue_reset = true;
7636 break;
7637 case 1:
7638 /* XXX compute */
7639 queue_reset = true;
7640 break;
7641 case 2:
7642 /* XXX compute */
7643 queue_reset = true;
7644 break;
7645 }
7646 break;
7647 case 1:
7648 switch (queue_id) {
7649 case 0:
7650 queue_reset = true;
7651 break;
7652 case 1:
7653 /* XXX compute */
7654 queue_reset = true;
7655 break;
7656 case 2:
7657 /* XXX compute */
7658 queue_reset = true;
7659 break;
7660 }
7661 break;
7662 }
7663 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7664 default:
7665 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7666 break;
7667 }
7668
7669 /* wptr/rptr are in bytes! */
7670 rptr += 16;
7671 rptr &= rdev->ih.ptr_mask;
7672 }
7673 if (queue_hotplug)
7674 schedule_work(&rdev->hotplug_work);
7675 if (queue_reset)
7676 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7677 if (queue_thermal)
7678 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7679 rdev->ih.rptr = rptr;
7680 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7681 atomic_set(&rdev->ih.lock, 0);
7682
7683 /* make sure wptr hasn't changed while processing */
7684 wptr = cik_get_ih_wptr(rdev);
7685 if (wptr != rptr)
7686 goto restart_ih;
7687
7688 return IRQ_HANDLED;
7689}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7690
7691/*
7692 * startup/shutdown callbacks
7693 */
7694/**
7695 * cik_startup - program the asic to a functional state
7696 *
7697 * @rdev: radeon_device pointer
7698 *
7699 * Programs the asic to a functional state (CIK).
7700 * Called by cik_init() and cik_resume().
7701 * Returns 0 for success, error for failure.
7702 */
7703static int cik_startup(struct radeon_device *rdev)
7704{
7705 struct radeon_ring *ring;
7706 int r;
7707
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7708 /* enable pcie gen2/3 link */
7709 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7710 /* enable aspm */
7711 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7712
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7713 /* scratch needs to be initialized before MC */
7714 r = r600_vram_scratch_init(rdev);
7715 if (r)
7716 return r;
7717
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7718 cik_mc_program(rdev);
7719
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7720 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7721 r = ci_mc_load_microcode(rdev);
7722 if (r) {
7723 DRM_ERROR("Failed to load MC firmware!\n");
7724 return r;
7725 }
7726 }
7727
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7728 r = cik_pcie_gart_enable(rdev);
7729 if (r)
7730 return r;
7731 cik_gpu_init(rdev);
7732
7733 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7734 if (rdev->flags & RADEON_IS_IGP) {
7735 if (rdev->family == CHIP_KAVERI) {
7736 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7737 rdev->rlc.reg_list_size =
7738 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7739 } else {
7740 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7741 rdev->rlc.reg_list_size =
7742 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7743 }
7744 }
7745 rdev->rlc.cs_data = ci_cs_data;
7746 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7747 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7748 if (r) {
7749 DRM_ERROR("Failed to init rlc BOs!\n");
7750 return r;
7751 }
7752
7753 /* allocate wb buffer */
7754 r = radeon_wb_init(rdev);
7755 if (r)
7756 return r;
7757
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7758 /* allocate mec buffers */
7759 r = cik_mec_init(rdev);
7760 if (r) {
7761 DRM_ERROR("Failed to init MEC BOs!\n");
7762 return r;
7763 }
7764
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7765 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7766 if (r) {
7767 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7768 return r;
7769 }
7770
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7771 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7772 if (r) {
7773 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7774 return r;
7775 }
7776
7777 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7778 if (r) {
7779 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7780 return r;
7781 }
7782
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7783 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7784 if (r) {
7785 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7786 return r;
7787 }
7788
7789 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7790 if (r) {
7791 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7792 return r;
7793 }
7794
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7795 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7796 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7797 r = uvd_v4_2_resume(rdev);
7798 if (!r) {
7799 r = radeon_fence_driver_start_ring(rdev,
7800 R600_RING_TYPE_UVD_INDEX);
7801 if (r)
7802 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7803 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7804 }
7805 if (r)
7806 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7807
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]7808 r = radeon_vce_resume(rdev);
7809 if (!r) {
7810 r = vce_v2_0_resume(rdev);
7811 if (!r)
7812 r = radeon_fence_driver_start_ring(rdev,
7813 TN_RING_TYPE_VCE1_INDEX);
7814 if (!r)
7815 r = radeon_fence_driver_start_ring(rdev,
7816 TN_RING_TYPE_VCE2_INDEX);
7817 }
7818 if (r) {
7819 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7820 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7821 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7822 }
7823
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7824 /* Enable IRQ */
7825 if (!rdev->irq.installed) {
7826 r = radeon_irq_kms_init(rdev);
7827 if (r)
7828 return r;
7829 }
7830
7831 r = cik_irq_init(rdev);
7832 if (r) {
7833 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7834 radeon_irq_kms_fini(rdev);
7835 return r;
7836 }
7837 cik_irq_set(rdev);
7838
7839 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7840 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7841 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7842 if (r)
7843 return r;
7844
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7845 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7846 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7847 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7848 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7849 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7850 if (r)
7851 return r;
7852 ring->me = 1; /* first MEC */
7853 ring->pipe = 0; /* first pipe */
7854 ring->queue = 0; /* first queue */
7855 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7856
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7857 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7858 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7859 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7860 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7861 if (r)
7862 return r;
7863 /* dGPU only have 1 MEC */
7864 ring->me = 1; /* first MEC */
7865 ring->pipe = 0; /* first pipe */
7866 ring->queue = 1; /* second queue */
7867 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7868
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7869 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7870 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7871 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7872 if (r)
7873 return r;
7874
7875 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7876 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7877 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7878 if (r)
7879 return r;
7880
7881 r = cik_cp_resume(rdev);
7882 if (r)
7883 return r;
7884
7885 r = cik_sdma_resume(rdev);
7886 if (r)
7887 return r;
7888
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7889 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7890 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7891 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7892 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7893 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7894 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7895 if (r)
7896 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7897 }
7898
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]7899 r = -ENOENT;
7900
7901 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7902 if (ring->ring_size)
7903 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7904 VCE_CMD_NO_OP);
7905
7906 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7907 if (ring->ring_size)
7908 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7909 VCE_CMD_NO_OP);
7910
7911 if (!r)
7912 r = vce_v1_0_init(rdev);
7913 else if (r != -ENOENT)
7914 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7915
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7916 r = radeon_ib_pool_init(rdev);
7917 if (r) {
7918 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7919 return r;
7920 }
7921
7922 r = radeon_vm_manager_init(rdev);
7923 if (r) {
7924 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7925 return r;
7926 }
7927
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7928 r = dce6_audio_init(rdev);
7929 if (r)
7930 return r;
7931
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7932 return 0;
7933}
7934
7935/**
7936 * cik_resume - resume the asic to a functional state
7937 *
7938 * @rdev: radeon_device pointer
7939 *
7940 * Programs the asic to a functional state (CIK).
7941 * Called at resume.
7942 * Returns 0 for success, error for failure.
7943 */
7944int cik_resume(struct radeon_device *rdev)
7945{
7946 int r;
7947
7948 /* post card */
7949 atom_asic_init(rdev->mode_info.atom_context);
7950
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7951 /* init golden registers */
7952 cik_init_golden_registers(rdev);
7953
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7954 radeon_pm_resume(rdev);
7955
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7956 rdev->accel_working = true;
7957 r = cik_startup(rdev);
7958 if (r) {
7959 DRM_ERROR("cik startup failed on resume\n");
7960 rdev->accel_working = false;
7961 return r;
7962 }
7963
7964 return r;
7965
7966}
7967
7968/**
7969 * cik_suspend - suspend the asic
7970 *
7971 * @rdev: radeon_device pointer
7972 *
7973 * Bring the chip into a state suitable for suspend (CIK).
7974 * Called at suspend.
7975 * Returns 0 for success.
7976 */
7977int cik_suspend(struct radeon_device *rdev)
7978{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7979 radeon_pm_suspend(rdev);
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7980 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7981 radeon_vm_manager_fini(rdev);
7982 cik_cp_enable(rdev, false);
7983 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7984 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7985 radeon_uvd_suspend(rdev);
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]7986 radeon_vce_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7987 cik_fini_pg(rdev);
7988 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7989 cik_irq_suspend(rdev);
7990 radeon_wb_disable(rdev);
7991 cik_pcie_gart_disable(rdev);
7992 return 0;
7993}
7994
7995/* Plan is to move initialization in that function and use
7996 * helper function so that radeon_device_init pretty much
7997 * do nothing more than calling asic specific function. This
7998 * should also allow to remove a bunch of callback function
7999 * like vram_info.
8000 */
8001/**
8002 * cik_init - asic specific driver and hw init
8003 *
8004 * @rdev: radeon_device pointer
8005 *
8006 * Setup asic specific driver variables and program the hw
8007 * to a functional state (CIK).
8008 * Called at driver startup.
8009 * Returns 0 for success, errors for failure.
8010 */
8011int cik_init(struct radeon_device *rdev)
8012{
8013 struct radeon_ring *ring;
8014 int r;
8015
8016 /* Read BIOS */
8017 if (!radeon_get_bios(rdev)) {
8018 if (ASIC_IS_AVIVO(rdev))
8019 return -EINVAL;
8020 }
8021 /* Must be an ATOMBIOS */
8022 if (!rdev->is_atom_bios) {
8023 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8024 return -EINVAL;
8025 }
8026 r = radeon_atombios_init(rdev);
8027 if (r)
8028 return r;
8029
8030 /* Post card if necessary */
8031 if (!radeon_card_posted(rdev)) {
8032 if (!rdev->bios) {
8033 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8034 return -EINVAL;
8035 }
8036 DRM_INFO("GPU not posted. posting now...\n");
8037 atom_asic_init(rdev->mode_info.atom_context);
8038 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]8039 /* init golden registers */
8040 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8041 /* Initialize scratch registers */
8042 cik_scratch_init(rdev);
8043 /* Initialize surface registers */
8044 radeon_surface_init(rdev);
8045 /* Initialize clocks */
8046 radeon_get_clock_info(rdev->ddev);
8047
8048 /* Fence driver */
8049 r = radeon_fence_driver_init(rdev);
8050 if (r)
8051 return r;
8052
8053 /* initialize memory controller */
8054 r = cik_mc_init(rdev);
8055 if (r)
8056 return r;
8057 /* Memory manager */
8058 r = radeon_bo_init(rdev);
8059 if (r)
8060 return r;
8061
Alex Deucher01ac8792013-12-18 19:11:27 -0500[diff] [blame]8062 if (rdev->flags & RADEON_IS_IGP) {
8063 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8064 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8065 r = cik_init_microcode(rdev);
8066 if (r) {
8067 DRM_ERROR("Failed to load firmware!\n");
8068 return r;
8069 }
8070 }
8071 } else {
8072 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8073 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8074 !rdev->mc_fw) {
8075 r = cik_init_microcode(rdev);
8076 if (r) {
8077 DRM_ERROR("Failed to load firmware!\n");
8078 return r;
8079 }
8080 }
8081 }
8082
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8083 /* Initialize power management */
8084 radeon_pm_init(rdev);
8085
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8086 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8087 ring->ring_obj = NULL;
8088 r600_ring_init(rdev, ring, 1024 * 1024);
8089
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8090 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8091 ring->ring_obj = NULL;
8092 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8093 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8094 if (r)
8095 return r;
8096
8097 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8098 ring->ring_obj = NULL;
8099 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8100 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8101 if (r)
8102 return r;
8103
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8104 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8105 ring->ring_obj = NULL;
8106 r600_ring_init(rdev, ring, 256 * 1024);
8107
8108 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8109 ring->ring_obj = NULL;
8110 r600_ring_init(rdev, ring, 256 * 1024);
8111
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8112 r = radeon_uvd_init(rdev);
8113 if (!r) {
8114 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8115 ring->ring_obj = NULL;
8116 r600_ring_init(rdev, ring, 4096);
8117 }
8118
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]8119 r = radeon_vce_init(rdev);
8120 if (!r) {
8121 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8122 ring->ring_obj = NULL;
8123 r600_ring_init(rdev, ring, 4096);
8124
8125 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8126 ring->ring_obj = NULL;
8127 r600_ring_init(rdev, ring, 4096);
8128 }
8129
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8130 rdev->ih.ring_obj = NULL;
8131 r600_ih_ring_init(rdev, 64 * 1024);
8132
8133 r = r600_pcie_gart_init(rdev);
8134 if (r)
8135 return r;
8136
8137 rdev->accel_working = true;
8138 r = cik_startup(rdev);
8139 if (r) {
8140 dev_err(rdev->dev, "disabling GPU acceleration\n");
8141 cik_cp_fini(rdev);
8142 cik_sdma_fini(rdev);
8143 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8144 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8145 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8146 radeon_wb_fini(rdev);
8147 radeon_ib_pool_fini(rdev);
8148 radeon_vm_manager_fini(rdev);
8149 radeon_irq_kms_fini(rdev);
8150 cik_pcie_gart_fini(rdev);
8151 rdev->accel_working = false;
8152 }
8153
8154 /* Don't start up if the MC ucode is missing.
8155 * The default clocks and voltages before the MC ucode
8156 * is loaded are not suffient for advanced operations.
8157 */
8158 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8159 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8160 return -EINVAL;
8161 }
8162
8163 return 0;
8164}
8165
8166/**
8167 * cik_fini - asic specific driver and hw fini
8168 *
8169 * @rdev: radeon_device pointer
8170 *
8171 * Tear down the asic specific driver variables and program the hw
8172 * to an idle state (CIK).
8173 * Called at driver unload.
8174 */
8175void cik_fini(struct radeon_device *rdev)
8176{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8177 radeon_pm_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8178 cik_cp_fini(rdev);
8179 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]8180 cik_fini_pg(rdev);
8181 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8182 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8183 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8184 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8185 radeon_wb_fini(rdev);
8186 radeon_vm_manager_fini(rdev);
8187 radeon_ib_pool_fini(rdev);
8188 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]8189 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8190 radeon_uvd_fini(rdev);
Christian Königd93f7932013-05-23 12:10:04 +0200[diff] [blame]8191 radeon_vce_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8192 cik_pcie_gart_fini(rdev);
8193 r600_vram_scratch_fini(rdev);
8194 radeon_gem_fini(rdev);
8195 radeon_fence_driver_fini(rdev);
8196 radeon_bo_fini(rdev);
8197 radeon_atombios_fini(rdev);
8198 kfree(rdev->bios);
8199 rdev->bios = NULL;
8200}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8201
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8202void dce8_program_fmt(struct drm_encoder *encoder)
8203{
8204 struct drm_device *dev = encoder->dev;
8205 struct radeon_device *rdev = dev->dev_private;
8206 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8207 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8208 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8209 int bpc = 0;
8210 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8211 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8212
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8213 if (connector) {
8214 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8215 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8216 dither = radeon_connector->dither;
8217 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8218
8219 /* LVDS/eDP FMT is set up by atom */
8220 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8221 return;
8222
8223 /* not needed for analog */
8224 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8225 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8226 return;
8227
8228 if (bpc == 0)
8229 return;
8230
8231 switch (bpc) {
8232 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8233 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8234 /* XXX sort out optimal dither settings */
8235 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8236 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8237 else
8238 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8239 break;
8240 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8241 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8242 /* XXX sort out optimal dither settings */
8243 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8244 FMT_RGB_RANDOM_ENABLE |
8245 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8246 else
8247 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8248 break;
8249 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8250 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8251 /* XXX sort out optimal dither settings */
8252 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8253 FMT_RGB_RANDOM_ENABLE |
8254 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8255 else
8256 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8257 break;
8258 default:
8259 /* not needed */
8260 break;
8261 }
8262
8263 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8264}
8265
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8266/* display watermark setup */
8267/**
8268 * dce8_line_buffer_adjust - Set up the line buffer
8269 *
8270 * @rdev: radeon_device pointer
8271 * @radeon_crtc: the selected display controller
8272 * @mode: the current display mode on the selected display
8273 * controller
8274 *
8275 * Setup up the line buffer allocation for
8276 * the selected display controller (CIK).
8277 * Returns the line buffer size in pixels.
8278 */
8279static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8280 struct radeon_crtc *radeon_crtc,
8281 struct drm_display_mode *mode)
8282{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8283 u32 tmp, buffer_alloc, i;
8284 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8285 /*
8286 * Line Buffer Setup
8287 * There are 6 line buffers, one for each display controllers.
8288 * There are 3 partitions per LB. Select the number of partitions
8289 * to enable based on the display width. For display widths larger
8290 * than 4096, you need use to use 2 display controllers and combine
8291 * them using the stereo blender.
8292 */
8293 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8294 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8295 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8296 buffer_alloc = 2;
8297 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8298 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8299 buffer_alloc = 2;
8300 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8301 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8302 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8303 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8304 DRM_DEBUG_KMS("Mode too big for LB!\n");
8305 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8306 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8307 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8308 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8309 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8310 buffer_alloc = 0;
8311 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8312
8313 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8314 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8315
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8316 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8317 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8318 for (i = 0; i < rdev->usec_timeout; i++) {
8319 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8320 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8321 break;
8322 udelay(1);
8323 }
8324
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8325 if (radeon_crtc->base.enabled && mode) {
8326 switch (tmp) {
8327 case 0:
8328 default:
8329 return 4096 * 2;
8330 case 1:
8331 return 1920 * 2;
8332 case 2:
8333 return 2560 * 2;
8334 }
8335 }
8336
8337 /* controller not enabled, so no lb used */
8338 return 0;
8339}
8340
8341/**
8342 * cik_get_number_of_dram_channels - get the number of dram channels
8343 *
8344 * @rdev: radeon_device pointer
8345 *
8346 * Look up the number of video ram channels (CIK).
8347 * Used for display watermark bandwidth calculations
8348 * Returns the number of dram channels
8349 */
8350static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8351{
8352 u32 tmp = RREG32(MC_SHARED_CHMAP);
8353
8354 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8355 case 0:
8356 default:
8357 return 1;
8358 case 1:
8359 return 2;
8360 case 2:
8361 return 4;
8362 case 3:
8363 return 8;
8364 case 4:
8365 return 3;
8366 case 5:
8367 return 6;
8368 case 6:
8369 return 10;
8370 case 7:
8371 return 12;
8372 case 8:
8373 return 16;
8374 }
8375}
8376
8377struct dce8_wm_params {
8378 u32 dram_channels; /* number of dram channels */
8379 u32 yclk; /* bandwidth per dram data pin in kHz */
8380 u32 sclk; /* engine clock in kHz */
8381 u32 disp_clk; /* display clock in kHz */
8382 u32 src_width; /* viewport width */
8383 u32 active_time; /* active display time in ns */
8384 u32 blank_time; /* blank time in ns */
8385 bool interlaced; /* mode is interlaced */
8386 fixed20_12 vsc; /* vertical scale ratio */
8387 u32 num_heads; /* number of active crtcs */
8388 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8389 u32 lb_size; /* line buffer allocated to pipe */
8390 u32 vtaps; /* vertical scaler taps */
8391};
8392
8393/**
8394 * dce8_dram_bandwidth - get the dram bandwidth
8395 *
8396 * @wm: watermark calculation data
8397 *
8398 * Calculate the raw dram bandwidth (CIK).
8399 * Used for display watermark bandwidth calculations
8400 * Returns the dram bandwidth in MBytes/s
8401 */
8402static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8403{
8404 /* Calculate raw DRAM Bandwidth */
8405 fixed20_12 dram_efficiency; /* 0.7 */
8406 fixed20_12 yclk, dram_channels, bandwidth;
8407 fixed20_12 a;
8408
8409 a.full = dfixed_const(1000);
8410 yclk.full = dfixed_const(wm->yclk);
8411 yclk.full = dfixed_div(yclk, a);
8412 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8413 a.full = dfixed_const(10);
8414 dram_efficiency.full = dfixed_const(7);
8415 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8416 bandwidth.full = dfixed_mul(dram_channels, yclk);
8417 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8418
8419 return dfixed_trunc(bandwidth);
8420}
8421
8422/**
8423 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8424 *
8425 * @wm: watermark calculation data
8426 *
8427 * Calculate the dram bandwidth used for display (CIK).
8428 * Used for display watermark bandwidth calculations
8429 * Returns the dram bandwidth for display in MBytes/s
8430 */
8431static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8432{
8433 /* Calculate DRAM Bandwidth and the part allocated to display. */
8434 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8435 fixed20_12 yclk, dram_channels, bandwidth;
8436 fixed20_12 a;
8437
8438 a.full = dfixed_const(1000);
8439 yclk.full = dfixed_const(wm->yclk);
8440 yclk.full = dfixed_div(yclk, a);
8441 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8442 a.full = dfixed_const(10);
8443 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8444 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8445 bandwidth.full = dfixed_mul(dram_channels, yclk);
8446 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8447
8448 return dfixed_trunc(bandwidth);
8449}
8450
8451/**
8452 * dce8_data_return_bandwidth - get the data return bandwidth
8453 *
8454 * @wm: watermark calculation data
8455 *
8456 * Calculate the data return bandwidth used for display (CIK).
8457 * Used for display watermark bandwidth calculations
8458 * Returns the data return bandwidth in MBytes/s
8459 */
8460static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8461{
8462 /* Calculate the display Data return Bandwidth */
8463 fixed20_12 return_efficiency; /* 0.8 */
8464 fixed20_12 sclk, bandwidth;
8465 fixed20_12 a;
8466
8467 a.full = dfixed_const(1000);
8468 sclk.full = dfixed_const(wm->sclk);
8469 sclk.full = dfixed_div(sclk, a);
8470 a.full = dfixed_const(10);
8471 return_efficiency.full = dfixed_const(8);
8472 return_efficiency.full = dfixed_div(return_efficiency, a);
8473 a.full = dfixed_const(32);
8474 bandwidth.full = dfixed_mul(a, sclk);
8475 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8476
8477 return dfixed_trunc(bandwidth);
8478}
8479
8480/**
8481 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8482 *
8483 * @wm: watermark calculation data
8484 *
8485 * Calculate the dmif bandwidth used for display (CIK).
8486 * Used for display watermark bandwidth calculations
8487 * Returns the dmif bandwidth in MBytes/s
8488 */
8489static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8490{
8491 /* Calculate the DMIF Request Bandwidth */
8492 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8493 fixed20_12 disp_clk, bandwidth;
8494 fixed20_12 a, b;
8495
8496 a.full = dfixed_const(1000);
8497 disp_clk.full = dfixed_const(wm->disp_clk);
8498 disp_clk.full = dfixed_div(disp_clk, a);
8499 a.full = dfixed_const(32);
8500 b.full = dfixed_mul(a, disp_clk);
8501
8502 a.full = dfixed_const(10);
8503 disp_clk_request_efficiency.full = dfixed_const(8);
8504 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8505
8506 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8507
8508 return dfixed_trunc(bandwidth);
8509}
8510
8511/**
8512 * dce8_available_bandwidth - get the min available bandwidth
8513 *
8514 * @wm: watermark calculation data
8515 *
8516 * Calculate the min available bandwidth used for display (CIK).
8517 * Used for display watermark bandwidth calculations
8518 * Returns the min available bandwidth in MBytes/s
8519 */
8520static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8521{
8522 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8523 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8524 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8525 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8526
8527 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8528}
8529
8530/**
8531 * dce8_average_bandwidth - get the average available bandwidth
8532 *
8533 * @wm: watermark calculation data
8534 *
8535 * Calculate the average available bandwidth used for display (CIK).
8536 * Used for display watermark bandwidth calculations
8537 * Returns the average available bandwidth in MBytes/s
8538 */
8539static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8540{
8541 /* Calculate the display mode Average Bandwidth
8542 * DisplayMode should contain the source and destination dimensions,
8543 * timing, etc.
8544 */
8545 fixed20_12 bpp;
8546 fixed20_12 line_time;
8547 fixed20_12 src_width;
8548 fixed20_12 bandwidth;
8549 fixed20_12 a;
8550
8551 a.full = dfixed_const(1000);
8552 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8553 line_time.full = dfixed_div(line_time, a);
8554 bpp.full = dfixed_const(wm->bytes_per_pixel);
8555 src_width.full = dfixed_const(wm->src_width);
8556 bandwidth.full = dfixed_mul(src_width, bpp);
8557 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8558 bandwidth.full = dfixed_div(bandwidth, line_time);
8559
8560 return dfixed_trunc(bandwidth);
8561}
8562
8563/**
8564 * dce8_latency_watermark - get the latency watermark
8565 *
8566 * @wm: watermark calculation data
8567 *
8568 * Calculate the latency watermark (CIK).
8569 * Used for display watermark bandwidth calculations
8570 * Returns the latency watermark in ns
8571 */
8572static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8573{
8574 /* First calculate the latency in ns */
8575 u32 mc_latency = 2000; /* 2000 ns. */
8576 u32 available_bandwidth = dce8_available_bandwidth(wm);
8577 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8578 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8579 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8580 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8581 (wm->num_heads * cursor_line_pair_return_time);
8582 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8583 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8584 u32 tmp, dmif_size = 12288;
8585 fixed20_12 a, b, c;
8586
8587 if (wm->num_heads == 0)
8588 return 0;
8589
8590 a.full = dfixed_const(2);
8591 b.full = dfixed_const(1);
8592 if ((wm->vsc.full > a.full) ||
8593 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8594 (wm->vtaps >= 5) ||
8595 ((wm->vsc.full >= a.full) && wm->interlaced))
8596 max_src_lines_per_dst_line = 4;
8597 else
8598 max_src_lines_per_dst_line = 2;
8599
8600 a.full = dfixed_const(available_bandwidth);
8601 b.full = dfixed_const(wm->num_heads);
8602 a.full = dfixed_div(a, b);
8603
8604 b.full = dfixed_const(mc_latency + 512);
8605 c.full = dfixed_const(wm->disp_clk);
8606 b.full = dfixed_div(b, c);
8607
8608 c.full = dfixed_const(dmif_size);
8609 b.full = dfixed_div(c, b);
8610
8611 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8612
8613 b.full = dfixed_const(1000);
8614 c.full = dfixed_const(wm->disp_clk);
8615 b.full = dfixed_div(c, b);
8616 c.full = dfixed_const(wm->bytes_per_pixel);
8617 b.full = dfixed_mul(b, c);
8618
8619 lb_fill_bw = min(tmp, dfixed_trunc(b));
8620
8621 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8622 b.full = dfixed_const(1000);
8623 c.full = dfixed_const(lb_fill_bw);
8624 b.full = dfixed_div(c, b);
8625 a.full = dfixed_div(a, b);
8626 line_fill_time = dfixed_trunc(a);
8627
8628 if (line_fill_time < wm->active_time)
8629 return latency;
8630 else
8631 return latency + (line_fill_time - wm->active_time);
8632
8633}
8634
8635/**
8636 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8637 * average and available dram bandwidth
8638 *
8639 * @wm: watermark calculation data
8640 *
8641 * Check if the display average bandwidth fits in the display
8642 * dram bandwidth (CIK).
8643 * Used for display watermark bandwidth calculations
8644 * Returns true if the display fits, false if not.
8645 */
8646static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8647{
8648 if (dce8_average_bandwidth(wm) <=
8649 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8650 return true;
8651 else
8652 return false;
8653}
8654
8655/**
8656 * dce8_average_bandwidth_vs_available_bandwidth - check
8657 * average and available bandwidth
8658 *
8659 * @wm: watermark calculation data
8660 *
8661 * Check if the display average bandwidth fits in the display
8662 * available bandwidth (CIK).
8663 * Used for display watermark bandwidth calculations
8664 * Returns true if the display fits, false if not.
8665 */
8666static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8667{
8668 if (dce8_average_bandwidth(wm) <=
8669 (dce8_available_bandwidth(wm) / wm->num_heads))
8670 return true;
8671 else
8672 return false;
8673}
8674
8675/**
8676 * dce8_check_latency_hiding - check latency hiding
8677 *
8678 * @wm: watermark calculation data
8679 *
8680 * Check latency hiding (CIK).
8681 * Used for display watermark bandwidth calculations
8682 * Returns true if the display fits, false if not.
8683 */
8684static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8685{
8686 u32 lb_partitions = wm->lb_size / wm->src_width;
8687 u32 line_time = wm->active_time + wm->blank_time;
8688 u32 latency_tolerant_lines;
8689 u32 latency_hiding;
8690 fixed20_12 a;
8691
8692 a.full = dfixed_const(1);
8693 if (wm->vsc.full > a.full)
8694 latency_tolerant_lines = 1;
8695 else {
8696 if (lb_partitions <= (wm->vtaps + 1))
8697 latency_tolerant_lines = 1;
8698 else
8699 latency_tolerant_lines = 2;
8700 }
8701
8702 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8703
8704 if (dce8_latency_watermark(wm) <= latency_hiding)
8705 return true;
8706 else
8707 return false;
8708}
8709
8710/**
8711 * dce8_program_watermarks - program display watermarks
8712 *
8713 * @rdev: radeon_device pointer
8714 * @radeon_crtc: the selected display controller
8715 * @lb_size: line buffer size
8716 * @num_heads: number of display controllers in use
8717 *
8718 * Calculate and program the display watermarks for the
8719 * selected display controller (CIK).
8720 */
8721static void dce8_program_watermarks(struct radeon_device *rdev,
8722 struct radeon_crtc *radeon_crtc,
8723 u32 lb_size, u32 num_heads)
8724{
8725 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8726 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8727 u32 pixel_period;
8728 u32 line_time = 0;
8729 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8730 u32 tmp, wm_mask;
8731
8732 if (radeon_crtc->base.enabled && num_heads && mode) {
8733 pixel_period = 1000000 / (u32)mode->clock;
8734 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8735
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8736 /* watermark for high clocks */
8737 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8738 rdev->pm.dpm_enabled) {
8739 wm_high.yclk =
8740 radeon_dpm_get_mclk(rdev, false) * 10;
8741 wm_high.sclk =
8742 radeon_dpm_get_sclk(rdev, false) * 10;
8743 } else {
8744 wm_high.yclk = rdev->pm.current_mclk * 10;
8745 wm_high.sclk = rdev->pm.current_sclk * 10;
8746 }
8747
8748 wm_high.disp_clk = mode->clock;
8749 wm_high.src_width = mode->crtc_hdisplay;
8750 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8751 wm_high.blank_time = line_time - wm_high.active_time;
8752 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8753 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8754 wm_high.interlaced = true;
8755 wm_high.vsc = radeon_crtc->vsc;
8756 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8757 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8758 wm_high.vtaps = 2;
8759 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8760 wm_high.lb_size = lb_size;
8761 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8762 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8763
8764 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8765 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8766
8767 /* possibly force display priority to high */
8768 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8769 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8770 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8771 !dce8_check_latency_hiding(&wm_high) ||
8772 (rdev->disp_priority == 2)) {
8773 DRM_DEBUG_KMS("force priority to high\n");
8774 }
8775
8776 /* watermark for low clocks */
8777 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8778 rdev->pm.dpm_enabled) {
8779 wm_low.yclk =
8780 radeon_dpm_get_mclk(rdev, true) * 10;
8781 wm_low.sclk =
8782 radeon_dpm_get_sclk(rdev, true) * 10;
8783 } else {
8784 wm_low.yclk = rdev->pm.current_mclk * 10;
8785 wm_low.sclk = rdev->pm.current_sclk * 10;
8786 }
8787
8788 wm_low.disp_clk = mode->clock;
8789 wm_low.src_width = mode->crtc_hdisplay;
8790 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8791 wm_low.blank_time = line_time - wm_low.active_time;
8792 wm_low.interlaced = false;
8793 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8794 wm_low.interlaced = true;
8795 wm_low.vsc = radeon_crtc->vsc;
8796 wm_low.vtaps = 1;
8797 if (radeon_crtc->rmx_type != RMX_OFF)
8798 wm_low.vtaps = 2;
8799 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8800 wm_low.lb_size = lb_size;
8801 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8802 wm_low.num_heads = num_heads;
8803
8804 /* set for low clocks */
8805 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8806
8807 /* possibly force display priority to high */
8808 /* should really do this at mode validation time... */
8809 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8810 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8811 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8812 (rdev->disp_priority == 2)) {
8813 DRM_DEBUG_KMS("force priority to high\n");
8814 }
8815 }
8816
8817 /* select wm A */
8818 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8819 tmp = wm_mask;
8820 tmp &= ~LATENCY_WATERMARK_MASK(3);
8821 tmp |= LATENCY_WATERMARK_MASK(1);
8822 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8823 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8824 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8825 LATENCY_HIGH_WATERMARK(line_time)));
8826 /* select wm B */
8827 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8828 tmp &= ~LATENCY_WATERMARK_MASK(3);
8829 tmp |= LATENCY_WATERMARK_MASK(2);
8830 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8831 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8832 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8833 LATENCY_HIGH_WATERMARK(line_time)));
8834 /* restore original selection */
8835 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8836
8837 /* save values for DPM */
8838 radeon_crtc->line_time = line_time;
8839 radeon_crtc->wm_high = latency_watermark_a;
8840 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8841}
8842
8843/**
8844 * dce8_bandwidth_update - program display watermarks
8845 *
8846 * @rdev: radeon_device pointer
8847 *
8848 * Calculate and program the display watermarks and line
8849 * buffer allocation (CIK).
8850 */
8851void dce8_bandwidth_update(struct radeon_device *rdev)
8852{
8853 struct drm_display_mode *mode = NULL;
8854 u32 num_heads = 0, lb_size;
8855 int i;
8856
8857 radeon_update_display_priority(rdev);
8858
8859 for (i = 0; i < rdev->num_crtc; i++) {
8860 if (rdev->mode_info.crtcs[i]->base.enabled)
8861 num_heads++;
8862 }
8863 for (i = 0; i < rdev->num_crtc; i++) {
8864 mode = &rdev->mode_info.crtcs[i]->base.mode;
8865 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8866 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8867 }
8868}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8869
8870/**
8871 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8872 *
8873 * @rdev: radeon_device pointer
8874 *
8875 * Fetches a GPU clock counter snapshot (SI).
8876 * Returns the 64 bit clock counter snapshot.
8877 */
8878uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8879{
8880 uint64_t clock;
8881
8882 mutex_lock(&rdev->gpu_clock_mutex);
8883 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8884 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8885 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8886 mutex_unlock(&rdev->gpu_clock_mutex);
8887 return clock;
8888}
8889
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8890static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8891 u32 cntl_reg, u32 status_reg)
8892{
8893 int r, i;
8894 struct atom_clock_dividers dividers;
8895 uint32_t tmp;
8896
8897 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8898 clock, false, &dividers);
8899 if (r)
8900 return r;
8901
8902 tmp = RREG32_SMC(cntl_reg);
8903 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8904 tmp |= dividers.post_divider;
8905 WREG32_SMC(cntl_reg, tmp);
8906
8907 for (i = 0; i < 100; i++) {
8908 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8909 break;
8910 mdelay(10);
8911 }
8912 if (i == 100)
8913 return -ETIMEDOUT;
8914
8915 return 0;
8916}
8917
8918int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8919{
8920 int r = 0;
8921
8922 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8923 if (r)
8924 return r;
8925
8926 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8927 return r;
8928}
8929
Alex Deucher5ad6bf92013-08-22 17:09:06 -0400[diff] [blame]8930int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
8931{
8932 int r, i;
8933 struct atom_clock_dividers dividers;
8934 u32 tmp;
8935
8936 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8937 ecclk, false, &dividers);
8938 if (r)
8939 return r;
8940
8941 for (i = 0; i < 100; i++) {
8942 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8943 break;
8944 mdelay(10);
8945 }
8946 if (i == 100)
8947 return -ETIMEDOUT;
8948
8949 tmp = RREG32_SMC(CG_ECLK_CNTL);
8950 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
8951 tmp |= dividers.post_divider;
8952 WREG32_SMC(CG_ECLK_CNTL, tmp);
8953
8954 for (i = 0; i < 100; i++) {
8955 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8956 break;
8957 mdelay(10);
8958 }
8959 if (i == 100)
8960 return -ETIMEDOUT;
8961
8962 return 0;
8963}
8964
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8965static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8966{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8967 struct pci_dev *root = rdev->pdev->bus->self;
8968 int bridge_pos, gpu_pos;
8969 u32 speed_cntl, mask, current_data_rate;
8970 int ret, i;
8971 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8972
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8973 if (radeon_pcie_gen2 == 0)
8974 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8975
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8976 if (rdev->flags & RADEON_IS_IGP)
8977 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8978
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8979 if (!(rdev->flags & RADEON_IS_PCIE))
8980 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8981
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8982 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8983 if (ret != 0)
8984 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8985
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8986 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8987 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8988
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8989 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8990 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8991 LC_CURRENT_DATA_RATE_SHIFT;
8992 if (mask & DRM_PCIE_SPEED_80) {
8993 if (current_data_rate == 2) {
8994 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8995 return;
8996 }
8997 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8998 } else if (mask & DRM_PCIE_SPEED_50) {
8999 if (current_data_rate == 1) {
9000 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9001 return;
9002 }
9003 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9004 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]9005
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]9006 bridge_pos = pci_pcie_cap(root);
9007 if (!bridge_pos)
9008 return;
9009
9010 gpu_pos = pci_pcie_cap(rdev->pdev);
9011 if (!gpu_pos)
9012 return;
9013
9014 if (mask & DRM_PCIE_SPEED_80) {
9015 /* re-try equalization if gen3 is not already enabled */
9016 if (current_data_rate != 2) {
9017 u16 bridge_cfg, gpu_cfg;
9018 u16 bridge_cfg2, gpu_cfg2;
9019 u32 max_lw, current_lw, tmp;
9020
9021 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9022 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9023
9024 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9025 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9026
9027 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9028 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9029
9030 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9031 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9032 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9033
9034 if (current_lw < max_lw) {
9035 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9036 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9037 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9038 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9039 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9040 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9041 }
9042 }
9043
9044 for (i = 0; i < 10; i++) {
9045 /* check status */
9046 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9047 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9048 break;
9049
9050 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9051 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9052
9053 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9054 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9055
9056 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9057 tmp |= LC_SET_QUIESCE;
9058 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9059
9060 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9061 tmp |= LC_REDO_EQ;
9062 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9063
9064 mdelay(100);
9065
9066 /* linkctl */
9067 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9068 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9069 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9070 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9071
9072 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9073 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9074 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9075 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9076
9077 /* linkctl2 */
9078 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9079 tmp16 &= ~((1 << 4) | (7 << 9));
9080 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9081 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9082
9083 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9084 tmp16 &= ~((1 << 4) | (7 << 9));
9085 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9086 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9087
9088 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9089 tmp &= ~LC_SET_QUIESCE;
9090 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9091 }
9092 }
9093 }
9094
9095 /* set the link speed */
9096 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9097 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9098 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9099
9100 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9101 tmp16 &= ~0xf;
9102 if (mask & DRM_PCIE_SPEED_80)
9103 tmp16 |= 3; /* gen3 */
9104 else if (mask & DRM_PCIE_SPEED_50)
9105 tmp16 |= 2; /* gen2 */
9106 else
9107 tmp16 |= 1; /* gen1 */
9108 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9109
9110 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9111 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9112 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9113
9114 for (i = 0; i < rdev->usec_timeout; i++) {
9115 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9116 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9117 break;
9118 udelay(1);
9119 }
9120}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]9121
9122static void cik_program_aspm(struct radeon_device *rdev)
9123{
9124 u32 data, orig;
9125 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9126 bool disable_clkreq = false;
9127
9128 if (radeon_aspm == 0)
9129 return;
9130
9131 /* XXX double check IGPs */
9132 if (rdev->flags & RADEON_IS_IGP)
9133 return;
9134
9135 if (!(rdev->flags & RADEON_IS_PCIE))
9136 return;
9137
9138 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9139 data &= ~LC_XMIT_N_FTS_MASK;
9140 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9141 if (orig != data)
9142 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9143
9144 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9145 data |= LC_GO_TO_RECOVERY;
9146 if (orig != data)
9147 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9148
9149 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9150 data |= P_IGNORE_EDB_ERR;
9151 if (orig != data)
9152 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9153
9154 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9155 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9156 data |= LC_PMI_TO_L1_DIS;
9157 if (!disable_l0s)
9158 data |= LC_L0S_INACTIVITY(7);
9159
9160 if (!disable_l1) {
9161 data |= LC_L1_INACTIVITY(7);
9162 data &= ~LC_PMI_TO_L1_DIS;
9163 if (orig != data)
9164 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9165
9166 if (!disable_plloff_in_l1) {
9167 bool clk_req_support;
9168
9169 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9170 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9171 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9172 if (orig != data)
9173 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9174
9175 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9176 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9177 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9178 if (orig != data)
9179 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9180
9181 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9182 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9183 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9184 if (orig != data)
9185 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9186
9187 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9188 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9189 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9190 if (orig != data)
9191 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9192
9193 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9194 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9195 data |= LC_DYN_LANES_PWR_STATE(3);
9196 if (orig != data)
9197 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9198
9199 if (!disable_clkreq) {
9200 struct pci_dev *root = rdev->pdev->bus->self;
9201 u32 lnkcap;
9202
9203 clk_req_support = false;
9204 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9205 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9206 clk_req_support = true;
9207 } else {
9208 clk_req_support = false;
9209 }
9210
9211 if (clk_req_support) {
9212 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9213 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9214 if (orig != data)
9215 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9216
9217 orig = data = RREG32_SMC(THM_CLK_CNTL);
9218 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9219 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9220 if (orig != data)
9221 WREG32_SMC(THM_CLK_CNTL, data);
9222
9223 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9224 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9225 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9226 if (orig != data)
9227 WREG32_SMC(MISC_CLK_CTRL, data);
9228
9229 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9230 data &= ~BCLK_AS_XCLK;
9231 if (orig != data)
9232 WREG32_SMC(CG_CLKPIN_CNTL, data);
9233
9234 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9235 data &= ~FORCE_BIF_REFCLK_EN;
9236 if (orig != data)
9237 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9238
9239 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9240 data &= ~MPLL_CLKOUT_SEL_MASK;
9241 data |= MPLL_CLKOUT_SEL(4);
9242 if (orig != data)
9243 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9244 }
9245 }
9246 } else {
9247 if (orig != data)
9248 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9249 }
9250
9251 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9252 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9253 if (orig != data)
9254 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9255
9256 if (!disable_l0s) {
9257 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9258 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9259 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9260 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9261 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9262 data &= ~LC_L0S_INACTIVITY_MASK;
9263 if (orig != data)
9264 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9265 }
9266 }
9267 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]9268}