Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / bbfe90bd4e3e5033e10bda3453c0055a9193c5d7 / . / drivers / gpu / drm / radeon / cik.c
blob: ae92aa041c6a59b4174c05bbfef02fcd60f53d43 [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]78static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]80static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]87
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]124
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]130 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]131 u32 r;
132
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]151}
152
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1532 default:
1533 break;
1534 }
1535}
1536
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
1563 * @offset: byte offset into the aperture
1564 *
1565 * Returns the value in the doorbell aperture at the
1566 * requested offset (CIK).
1567 */
1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1569{
1570 if (offset < rdev->doorbell.size) {
1571 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1572 } else {
1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
1582 * @offset: byte offset into the aperture
1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
1586 * requested offset (CIK).
1587 */
1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1589{
1590 if (offset < rdev->doorbell.size) {
1591 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1592 } else {
1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1594 }
1595}
1596
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
1700static int ci_mc_load_microcode(struct radeon_device *rdev)
1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1802 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1904 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1925 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1932 }
1933
1934out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1952 }
1953 return err;
1954}
1955
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1994 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1995
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
2430 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2431 }
2432 } else if (num_pipe_configs == 4) {
2433 if (num_rbs == 4) {
2434 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2435 switch (reg_offset) {
2436 case 0:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2441 break;
2442 case 1:
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2447 break;
2448 case 2:
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2453 break;
2454 case 3:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2459 break;
2460 case 4:
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 TILE_SPLIT(split_equal_to_row_size));
2465 break;
2466 case 5:
2467 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2469 break;
2470 case 6:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2475 break;
2476 case 7:
2477 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2478 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2479 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2480 TILE_SPLIT(split_equal_to_row_size));
2481 break;
2482 case 8:
2483 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2484 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2485 break;
2486 case 9:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2489 break;
2490 case 10:
2491 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2492 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2493 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2494 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2495 break;
2496 case 11:
2497 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2498 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2499 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2500 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2501 break;
2502 case 12:
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2507 break;
2508 case 13:
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2511 break;
2512 case 14:
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517 break;
2518 case 16:
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523 break;
2524 case 17:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529 break;
2530 case 27:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2533 break;
2534 case 28:
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2539 break;
2540 case 29:
2541 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2542 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2545 break;
2546 case 30:
2547 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2551 break;
2552 default:
2553 gb_tile_moden = 0;
2554 break;
2555 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2556 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2557 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2558 }
2559 } else if (num_rbs < 4) {
2560 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2561 switch (reg_offset) {
2562 case 0:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2567 break;
2568 case 1:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2573 break;
2574 case 2:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2579 break;
2580 case 3:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2585 break;
2586 case 4:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(split_equal_to_row_size));
2591 break;
2592 case 5:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2595 break;
2596 case 6:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2601 break;
2602 case 7:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(split_equal_to_row_size));
2607 break;
2608 case 8:
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2610 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2611 break;
2612 case 9:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2615 break;
2616 case 10:
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2620 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2621 break;
2622 case 11:
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2627 break;
2628 case 12:
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2632 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633 break;
2634 case 13:
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2637 break;
2638 case 14:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 case 16:
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2646 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649 break;
2650 case 17:
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 break;
2656 case 27:
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2659 break;
2660 case 28:
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2665 break;
2666 case 29:
2667 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2668 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2671 break;
2672 case 30:
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2677 break;
2678 default:
2679 gb_tile_moden = 0;
2680 break;
2681 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2682 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2683 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2684 }
2685 }
2686 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2687 switch (reg_offset) {
2688 case 0:
2689 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692 NUM_BANKS(ADDR_SURF_16_BANK));
2693 break;
2694 case 1:
2695 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698 NUM_BANKS(ADDR_SURF_16_BANK));
2699 break;
2700 case 2:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 3:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_16_BANK));
2711 break;
2712 case 4:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716 NUM_BANKS(ADDR_SURF_16_BANK));
2717 break;
2718 case 5:
2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722 NUM_BANKS(ADDR_SURF_8_BANK));
2723 break;
2724 case 6:
2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728 NUM_BANKS(ADDR_SURF_4_BANK));
2729 break;
2730 case 8:
2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2734 NUM_BANKS(ADDR_SURF_16_BANK));
2735 break;
2736 case 9:
2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2740 NUM_BANKS(ADDR_SURF_16_BANK));
2741 break;
2742 case 10:
2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746 NUM_BANKS(ADDR_SURF_16_BANK));
2747 break;
2748 case 11:
2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752 NUM_BANKS(ADDR_SURF_16_BANK));
2753 break;
2754 case 12:
2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2758 NUM_BANKS(ADDR_SURF_16_BANK));
2759 break;
2760 case 13:
2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2764 NUM_BANKS(ADDR_SURF_8_BANK));
2765 break;
2766 case 14:
2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2770 NUM_BANKS(ADDR_SURF_4_BANK));
2771 break;
2772 default:
2773 gb_tile_moden = 0;
2774 break;
2775 }
2776 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2777 }
2778 } else if (num_pipe_configs == 2) {
2779 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2780 switch (reg_offset) {
2781 case 0:
2782 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784 PIPE_CONFIG(ADDR_SURF_P2) |
2785 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2786 break;
2787 case 1:
2788 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2790 PIPE_CONFIG(ADDR_SURF_P2) |
2791 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2792 break;
2793 case 2:
2794 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2796 PIPE_CONFIG(ADDR_SURF_P2) |
2797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2798 break;
2799 case 3:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2802 PIPE_CONFIG(ADDR_SURF_P2) |
2803 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2804 break;
2805 case 4:
2806 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P2) |
2809 TILE_SPLIT(split_equal_to_row_size));
2810 break;
2811 case 5:
2812 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2813 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2814 break;
2815 case 6:
2816 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2817 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2818 PIPE_CONFIG(ADDR_SURF_P2) |
2819 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2820 break;
2821 case 7:
2822 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2823 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2824 PIPE_CONFIG(ADDR_SURF_P2) |
2825 TILE_SPLIT(split_equal_to_row_size));
2826 break;
2827 case 8:
2828 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2829 break;
2830 case 9:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2833 break;
2834 case 10:
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2839 break;
2840 case 11:
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2842 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2843 PIPE_CONFIG(ADDR_SURF_P2) |
2844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2845 break;
2846 case 12:
2847 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2849 PIPE_CONFIG(ADDR_SURF_P2) |
2850 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2851 break;
2852 case 13:
2853 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2854 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2855 break;
2856 case 14:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P2) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 16:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P2) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 case 17:
2869 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2870 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2871 PIPE_CONFIG(ADDR_SURF_P2) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 break;
2874 case 27:
2875 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2876 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2877 break;
2878 case 28:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883 break;
2884 case 29:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2889 break;
2890 case 30:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P2) |
2894 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2895 break;
2896 default:
2897 gb_tile_moden = 0;
2898 break;
2899 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2900 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2901 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2902 }
2903 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2904 switch (reg_offset) {
2905 case 0:
2906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 NUM_BANKS(ADDR_SURF_16_BANK));
2910 break;
2911 case 1:
2912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915 NUM_BANKS(ADDR_SURF_16_BANK));
2916 break;
2917 case 2:
2918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922 break;
2923 case 3:
2924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927 NUM_BANKS(ADDR_SURF_16_BANK));
2928 break;
2929 case 4:
2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2933 NUM_BANKS(ADDR_SURF_16_BANK));
2934 break;
2935 case 5:
2936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2939 NUM_BANKS(ADDR_SURF_16_BANK));
2940 break;
2941 case 6:
2942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2945 NUM_BANKS(ADDR_SURF_8_BANK));
2946 break;
2947 case 8:
2948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2951 NUM_BANKS(ADDR_SURF_16_BANK));
2952 break;
2953 case 9:
2954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2957 NUM_BANKS(ADDR_SURF_16_BANK));
2958 break;
2959 case 10:
2960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963 NUM_BANKS(ADDR_SURF_16_BANK));
2964 break;
2965 case 11:
2966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2969 NUM_BANKS(ADDR_SURF_16_BANK));
2970 break;
2971 case 12:
2972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975 NUM_BANKS(ADDR_SURF_16_BANK));
2976 break;
2977 case 13:
2978 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2981 NUM_BANKS(ADDR_SURF_16_BANK));
2982 break;
2983 case 14:
2984 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2987 NUM_BANKS(ADDR_SURF_8_BANK));
2988 break;
2989 default:
2990 gb_tile_moden = 0;
2991 break;
2992 }
2993 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2994 }
2995 } else
2996 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2997}
2998
2999/**
3000 * cik_select_se_sh - select which SE, SH to address
3001 *
3002 * @rdev: radeon_device pointer
3003 * @se_num: shader engine to address
3004 * @sh_num: sh block to address
3005 *
3006 * Select which SE, SH combinations to address. Certain
3007 * registers are instanced per SE or SH. 0xffffffff means
3008 * broadcast to all SEs or SHs (CIK).
3009 */
3010static void cik_select_se_sh(struct radeon_device *rdev,
3011 u32 se_num, u32 sh_num)
3012{
3013 u32 data = INSTANCE_BROADCAST_WRITES;
3014
3015 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3016 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3017 else if (se_num == 0xffffffff)
3018 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3019 else if (sh_num == 0xffffffff)
3020 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3021 else
3022 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3023 WREG32(GRBM_GFX_INDEX, data);
3024}
3025
3026/**
3027 * cik_create_bitmask - create a bitmask
3028 *
3029 * @bit_width: length of the mask
3030 *
3031 * create a variable length bit mask (CIK).
3032 * Returns the bitmask.
3033 */
3034static u32 cik_create_bitmask(u32 bit_width)
3035{
3036 u32 i, mask = 0;
3037
3038 for (i = 0; i < bit_width; i++) {
3039 mask <<= 1;
3040 mask |= 1;
3041 }
3042 return mask;
3043}
3044
3045/**
3046 * cik_select_se_sh - select which SE, SH to address
3047 *
3048 * @rdev: radeon_device pointer
3049 * @max_rb_num: max RBs (render backends) for the asic
3050 * @se_num: number of SEs (shader engines) for the asic
3051 * @sh_per_se: number of SH blocks per SE for the asic
3052 *
3053 * Calculates the bitmask of disabled RBs (CIK).
3054 * Returns the disabled RB bitmask.
3055 */
3056static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3057 u32 max_rb_num, u32 se_num,
3058 u32 sh_per_se)
3059{
3060 u32 data, mask;
3061
3062 data = RREG32(CC_RB_BACKEND_DISABLE);
3063 if (data & 1)
3064 data &= BACKEND_DISABLE_MASK;
3065 else
3066 data = 0;
3067 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3068
3069 data >>= BACKEND_DISABLE_SHIFT;
3070
3071 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3072
3073 return data & mask;
3074}
3075
3076/**
3077 * cik_setup_rb - setup the RBs on the asic
3078 *
3079 * @rdev: radeon_device pointer
3080 * @se_num: number of SEs (shader engines) for the asic
3081 * @sh_per_se: number of SH blocks per SE for the asic
3082 * @max_rb_num: max RBs (render backends) for the asic
3083 *
3084 * Configures per-SE/SH RB registers (CIK).
3085 */
3086static void cik_setup_rb(struct radeon_device *rdev,
3087 u32 se_num, u32 sh_per_se,
3088 u32 max_rb_num)
3089{
3090 int i, j;
3091 u32 data, mask;
3092 u32 disabled_rbs = 0;
3093 u32 enabled_rbs = 0;
3094
3095 for (i = 0; i < se_num; i++) {
3096 for (j = 0; j < sh_per_se; j++) {
3097 cik_select_se_sh(rdev, i, j);
3098 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3099 if (rdev->family == CHIP_HAWAII)
3100 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3101 else
3102 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3103 }
3104 }
3105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3106
3107 mask = 1;
3108 for (i = 0; i < max_rb_num; i++) {
3109 if (!(disabled_rbs & mask))
3110 enabled_rbs |= mask;
3111 mask <<= 1;
3112 }
3113
3114 for (i = 0; i < se_num; i++) {
3115 cik_select_se_sh(rdev, i, 0xffffffff);
3116 data = 0;
3117 for (j = 0; j < sh_per_se; j++) {
3118 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3119 case 0:
3120 if (j == 0)
3121 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3122 else
3123 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3124 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3125 case 1:
3126 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3127 break;
3128 case 2:
3129 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3130 break;
3131 case 3:
3132 default:
3133 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3134 break;
3135 }
3136 enabled_rbs >>= 2;
3137 }
3138 WREG32(PA_SC_RASTER_CONFIG, data);
3139 }
3140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141}
3142
3143/**
3144 * cik_gpu_init - setup the 3D engine
3145 *
3146 * @rdev: radeon_device pointer
3147 *
3148 * Configures the 3D engine and tiling configuration
3149 * registers so that the 3D engine is usable.
3150 */
3151static void cik_gpu_init(struct radeon_device *rdev)
3152{
3153 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3154 u32 mc_shared_chmap, mc_arb_ramcfg;
3155 u32 hdp_host_path_cntl;
3156 u32 tmp;
3157 int i, j;
3158
3159 switch (rdev->family) {
3160 case CHIP_BONAIRE:
3161 rdev->config.cik.max_shader_engines = 2;
3162 rdev->config.cik.max_tile_pipes = 4;
3163 rdev->config.cik.max_cu_per_sh = 7;
3164 rdev->config.cik.max_sh_per_se = 1;
3165 rdev->config.cik.max_backends_per_se = 2;
3166 rdev->config.cik.max_texture_channel_caches = 4;
3167 rdev->config.cik.max_gprs = 256;
3168 rdev->config.cik.max_gs_threads = 32;
3169 rdev->config.cik.max_hw_contexts = 8;
3170
3171 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3172 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3173 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3174 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3175 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3176 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3177 case CHIP_HAWAII:
3178 rdev->config.cik.max_shader_engines = 4;
3179 rdev->config.cik.max_tile_pipes = 16;
3180 rdev->config.cik.max_cu_per_sh = 11;
3181 rdev->config.cik.max_sh_per_se = 1;
3182 rdev->config.cik.max_backends_per_se = 4;
3183 rdev->config.cik.max_texture_channel_caches = 16;
3184 rdev->config.cik.max_gprs = 256;
3185 rdev->config.cik.max_gs_threads = 32;
3186 rdev->config.cik.max_hw_contexts = 8;
3187
3188 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3193 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3194 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3195 rdev->config.cik.max_shader_engines = 1;
3196 rdev->config.cik.max_tile_pipes = 4;
3197 if ((rdev->pdev->device == 0x1304) ||
3198 (rdev->pdev->device == 0x1305) ||
3199 (rdev->pdev->device == 0x130C) ||
3200 (rdev->pdev->device == 0x130F) ||
3201 (rdev->pdev->device == 0x1310) ||
3202 (rdev->pdev->device == 0x1311) ||
3203 (rdev->pdev->device == 0x131C)) {
3204 rdev->config.cik.max_cu_per_sh = 8;
3205 rdev->config.cik.max_backends_per_se = 2;
3206 } else if ((rdev->pdev->device == 0x1309) ||
3207 (rdev->pdev->device == 0x130A) ||
3208 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3209 (rdev->pdev->device == 0x1313) ||
3210 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3211 rdev->config.cik.max_cu_per_sh = 6;
3212 rdev->config.cik.max_backends_per_se = 2;
3213 } else if ((rdev->pdev->device == 0x1306) ||
3214 (rdev->pdev->device == 0x1307) ||
3215 (rdev->pdev->device == 0x130B) ||
3216 (rdev->pdev->device == 0x130E) ||
3217 (rdev->pdev->device == 0x1315) ||
3218 (rdev->pdev->device == 0x131B)) {
3219 rdev->config.cik.max_cu_per_sh = 4;
3220 rdev->config.cik.max_backends_per_se = 1;
3221 } else {
3222 rdev->config.cik.max_cu_per_sh = 3;
3223 rdev->config.cik.max_backends_per_se = 1;
3224 }
3225 rdev->config.cik.max_sh_per_se = 1;
3226 rdev->config.cik.max_texture_channel_caches = 4;
3227 rdev->config.cik.max_gprs = 256;
3228 rdev->config.cik.max_gs_threads = 16;
3229 rdev->config.cik.max_hw_contexts = 8;
3230
3231 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3232 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3233 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3234 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3235 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3236 break;
3237 case CHIP_KABINI:
3238 default:
3239 rdev->config.cik.max_shader_engines = 1;
3240 rdev->config.cik.max_tile_pipes = 2;
3241 rdev->config.cik.max_cu_per_sh = 2;
3242 rdev->config.cik.max_sh_per_se = 1;
3243 rdev->config.cik.max_backends_per_se = 1;
3244 rdev->config.cik.max_texture_channel_caches = 2;
3245 rdev->config.cik.max_gprs = 256;
3246 rdev->config.cik.max_gs_threads = 16;
3247 rdev->config.cik.max_hw_contexts = 8;
3248
3249 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3250 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3251 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3252 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3253 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3254 break;
3255 }
3256
3257 /* Initialize HDP */
3258 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3259 WREG32((0x2c14 + j), 0x00000000);
3260 WREG32((0x2c18 + j), 0x00000000);
3261 WREG32((0x2c1c + j), 0x00000000);
3262 WREG32((0x2c20 + j), 0x00000000);
3263 WREG32((0x2c24 + j), 0x00000000);
3264 }
3265
3266 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3267
3268 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3269
3270 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3271 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3272
3273 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3274 rdev->config.cik.mem_max_burst_length_bytes = 256;
3275 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3276 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3277 if (rdev->config.cik.mem_row_size_in_kb > 4)
3278 rdev->config.cik.mem_row_size_in_kb = 4;
3279 /* XXX use MC settings? */
3280 rdev->config.cik.shader_engine_tile_size = 32;
3281 rdev->config.cik.num_gpus = 1;
3282 rdev->config.cik.multi_gpu_tile_size = 64;
3283
3284 /* fix up row size */
3285 gb_addr_config &= ~ROW_SIZE_MASK;
3286 switch (rdev->config.cik.mem_row_size_in_kb) {
3287 case 1:
3288 default:
3289 gb_addr_config |= ROW_SIZE(0);
3290 break;
3291 case 2:
3292 gb_addr_config |= ROW_SIZE(1);
3293 break;
3294 case 4:
3295 gb_addr_config |= ROW_SIZE(2);
3296 break;
3297 }
3298
3299 /* setup tiling info dword. gb_addr_config is not adequate since it does
3300 * not have bank info, so create a custom tiling dword.
3301 * bits 3:0 num_pipes
3302 * bits 7:4 num_banks
3303 * bits 11:8 group_size
3304 * bits 15:12 row_size
3305 */
3306 rdev->config.cik.tile_config = 0;
3307 switch (rdev->config.cik.num_tile_pipes) {
3308 case 1:
3309 rdev->config.cik.tile_config |= (0 << 0);
3310 break;
3311 case 2:
3312 rdev->config.cik.tile_config |= (1 << 0);
3313 break;
3314 case 4:
3315 rdev->config.cik.tile_config |= (2 << 0);
3316 break;
3317 case 8:
3318 default:
3319 /* XXX what about 12? */
3320 rdev->config.cik.tile_config |= (3 << 0);
3321 break;
3322 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3323 rdev->config.cik.tile_config |=
3324 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3325 rdev->config.cik.tile_config |=
3326 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3327 rdev->config.cik.tile_config |=
3328 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3329
3330 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3331 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3332 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3333 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3334 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3335 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3336 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3337 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3338
3339 cik_tiling_mode_table_init(rdev);
3340
3341 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3342 rdev->config.cik.max_sh_per_se,
3343 rdev->config.cik.max_backends_per_se);
3344
3345 /* set HW defaults for 3D engine */
3346 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3347
3348 WREG32(SX_DEBUG_1, 0x20);
3349
3350 WREG32(TA_CNTL_AUX, 0x00010000);
3351
3352 tmp = RREG32(SPI_CONFIG_CNTL);
3353 tmp |= 0x03000000;
3354 WREG32(SPI_CONFIG_CNTL, tmp);
3355
3356 WREG32(SQ_CONFIG, 1);
3357
3358 WREG32(DB_DEBUG, 0);
3359
3360 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3361 tmp |= 0x00000400;
3362 WREG32(DB_DEBUG2, tmp);
3363
3364 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3365 tmp |= 0x00020200;
3366 WREG32(DB_DEBUG3, tmp);
3367
3368 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3369 tmp |= 0x00018208;
3370 WREG32(CB_HW_CONTROL, tmp);
3371
3372 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3373
3374 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3375 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3376 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3377 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3378
3379 WREG32(VGT_NUM_INSTANCES, 1);
3380
3381 WREG32(CP_PERFMON_CNTL, 0);
3382
3383 WREG32(SQ_CONFIG, 0);
3384
3385 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3386 FORCE_EOV_MAX_REZ_CNT(255)));
3387
3388 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3389 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3390
3391 WREG32(VGT_GS_VERTEX_REUSE, 16);
3392 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3393
3394 tmp = RREG32(HDP_MISC_CNTL);
3395 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3396 WREG32(HDP_MISC_CNTL, tmp);
3397
3398 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3399 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3400
3401 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3402 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3403
3404 udelay(50);
3405}
3406
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3407/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3408 * GPU scratch registers helpers function.
3409 */
3410/**
3411 * cik_scratch_init - setup driver info for CP scratch regs
3412 *
3413 * @rdev: radeon_device pointer
3414 *
3415 * Set up the number and offset of the CP scratch registers.
3416 * NOTE: use of CP scratch registers is a legacy inferface and
3417 * is not used by default on newer asics (r6xx+). On newer asics,
3418 * memory buffers are used for fences rather than scratch regs.
3419 */
3420static void cik_scratch_init(struct radeon_device *rdev)
3421{
3422 int i;
3423
3424 rdev->scratch.num_reg = 7;
3425 rdev->scratch.reg_base = SCRATCH_REG0;
3426 for (i = 0; i < rdev->scratch.num_reg; i++) {
3427 rdev->scratch.free[i] = true;
3428 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3429 }
3430}
3431
3432/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3433 * cik_ring_test - basic gfx ring test
3434 *
3435 * @rdev: radeon_device pointer
3436 * @ring: radeon_ring structure holding ring information
3437 *
3438 * Allocate a scratch register and write to it using the gfx ring (CIK).
3439 * Provides a basic gfx ring test to verify that the ring is working.
3440 * Used by cik_cp_gfx_resume();
3441 * Returns 0 on success, error on failure.
3442 */
3443int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3444{
3445 uint32_t scratch;
3446 uint32_t tmp = 0;
3447 unsigned i;
3448 int r;
3449
3450 r = radeon_scratch_get(rdev, &scratch);
3451 if (r) {
3452 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3453 return r;
3454 }
3455 WREG32(scratch, 0xCAFEDEAD);
3456 r = radeon_ring_lock(rdev, ring, 3);
3457 if (r) {
3458 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3459 radeon_scratch_free(rdev, scratch);
3460 return r;
3461 }
3462 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3463 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3464 radeon_ring_write(ring, 0xDEADBEEF);
3465 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3466
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3467 for (i = 0; i < rdev->usec_timeout; i++) {
3468 tmp = RREG32(scratch);
3469 if (tmp == 0xDEADBEEF)
3470 break;
3471 DRM_UDELAY(1);
3472 }
3473 if (i < rdev->usec_timeout) {
3474 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3475 } else {
3476 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3477 ring->idx, scratch, tmp);
3478 r = -EINVAL;
3479 }
3480 radeon_scratch_free(rdev, scratch);
3481 return r;
3482}
3483
3484/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3485 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3486 *
3487 * @rdev: radeon_device pointer
3488 * @fence: radeon fence object
3489 *
3490 * Emits a fence sequnce number on the gfx ring and flushes
3491 * GPU caches.
3492 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3493void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3494 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3495{
3496 struct radeon_ring *ring = &rdev->ring[fence->ring];
3497 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3498
3499 /* EVENT_WRITE_EOP - flush caches, send int */
3500 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3501 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3502 EOP_TC_ACTION_EN |
3503 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3504 EVENT_INDEX(5)));
3505 radeon_ring_write(ring, addr & 0xfffffffc);
3506 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3507 radeon_ring_write(ring, fence->seq);
3508 radeon_ring_write(ring, 0);
3509 /* HDP flush */
3510 /* We should be using the new WAIT_REG_MEM special op packet here
3511 * but it causes the CP to hang
3512 */
3513 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3514 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3515 WRITE_DATA_DST_SEL(0)));
3516 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3517 radeon_ring_write(ring, 0);
3518 radeon_ring_write(ring, 0);
3519}
3520
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3521/**
3522 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3523 *
3524 * @rdev: radeon_device pointer
3525 * @fence: radeon fence object
3526 *
3527 * Emits a fence sequnce number on the compute ring and flushes
3528 * GPU caches.
3529 */
3530void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3531 struct radeon_fence *fence)
3532{
3533 struct radeon_ring *ring = &rdev->ring[fence->ring];
3534 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3535
3536 /* RELEASE_MEM - flush caches, send int */
3537 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3538 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3539 EOP_TC_ACTION_EN |
3540 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3541 EVENT_INDEX(5)));
3542 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3543 radeon_ring_write(ring, addr & 0xfffffffc);
3544 radeon_ring_write(ring, upper_32_bits(addr));
3545 radeon_ring_write(ring, fence->seq);
3546 radeon_ring_write(ring, 0);
3547 /* HDP flush */
3548 /* We should be using the new WAIT_REG_MEM special op packet here
3549 * but it causes the CP to hang
3550 */
3551 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3552 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3553 WRITE_DATA_DST_SEL(0)));
3554 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3555 radeon_ring_write(ring, 0);
3556 radeon_ring_write(ring, 0);
3557}
3558
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3559void cik_semaphore_ring_emit(struct radeon_device *rdev,
3560 struct radeon_ring *ring,
3561 struct radeon_semaphore *semaphore,
3562 bool emit_wait)
3563{
3564 uint64_t addr = semaphore->gpu_addr;
3565 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3566
3567 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3568 radeon_ring_write(ring, addr & 0xffffffff);
3569 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3570}
3571
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3572/**
3573 * cik_copy_cpdma - copy pages using the CP DMA engine
3574 *
3575 * @rdev: radeon_device pointer
3576 * @src_offset: src GPU address
3577 * @dst_offset: dst GPU address
3578 * @num_gpu_pages: number of GPU pages to xfer
3579 * @fence: radeon fence object
3580 *
3581 * Copy GPU paging using the CP DMA engine (CIK+).
3582 * Used by the radeon ttm implementation to move pages if
3583 * registered as the asic copy callback.
3584 */
3585int cik_copy_cpdma(struct radeon_device *rdev,
3586 uint64_t src_offset, uint64_t dst_offset,
3587 unsigned num_gpu_pages,
3588 struct radeon_fence **fence)
3589{
3590 struct radeon_semaphore *sem = NULL;
3591 int ring_index = rdev->asic->copy.blit_ring_index;
3592 struct radeon_ring *ring = &rdev->ring[ring_index];
3593 u32 size_in_bytes, cur_size_in_bytes, control;
3594 int i, num_loops;
3595 int r = 0;
3596
3597 r = radeon_semaphore_create(rdev, &sem);
3598 if (r) {
3599 DRM_ERROR("radeon: moving bo (%d).\n", r);
3600 return r;
3601 }
3602
3603 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3604 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3605 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3606 if (r) {
3607 DRM_ERROR("radeon: moving bo (%d).\n", r);
3608 radeon_semaphore_free(rdev, &sem, NULL);
3609 return r;
3610 }
3611
3612 if (radeon_fence_need_sync(*fence, ring->idx)) {
3613 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3614 ring->idx);
3615 radeon_fence_note_sync(*fence, ring->idx);
3616 } else {
3617 radeon_semaphore_free(rdev, &sem, NULL);
3618 }
3619
3620 for (i = 0; i < num_loops; i++) {
3621 cur_size_in_bytes = size_in_bytes;
3622 if (cur_size_in_bytes > 0x1fffff)
3623 cur_size_in_bytes = 0x1fffff;
3624 size_in_bytes -= cur_size_in_bytes;
3625 control = 0;
3626 if (size_in_bytes == 0)
3627 control |= PACKET3_DMA_DATA_CP_SYNC;
3628 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3629 radeon_ring_write(ring, control);
3630 radeon_ring_write(ring, lower_32_bits(src_offset));
3631 radeon_ring_write(ring, upper_32_bits(src_offset));
3632 radeon_ring_write(ring, lower_32_bits(dst_offset));
3633 radeon_ring_write(ring, upper_32_bits(dst_offset));
3634 radeon_ring_write(ring, cur_size_in_bytes);
3635 src_offset += cur_size_in_bytes;
3636 dst_offset += cur_size_in_bytes;
3637 }
3638
3639 r = radeon_fence_emit(rdev, fence, ring->idx);
3640 if (r) {
3641 radeon_ring_unlock_undo(rdev, ring);
3642 return r;
3643 }
3644
3645 radeon_ring_unlock_commit(rdev, ring);
3646 radeon_semaphore_free(rdev, &sem, *fence);
3647
3648 return r;
3649}
3650
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3651/*
3652 * IB stuff
3653 */
3654/**
3655 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3656 *
3657 * @rdev: radeon_device pointer
3658 * @ib: radeon indirect buffer object
3659 *
3660 * Emits an DE (drawing engine) or CE (constant engine) IB
3661 * on the gfx ring. IBs are usually generated by userspace
3662 * acceleration drivers and submitted to the kernel for
3663 * sheduling on the ring. This function schedules the IB
3664 * on the gfx ring for execution by the GPU.
3665 */
3666void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3667{
3668 struct radeon_ring *ring = &rdev->ring[ib->ring];
3669 u32 header, control = INDIRECT_BUFFER_VALID;
3670
3671 if (ib->is_const_ib) {
3672 /* set switch buffer packet before const IB */
3673 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3674 radeon_ring_write(ring, 0);
3675
3676 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3677 } else {
3678 u32 next_rptr;
3679 if (ring->rptr_save_reg) {
3680 next_rptr = ring->wptr + 3 + 4;
3681 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3682 radeon_ring_write(ring, ((ring->rptr_save_reg -
3683 PACKET3_SET_UCONFIG_REG_START) >> 2));
3684 radeon_ring_write(ring, next_rptr);
3685 } else if (rdev->wb.enabled) {
3686 next_rptr = ring->wptr + 5 + 4;
3687 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3688 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3689 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3690 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3691 radeon_ring_write(ring, next_rptr);
3692 }
3693
3694 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3695 }
3696
3697 control |= ib->length_dw |
3698 (ib->vm ? (ib->vm->id << 24) : 0);
3699
3700 radeon_ring_write(ring, header);
3701 radeon_ring_write(ring,
3702#ifdef __BIG_ENDIAN
3703 (2 << 0) |
3704#endif
3705 (ib->gpu_addr & 0xFFFFFFFC));
3706 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3707 radeon_ring_write(ring, control);
3708}
3709
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3710/**
3711 * cik_ib_test - basic gfx ring IB test
3712 *
3713 * @rdev: radeon_device pointer
3714 * @ring: radeon_ring structure holding ring information
3715 *
3716 * Allocate an IB and execute it on the gfx ring (CIK).
3717 * Provides a basic gfx ring test to verify that IBs are working.
3718 * Returns 0 on success, error on failure.
3719 */
3720int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3721{
3722 struct radeon_ib ib;
3723 uint32_t scratch;
3724 uint32_t tmp = 0;
3725 unsigned i;
3726 int r;
3727
3728 r = radeon_scratch_get(rdev, &scratch);
3729 if (r) {
3730 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3731 return r;
3732 }
3733 WREG32(scratch, 0xCAFEDEAD);
3734 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3735 if (r) {
3736 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3737 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3738 return r;
3739 }
3740 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3741 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3742 ib.ptr[2] = 0xDEADBEEF;
3743 ib.length_dw = 3;
3744 r = radeon_ib_schedule(rdev, &ib, NULL);
3745 if (r) {
3746 radeon_scratch_free(rdev, scratch);
3747 radeon_ib_free(rdev, &ib);
3748 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3749 return r;
3750 }
3751 r = radeon_fence_wait(ib.fence, false);
3752 if (r) {
3753 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3754 radeon_scratch_free(rdev, scratch);
3755 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3756 return r;
3757 }
3758 for (i = 0; i < rdev->usec_timeout; i++) {
3759 tmp = RREG32(scratch);
3760 if (tmp == 0xDEADBEEF)
3761 break;
3762 DRM_UDELAY(1);
3763 }
3764 if (i < rdev->usec_timeout) {
3765 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3766 } else {
3767 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3768 scratch, tmp);
3769 r = -EINVAL;
3770 }
3771 radeon_scratch_free(rdev, scratch);
3772 radeon_ib_free(rdev, &ib);
3773 return r;
3774}
3775
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3776/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3777 * CP.
3778 * On CIK, gfx and compute now have independant command processors.
3779 *
3780 * GFX
3781 * Gfx consists of a single ring and can process both gfx jobs and
3782 * compute jobs. The gfx CP consists of three microengines (ME):
3783 * PFP - Pre-Fetch Parser
3784 * ME - Micro Engine
3785 * CE - Constant Engine
3786 * The PFP and ME make up what is considered the Drawing Engine (DE).
3787 * The CE is an asynchronous engine used for updating buffer desciptors
3788 * used by the DE so that they can be loaded into cache in parallel
3789 * while the DE is processing state update packets.
3790 *
3791 * Compute
3792 * The compute CP consists of two microengines (ME):
3793 * MEC1 - Compute MicroEngine 1
3794 * MEC2 - Compute MicroEngine 2
3795 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3796 * The queues are exposed to userspace and are programmed directly
3797 * by the compute runtime.
3798 */
3799/**
3800 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3801 *
3802 * @rdev: radeon_device pointer
3803 * @enable: enable or disable the MEs
3804 *
3805 * Halts or unhalts the gfx MEs.
3806 */
3807static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3808{
3809 if (enable)
3810 WREG32(CP_ME_CNTL, 0);
3811 else {
3812 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3813 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3814 }
3815 udelay(50);
3816}
3817
3818/**
3819 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3820 *
3821 * @rdev: radeon_device pointer
3822 *
3823 * Loads the gfx PFP, ME, and CE ucode.
3824 * Returns 0 for success, -EINVAL if the ucode is not available.
3825 */
3826static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3827{
3828 const __be32 *fw_data;
3829 int i;
3830
3831 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3832 return -EINVAL;
3833
3834 cik_cp_gfx_enable(rdev, false);
3835
3836 /* PFP */
3837 fw_data = (const __be32 *)rdev->pfp_fw->data;
3838 WREG32(CP_PFP_UCODE_ADDR, 0);
3839 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3840 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3841 WREG32(CP_PFP_UCODE_ADDR, 0);
3842
3843 /* CE */
3844 fw_data = (const __be32 *)rdev->ce_fw->data;
3845 WREG32(CP_CE_UCODE_ADDR, 0);
3846 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3847 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3848 WREG32(CP_CE_UCODE_ADDR, 0);
3849
3850 /* ME */
3851 fw_data = (const __be32 *)rdev->me_fw->data;
3852 WREG32(CP_ME_RAM_WADDR, 0);
3853 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3854 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3855 WREG32(CP_ME_RAM_WADDR, 0);
3856
3857 WREG32(CP_PFP_UCODE_ADDR, 0);
3858 WREG32(CP_CE_UCODE_ADDR, 0);
3859 WREG32(CP_ME_RAM_WADDR, 0);
3860 WREG32(CP_ME_RAM_RADDR, 0);
3861 return 0;
3862}
3863
3864/**
3865 * cik_cp_gfx_start - start the gfx ring
3866 *
3867 * @rdev: radeon_device pointer
3868 *
3869 * Enables the ring and loads the clear state context and other
3870 * packets required to init the ring.
3871 * Returns 0 for success, error for failure.
3872 */
3873static int cik_cp_gfx_start(struct radeon_device *rdev)
3874{
3875 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3876 int r, i;
3877
3878 /* init the CP */
3879 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3880 WREG32(CP_ENDIAN_SWAP, 0);
3881 WREG32(CP_DEVICE_ID, 1);
3882
3883 cik_cp_gfx_enable(rdev, true);
3884
3885 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3886 if (r) {
3887 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3888 return r;
3889 }
3890
3891 /* init the CE partitions. CE only used for gfx on CIK */
3892 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3893 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3894 radeon_ring_write(ring, 0xc000);
3895 radeon_ring_write(ring, 0xc000);
3896
3897 /* setup clear context state */
3898 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3899 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3900
3901 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3902 radeon_ring_write(ring, 0x80000000);
3903 radeon_ring_write(ring, 0x80000000);
3904
3905 for (i = 0; i < cik_default_size; i++)
3906 radeon_ring_write(ring, cik_default_state[i]);
3907
3908 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3909 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3910
3911 /* set clear context state */
3912 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3913 radeon_ring_write(ring, 0);
3914
3915 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3916 radeon_ring_write(ring, 0x00000316);
3917 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3918 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3919
3920 radeon_ring_unlock_commit(rdev, ring);
3921
3922 return 0;
3923}
3924
3925/**
3926 * cik_cp_gfx_fini - stop the gfx ring
3927 *
3928 * @rdev: radeon_device pointer
3929 *
3930 * Stop the gfx ring and tear down the driver ring
3931 * info.
3932 */
3933static void cik_cp_gfx_fini(struct radeon_device *rdev)
3934{
3935 cik_cp_gfx_enable(rdev, false);
3936 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3937}
3938
3939/**
3940 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3941 *
3942 * @rdev: radeon_device pointer
3943 *
3944 * Program the location and size of the gfx ring buffer
3945 * and test it to make sure it's working.
3946 * Returns 0 for success, error for failure.
3947 */
3948static int cik_cp_gfx_resume(struct radeon_device *rdev)
3949{
3950 struct radeon_ring *ring;
3951 u32 tmp;
3952 u32 rb_bufsz;
3953 u64 rb_addr;
3954 int r;
3955
3956 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3957 if (rdev->family != CHIP_HAWAII)
3958 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3959
3960 /* Set the write pointer delay */
3961 WREG32(CP_RB_WPTR_DELAY, 0);
3962
3963 /* set the RB to use vmid 0 */
3964 WREG32(CP_RB_VMID, 0);
3965
3966 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3967
3968 /* ring 0 - compute and gfx */
3969 /* Set ring buffer size */
3970 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3971 rb_bufsz = order_base_2(ring->ring_size / 8);
3972 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3973#ifdef __BIG_ENDIAN
3974 tmp |= BUF_SWAP_32BIT;
3975#endif
3976 WREG32(CP_RB0_CNTL, tmp);
3977
3978 /* Initialize the ring buffer's read and write pointers */
3979 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3980 ring->wptr = 0;
3981 WREG32(CP_RB0_WPTR, ring->wptr);
3982
3983 /* set the wb address wether it's enabled or not */
3984 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3985 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3986
3987 /* scratch register shadowing is no longer supported */
3988 WREG32(SCRATCH_UMSK, 0);
3989
3990 if (!rdev->wb.enabled)
3991 tmp |= RB_NO_UPDATE;
3992
3993 mdelay(1);
3994 WREG32(CP_RB0_CNTL, tmp);
3995
3996 rb_addr = ring->gpu_addr >> 8;
3997 WREG32(CP_RB0_BASE, rb_addr);
3998 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3999
4000 ring->rptr = RREG32(CP_RB0_RPTR);
4001
4002 /* start the ring */
4003 cik_cp_gfx_start(rdev);
4004 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4005 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4006 if (r) {
4007 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4008 return r;
4009 }
4010 return 0;
4011}
4012
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4013u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4014 struct radeon_ring *ring)
4015{
4016 u32 rptr;
4017
4018
4019
4020 if (rdev->wb.enabled) {
4021 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4022 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4023 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4024 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4025 rptr = RREG32(CP_HQD_PQ_RPTR);
4026 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4027 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4028 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4029
4030 return rptr;
4031}
4032
4033u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4034 struct radeon_ring *ring)
4035{
4036 u32 wptr;
4037
4038 if (rdev->wb.enabled) {
4039 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4040 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4041 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4042 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4043 wptr = RREG32(CP_HQD_PQ_WPTR);
4044 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4045 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4046 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4047
4048 return wptr;
4049}
4050
4051void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4052 struct radeon_ring *ring)
4053{
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]4054 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4055 WDOORBELL32(ring->doorbell_offset, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4056}
4057
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4058/**
4059 * cik_cp_compute_enable - enable/disable the compute CP MEs
4060 *
4061 * @rdev: radeon_device pointer
4062 * @enable: enable or disable the MEs
4063 *
4064 * Halts or unhalts the compute MEs.
4065 */
4066static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4067{
4068 if (enable)
4069 WREG32(CP_MEC_CNTL, 0);
4070 else
4071 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4072 udelay(50);
4073}
4074
4075/**
4076 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4077 *
4078 * @rdev: radeon_device pointer
4079 *
4080 * Loads the compute MEC1&2 ucode.
4081 * Returns 0 for success, -EINVAL if the ucode is not available.
4082 */
4083static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4084{
4085 const __be32 *fw_data;
4086 int i;
4087
4088 if (!rdev->mec_fw)
4089 return -EINVAL;
4090
4091 cik_cp_compute_enable(rdev, false);
4092
4093 /* MEC1 */
4094 fw_data = (const __be32 *)rdev->mec_fw->data;
4095 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4096 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4097 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4098 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4099
4100 if (rdev->family == CHIP_KAVERI) {
4101 /* MEC2 */
4102 fw_data = (const __be32 *)rdev->mec_fw->data;
4103 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4104 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4105 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4106 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4107 }
4108
4109 return 0;
4110}
4111
4112/**
4113 * cik_cp_compute_start - start the compute queues
4114 *
4115 * @rdev: radeon_device pointer
4116 *
4117 * Enable the compute queues.
4118 * Returns 0 for success, error for failure.
4119 */
4120static int cik_cp_compute_start(struct radeon_device *rdev)
4121{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4122 cik_cp_compute_enable(rdev, true);
4123
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4124 return 0;
4125}
4126
4127/**
4128 * cik_cp_compute_fini - stop the compute queues
4129 *
4130 * @rdev: radeon_device pointer
4131 *
4132 * Stop the compute queues and tear down the driver queue
4133 * info.
4134 */
4135static void cik_cp_compute_fini(struct radeon_device *rdev)
4136{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4137 int i, idx, r;
4138
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4139 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4140
4141 for (i = 0; i < 2; i++) {
4142 if (i == 0)
4143 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4144 else
4145 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4146
4147 if (rdev->ring[idx].mqd_obj) {
4148 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4149 if (unlikely(r != 0))
4150 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4151
4152 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4153 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4154
4155 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4156 rdev->ring[idx].mqd_obj = NULL;
4157 }
4158 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4159}
4160
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4161static void cik_mec_fini(struct radeon_device *rdev)
4162{
4163 int r;
4164
4165 if (rdev->mec.hpd_eop_obj) {
4166 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4167 if (unlikely(r != 0))
4168 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4169 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4170 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4171
4172 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4173 rdev->mec.hpd_eop_obj = NULL;
4174 }
4175}
4176
4177#define MEC_HPD_SIZE 2048
4178
4179static int cik_mec_init(struct radeon_device *rdev)
4180{
4181 int r;
4182 u32 *hpd;
4183
4184 /*
4185 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4186 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4187 */
4188 if (rdev->family == CHIP_KAVERI)
4189 rdev->mec.num_mec = 2;
4190 else
4191 rdev->mec.num_mec = 1;
4192 rdev->mec.num_pipe = 4;
4193 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4194
4195 if (rdev->mec.hpd_eop_obj == NULL) {
4196 r = radeon_bo_create(rdev,
4197 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4198 PAGE_SIZE, true,
4199 RADEON_GEM_DOMAIN_GTT, NULL,
4200 &rdev->mec.hpd_eop_obj);
4201 if (r) {
4202 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4203 return r;
4204 }
4205 }
4206
4207 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4208 if (unlikely(r != 0)) {
4209 cik_mec_fini(rdev);
4210 return r;
4211 }
4212 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4213 &rdev->mec.hpd_eop_gpu_addr);
4214 if (r) {
4215 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4216 cik_mec_fini(rdev);
4217 return r;
4218 }
4219 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4220 if (r) {
4221 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4222 cik_mec_fini(rdev);
4223 return r;
4224 }
4225
4226 /* clear memory. Not sure if this is required or not */
4227 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4228
4229 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4230 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4231
4232 return 0;
4233}
4234
4235struct hqd_registers
4236{
4237 u32 cp_mqd_base_addr;
4238 u32 cp_mqd_base_addr_hi;
4239 u32 cp_hqd_active;
4240 u32 cp_hqd_vmid;
4241 u32 cp_hqd_persistent_state;
4242 u32 cp_hqd_pipe_priority;
4243 u32 cp_hqd_queue_priority;
4244 u32 cp_hqd_quantum;
4245 u32 cp_hqd_pq_base;
4246 u32 cp_hqd_pq_base_hi;
4247 u32 cp_hqd_pq_rptr;
4248 u32 cp_hqd_pq_rptr_report_addr;
4249 u32 cp_hqd_pq_rptr_report_addr_hi;
4250 u32 cp_hqd_pq_wptr_poll_addr;
4251 u32 cp_hqd_pq_wptr_poll_addr_hi;
4252 u32 cp_hqd_pq_doorbell_control;
4253 u32 cp_hqd_pq_wptr;
4254 u32 cp_hqd_pq_control;
4255 u32 cp_hqd_ib_base_addr;
4256 u32 cp_hqd_ib_base_addr_hi;
4257 u32 cp_hqd_ib_rptr;
4258 u32 cp_hqd_ib_control;
4259 u32 cp_hqd_iq_timer;
4260 u32 cp_hqd_iq_rptr;
4261 u32 cp_hqd_dequeue_request;
4262 u32 cp_hqd_dma_offload;
4263 u32 cp_hqd_sema_cmd;
4264 u32 cp_hqd_msg_type;
4265 u32 cp_hqd_atomic0_preop_lo;
4266 u32 cp_hqd_atomic0_preop_hi;
4267 u32 cp_hqd_atomic1_preop_lo;
4268 u32 cp_hqd_atomic1_preop_hi;
4269 u32 cp_hqd_hq_scheduler0;
4270 u32 cp_hqd_hq_scheduler1;
4271 u32 cp_mqd_control;
4272};
4273
4274struct bonaire_mqd
4275{
4276 u32 header;
4277 u32 dispatch_initiator;
4278 u32 dimensions[3];
4279 u32 start_idx[3];
4280 u32 num_threads[3];
4281 u32 pipeline_stat_enable;
4282 u32 perf_counter_enable;
4283 u32 pgm[2];
4284 u32 tba[2];
4285 u32 tma[2];
4286 u32 pgm_rsrc[2];
4287 u32 vmid;
4288 u32 resource_limits;
4289 u32 static_thread_mgmt01[2];
4290 u32 tmp_ring_size;
4291 u32 static_thread_mgmt23[2];
4292 u32 restart[3];
4293 u32 thread_trace_enable;
4294 u32 reserved1;
4295 u32 user_data[16];
4296 u32 vgtcs_invoke_count[2];
4297 struct hqd_registers queue_state;
4298 u32 dequeue_cntr;
4299 u32 interrupt_queue[64];
4300};
4301
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4302/**
4303 * cik_cp_compute_resume - setup the compute queue registers
4304 *
4305 * @rdev: radeon_device pointer
4306 *
4307 * Program the compute queues and test them to make sure they
4308 * are working.
4309 * Returns 0 for success, error for failure.
4310 */
4311static int cik_cp_compute_resume(struct radeon_device *rdev)
4312{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4313 int r, i, idx;
4314 u32 tmp;
4315 bool use_doorbell = true;
4316 u64 hqd_gpu_addr;
4317 u64 mqd_gpu_addr;
4318 u64 eop_gpu_addr;
4319 u64 wb_gpu_addr;
4320 u32 *buf;
4321 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4322
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4323 r = cik_cp_compute_start(rdev);
4324 if (r)
4325 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4326
4327 /* fix up chicken bits */
4328 tmp = RREG32(CP_CPF_DEBUG);
4329 tmp |= (1 << 23);
4330 WREG32(CP_CPF_DEBUG, tmp);
4331
4332 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4333 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4334 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4335 int me = (i < 4) ? 1 : 2;
4336 int pipe = (i < 4) ? i : (i - 4);
4337
4338 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4339
4340 cik_srbm_select(rdev, me, pipe, 0, 0);
4341
4342 /* write the EOP addr */
4343 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4344 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4345
4346 /* set the VMID assigned */
4347 WREG32(CP_HPD_EOP_VMID, 0);
4348
4349 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4350 tmp = RREG32(CP_HPD_EOP_CONTROL);
4351 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4352 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4353 WREG32(CP_HPD_EOP_CONTROL, tmp);
4354 }
4355 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4356 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4357
4358 /* init the queues. Just two for now. */
4359 for (i = 0; i < 2; i++) {
4360 if (i == 0)
4361 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4362 else
4363 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4364
4365 if (rdev->ring[idx].mqd_obj == NULL) {
4366 r = radeon_bo_create(rdev,
4367 sizeof(struct bonaire_mqd),
4368 PAGE_SIZE, true,
4369 RADEON_GEM_DOMAIN_GTT, NULL,
4370 &rdev->ring[idx].mqd_obj);
4371 if (r) {
4372 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4373 return r;
4374 }
4375 }
4376
4377 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4378 if (unlikely(r != 0)) {
4379 cik_cp_compute_fini(rdev);
4380 return r;
4381 }
4382 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4383 &mqd_gpu_addr);
4384 if (r) {
4385 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4386 cik_cp_compute_fini(rdev);
4387 return r;
4388 }
4389 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4390 if (r) {
4391 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4392 cik_cp_compute_fini(rdev);
4393 return r;
4394 }
4395
4396 /* doorbell offset */
4397 rdev->ring[idx].doorbell_offset =
4398 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
4399
4400 /* init the mqd struct */
4401 memset(buf, 0, sizeof(struct bonaire_mqd));
4402
4403 mqd = (struct bonaire_mqd *)buf;
4404 mqd->header = 0xC0310800;
4405 mqd->static_thread_mgmt01[0] = 0xffffffff;
4406 mqd->static_thread_mgmt01[1] = 0xffffffff;
4407 mqd->static_thread_mgmt23[0] = 0xffffffff;
4408 mqd->static_thread_mgmt23[1] = 0xffffffff;
4409
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4410 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4411 cik_srbm_select(rdev, rdev->ring[idx].me,
4412 rdev->ring[idx].pipe,
4413 rdev->ring[idx].queue, 0);
4414
4415 /* disable wptr polling */
4416 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4417 tmp &= ~WPTR_POLL_EN;
4418 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4419
4420 /* enable doorbell? */
4421 mqd->queue_state.cp_hqd_pq_doorbell_control =
4422 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4423 if (use_doorbell)
4424 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4425 else
4426 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4427 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4428 mqd->queue_state.cp_hqd_pq_doorbell_control);
4429
4430 /* disable the queue if it's active */
4431 mqd->queue_state.cp_hqd_dequeue_request = 0;
4432 mqd->queue_state.cp_hqd_pq_rptr = 0;
4433 mqd->queue_state.cp_hqd_pq_wptr= 0;
4434 if (RREG32(CP_HQD_ACTIVE) & 1) {
4435 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4436 for (i = 0; i < rdev->usec_timeout; i++) {
4437 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4438 break;
4439 udelay(1);
4440 }
4441 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4442 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4443 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4444 }
4445
4446 /* set the pointer to the MQD */
4447 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4448 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4449 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4450 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4451 /* set MQD vmid to 0 */
4452 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4453 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4454 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4455
4456 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4457 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4458 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4459 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4460 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4461 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4462
4463 /* set up the HQD, this is similar to CP_RB0_CNTL */
4464 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4465 mqd->queue_state.cp_hqd_pq_control &=
4466 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4467
4468 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4469 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4470 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4471 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4472#ifdef __BIG_ENDIAN
4473 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4474#endif
4475 mqd->queue_state.cp_hqd_pq_control &=
4476 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4477 mqd->queue_state.cp_hqd_pq_control |=
4478 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4479 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4480
4481 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4482 if (i == 0)
4483 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4484 else
4485 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4486 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4487 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4488 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4489 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4490 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4491
4492 /* set the wb address wether it's enabled or not */
4493 if (i == 0)
4494 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4495 else
4496 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4497 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4498 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4499 upper_32_bits(wb_gpu_addr) & 0xffff;
4500 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4501 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4502 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4503 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4504
4505 /* enable the doorbell if requested */
4506 if (use_doorbell) {
4507 mqd->queue_state.cp_hqd_pq_doorbell_control =
4508 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4509 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4510 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4511 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4512 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4513 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4514 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4515
4516 } else {
4517 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4518 }
4519 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4520 mqd->queue_state.cp_hqd_pq_doorbell_control);
4521
4522 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4523 rdev->ring[idx].wptr = 0;
4524 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4525 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4526 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4527 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4528
4529 /* set the vmid for the queue */
4530 mqd->queue_state.cp_hqd_vmid = 0;
4531 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4532
4533 /* activate the queue */
4534 mqd->queue_state.cp_hqd_active = 1;
4535 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4536
4537 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4538 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4539
4540 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4541 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4542
4543 rdev->ring[idx].ready = true;
4544 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4545 if (r)
4546 rdev->ring[idx].ready = false;
4547 }
4548
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4549 return 0;
4550}
4551
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4552static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4553{
4554 cik_cp_gfx_enable(rdev, enable);
4555 cik_cp_compute_enable(rdev, enable);
4556}
4557
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4558static int cik_cp_load_microcode(struct radeon_device *rdev)
4559{
4560 int r;
4561
4562 r = cik_cp_gfx_load_microcode(rdev);
4563 if (r)
4564 return r;
4565 r = cik_cp_compute_load_microcode(rdev);
4566 if (r)
4567 return r;
4568
4569 return 0;
4570}
4571
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4572static void cik_cp_fini(struct radeon_device *rdev)
4573{
4574 cik_cp_gfx_fini(rdev);
4575 cik_cp_compute_fini(rdev);
4576}
4577
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4578static int cik_cp_resume(struct radeon_device *rdev)
4579{
4580 int r;
4581
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4582 cik_enable_gui_idle_interrupt(rdev, false);
4583
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4584 r = cik_cp_load_microcode(rdev);
4585 if (r)
4586 return r;
4587
4588 r = cik_cp_gfx_resume(rdev);
4589 if (r)
4590 return r;
4591 r = cik_cp_compute_resume(rdev);
4592 if (r)
4593 return r;
4594
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4595 cik_enable_gui_idle_interrupt(rdev, true);
4596
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4597 return 0;
4598}
4599
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4600static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4601{
4602 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4603 RREG32(GRBM_STATUS));
4604 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4605 RREG32(GRBM_STATUS2));
4606 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4607 RREG32(GRBM_STATUS_SE0));
4608 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4609 RREG32(GRBM_STATUS_SE1));
4610 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4611 RREG32(GRBM_STATUS_SE2));
4612 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4613 RREG32(GRBM_STATUS_SE3));
4614 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4615 RREG32(SRBM_STATUS));
4616 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4617 RREG32(SRBM_STATUS2));
4618 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4619 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4620 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4621 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4622 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4623 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4624 RREG32(CP_STALLED_STAT1));
4625 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4626 RREG32(CP_STALLED_STAT2));
4627 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4628 RREG32(CP_STALLED_STAT3));
4629 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4630 RREG32(CP_CPF_BUSY_STAT));
4631 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4632 RREG32(CP_CPF_STALLED_STAT1));
4633 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4634 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4635 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4636 RREG32(CP_CPC_STALLED_STAT1));
4637 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4638}
4639
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4640/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4641 * cik_gpu_check_soft_reset - check which blocks are busy
4642 *
4643 * @rdev: radeon_device pointer
4644 *
4645 * Check which blocks are busy and return the relevant reset
4646 * mask to be used by cik_gpu_soft_reset().
4647 * Returns a mask of the blocks to be reset.
4648 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4649u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4650{
4651 u32 reset_mask = 0;
4652 u32 tmp;
4653
4654 /* GRBM_STATUS */
4655 tmp = RREG32(GRBM_STATUS);
4656 if (tmp & (PA_BUSY | SC_BUSY |
4657 BCI_BUSY | SX_BUSY |
4658 TA_BUSY | VGT_BUSY |
4659 DB_BUSY | CB_BUSY |
4660 GDS_BUSY | SPI_BUSY |
4661 IA_BUSY | IA_BUSY_NO_DMA))
4662 reset_mask |= RADEON_RESET_GFX;
4663
4664 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4665 reset_mask |= RADEON_RESET_CP;
4666
4667 /* GRBM_STATUS2 */
4668 tmp = RREG32(GRBM_STATUS2);
4669 if (tmp & RLC_BUSY)
4670 reset_mask |= RADEON_RESET_RLC;
4671
4672 /* SDMA0_STATUS_REG */
4673 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4674 if (!(tmp & SDMA_IDLE))
4675 reset_mask |= RADEON_RESET_DMA;
4676
4677 /* SDMA1_STATUS_REG */
4678 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4679 if (!(tmp & SDMA_IDLE))
4680 reset_mask |= RADEON_RESET_DMA1;
4681
4682 /* SRBM_STATUS2 */
4683 tmp = RREG32(SRBM_STATUS2);
4684 if (tmp & SDMA_BUSY)
4685 reset_mask |= RADEON_RESET_DMA;
4686
4687 if (tmp & SDMA1_BUSY)
4688 reset_mask |= RADEON_RESET_DMA1;
4689
4690 /* SRBM_STATUS */
4691 tmp = RREG32(SRBM_STATUS);
4692
4693 if (tmp & IH_BUSY)
4694 reset_mask |= RADEON_RESET_IH;
4695
4696 if (tmp & SEM_BUSY)
4697 reset_mask |= RADEON_RESET_SEM;
4698
4699 if (tmp & GRBM_RQ_PENDING)
4700 reset_mask |= RADEON_RESET_GRBM;
4701
4702 if (tmp & VMC_BUSY)
4703 reset_mask |= RADEON_RESET_VMC;
4704
4705 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4706 MCC_BUSY | MCD_BUSY))
4707 reset_mask |= RADEON_RESET_MC;
4708
4709 if (evergreen_is_display_hung(rdev))
4710 reset_mask |= RADEON_RESET_DISPLAY;
4711
4712 /* Skip MC reset as it's mostly likely not hung, just busy */
4713 if (reset_mask & RADEON_RESET_MC) {
4714 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4715 reset_mask &= ~RADEON_RESET_MC;
4716 }
4717
4718 return reset_mask;
4719}
4720
4721/**
4722 * cik_gpu_soft_reset - soft reset GPU
4723 *
4724 * @rdev: radeon_device pointer
4725 * @reset_mask: mask of which blocks to reset
4726 *
4727 * Soft reset the blocks specified in @reset_mask.
4728 */
4729static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4730{
4731 struct evergreen_mc_save save;
4732 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4733 u32 tmp;
4734
4735 if (reset_mask == 0)
4736 return;
4737
4738 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4739
4740 cik_print_gpu_status_regs(rdev);
4741 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4742 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4743 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4744 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4745
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4746 /* disable CG/PG */
4747 cik_fini_pg(rdev);
4748 cik_fini_cg(rdev);
4749
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4750 /* stop the rlc */
4751 cik_rlc_stop(rdev);
4752
4753 /* Disable GFX parsing/prefetching */
4754 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4755
4756 /* Disable MEC parsing/prefetching */
4757 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4758
4759 if (reset_mask & RADEON_RESET_DMA) {
4760 /* sdma0 */
4761 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4762 tmp |= SDMA_HALT;
4763 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4764 }
4765 if (reset_mask & RADEON_RESET_DMA1) {
4766 /* sdma1 */
4767 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4768 tmp |= SDMA_HALT;
4769 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4770 }
4771
4772 evergreen_mc_stop(rdev, &save);
4773 if (evergreen_mc_wait_for_idle(rdev)) {
4774 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4775 }
4776
4777 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4778 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4779
4780 if (reset_mask & RADEON_RESET_CP) {
4781 grbm_soft_reset |= SOFT_RESET_CP;
4782
4783 srbm_soft_reset |= SOFT_RESET_GRBM;
4784 }
4785
4786 if (reset_mask & RADEON_RESET_DMA)
4787 srbm_soft_reset |= SOFT_RESET_SDMA;
4788
4789 if (reset_mask & RADEON_RESET_DMA1)
4790 srbm_soft_reset |= SOFT_RESET_SDMA1;
4791
4792 if (reset_mask & RADEON_RESET_DISPLAY)
4793 srbm_soft_reset |= SOFT_RESET_DC;
4794
4795 if (reset_mask & RADEON_RESET_RLC)
4796 grbm_soft_reset |= SOFT_RESET_RLC;
4797
4798 if (reset_mask & RADEON_RESET_SEM)
4799 srbm_soft_reset |= SOFT_RESET_SEM;
4800
4801 if (reset_mask & RADEON_RESET_IH)
4802 srbm_soft_reset |= SOFT_RESET_IH;
4803
4804 if (reset_mask & RADEON_RESET_GRBM)
4805 srbm_soft_reset |= SOFT_RESET_GRBM;
4806
4807 if (reset_mask & RADEON_RESET_VMC)
4808 srbm_soft_reset |= SOFT_RESET_VMC;
4809
4810 if (!(rdev->flags & RADEON_IS_IGP)) {
4811 if (reset_mask & RADEON_RESET_MC)
4812 srbm_soft_reset |= SOFT_RESET_MC;
4813 }
4814
4815 if (grbm_soft_reset) {
4816 tmp = RREG32(GRBM_SOFT_RESET);
4817 tmp |= grbm_soft_reset;
4818 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4819 WREG32(GRBM_SOFT_RESET, tmp);
4820 tmp = RREG32(GRBM_SOFT_RESET);
4821
4822 udelay(50);
4823
4824 tmp &= ~grbm_soft_reset;
4825 WREG32(GRBM_SOFT_RESET, tmp);
4826 tmp = RREG32(GRBM_SOFT_RESET);
4827 }
4828
4829 if (srbm_soft_reset) {
4830 tmp = RREG32(SRBM_SOFT_RESET);
4831 tmp |= srbm_soft_reset;
4832 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4833 WREG32(SRBM_SOFT_RESET, tmp);
4834 tmp = RREG32(SRBM_SOFT_RESET);
4835
4836 udelay(50);
4837
4838 tmp &= ~srbm_soft_reset;
4839 WREG32(SRBM_SOFT_RESET, tmp);
4840 tmp = RREG32(SRBM_SOFT_RESET);
4841 }
4842
4843 /* Wait a little for things to settle down */
4844 udelay(50);
4845
4846 evergreen_mc_resume(rdev, &save);
4847 udelay(50);
4848
4849 cik_print_gpu_status_regs(rdev);
4850}
4851
4852/**
4853 * cik_asic_reset - soft reset GPU
4854 *
4855 * @rdev: radeon_device pointer
4856 *
4857 * Look up which blocks are hung and attempt
4858 * to reset them.
4859 * Returns 0 for success.
4860 */
4861int cik_asic_reset(struct radeon_device *rdev)
4862{
4863 u32 reset_mask;
4864
4865 reset_mask = cik_gpu_check_soft_reset(rdev);
4866
4867 if (reset_mask)
4868 r600_set_bios_scratch_engine_hung(rdev, true);
4869
4870 cik_gpu_soft_reset(rdev, reset_mask);
4871
4872 reset_mask = cik_gpu_check_soft_reset(rdev);
4873
4874 if (!reset_mask)
4875 r600_set_bios_scratch_engine_hung(rdev, false);
4876
4877 return 0;
4878}
4879
4880/**
4881 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4882 *
4883 * @rdev: radeon_device pointer
4884 * @ring: radeon_ring structure holding ring information
4885 *
4886 * Check if the 3D engine is locked up (CIK).
4887 * Returns true if the engine is locked, false if not.
4888 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4889bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4890{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4891 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4892
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4893 if (!(reset_mask & (RADEON_RESET_GFX |
4894 RADEON_RESET_COMPUTE |
4895 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4896 radeon_ring_lockup_update(ring);
4897 return false;
4898 }
4899 /* force CP activities */
4900 radeon_ring_force_activity(rdev, ring);
4901 return radeon_ring_test_lockup(rdev, ring);
4902}
4903
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4904/* MC */
4905/**
4906 * cik_mc_program - program the GPU memory controller
4907 *
4908 * @rdev: radeon_device pointer
4909 *
4910 * Set the location of vram, gart, and AGP in the GPU's
4911 * physical address space (CIK).
4912 */
4913static void cik_mc_program(struct radeon_device *rdev)
4914{
4915 struct evergreen_mc_save save;
4916 u32 tmp;
4917 int i, j;
4918
4919 /* Initialize HDP */
4920 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4921 WREG32((0x2c14 + j), 0x00000000);
4922 WREG32((0x2c18 + j), 0x00000000);
4923 WREG32((0x2c1c + j), 0x00000000);
4924 WREG32((0x2c20 + j), 0x00000000);
4925 WREG32((0x2c24 + j), 0x00000000);
4926 }
4927 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4928
4929 evergreen_mc_stop(rdev, &save);
4930 if (radeon_mc_wait_for_idle(rdev)) {
4931 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4932 }
4933 /* Lockout access through VGA aperture*/
4934 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4935 /* Update configuration */
4936 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4937 rdev->mc.vram_start >> 12);
4938 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4939 rdev->mc.vram_end >> 12);
4940 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4941 rdev->vram_scratch.gpu_addr >> 12);
4942 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4943 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4944 WREG32(MC_VM_FB_LOCATION, tmp);
4945 /* XXX double check these! */
4946 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4947 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4948 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4949 WREG32(MC_VM_AGP_BASE, 0);
4950 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4951 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4952 if (radeon_mc_wait_for_idle(rdev)) {
4953 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4954 }
4955 evergreen_mc_resume(rdev, &save);
4956 /* we need to own VRAM, so turn off the VGA renderer here
4957 * to stop it overwriting our objects */
4958 rv515_vga_render_disable(rdev);
4959}
4960
4961/**
4962 * cik_mc_init - initialize the memory controller driver params
4963 *
4964 * @rdev: radeon_device pointer
4965 *
4966 * Look up the amount of vram, vram width, and decide how to place
4967 * vram and gart within the GPU's physical address space (CIK).
4968 * Returns 0 for success.
4969 */
4970static int cik_mc_init(struct radeon_device *rdev)
4971{
4972 u32 tmp;
4973 int chansize, numchan;
4974
4975 /* Get VRAM informations */
4976 rdev->mc.vram_is_ddr = true;
4977 tmp = RREG32(MC_ARB_RAMCFG);
4978 if (tmp & CHANSIZE_MASK) {
4979 chansize = 64;
4980 } else {
4981 chansize = 32;
4982 }
4983 tmp = RREG32(MC_SHARED_CHMAP);
4984 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4985 case 0:
4986 default:
4987 numchan = 1;
4988 break;
4989 case 1:
4990 numchan = 2;
4991 break;
4992 case 2:
4993 numchan = 4;
4994 break;
4995 case 3:
4996 numchan = 8;
4997 break;
4998 case 4:
4999 numchan = 3;
5000 break;
5001 case 5:
5002 numchan = 6;
5003 break;
5004 case 6:
5005 numchan = 10;
5006 break;
5007 case 7:
5008 numchan = 12;
5009 break;
5010 case 8:
5011 numchan = 16;
5012 break;
5013 }
5014 rdev->mc.vram_width = numchan * chansize;
5015 /* Could aper size report 0 ? */
5016 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5017 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5018 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5019 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5020 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5021 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5022 si_vram_gtt_location(rdev, &rdev->mc);
5023 radeon_update_bandwidth_info(rdev);
5024
5025 return 0;
5026}
5027
5028/*
5029 * GART
5030 * VMID 0 is the physical GPU addresses as used by the kernel.
5031 * VMIDs 1-15 are used for userspace clients and are handled
5032 * by the radeon vm/hsa code.
5033 */
5034/**
5035 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5036 *
5037 * @rdev: radeon_device pointer
5038 *
5039 * Flush the TLB for the VMID 0 page table (CIK).
5040 */
5041void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5042{
5043 /* flush hdp cache */
5044 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5045
5046 /* bits 0-15 are the VM contexts0-15 */
5047 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5048}
5049
5050/**
5051 * cik_pcie_gart_enable - gart enable
5052 *
5053 * @rdev: radeon_device pointer
5054 *
5055 * This sets up the TLBs, programs the page tables for VMID0,
5056 * sets up the hw for VMIDs 1-15 which are allocated on
5057 * demand, and sets up the global locations for the LDS, GDS,
5058 * and GPUVM for FSA64 clients (CIK).
5059 * Returns 0 for success, errors for failure.
5060 */
5061static int cik_pcie_gart_enable(struct radeon_device *rdev)
5062{
5063 int r, i;
5064
5065 if (rdev->gart.robj == NULL) {
5066 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5067 return -EINVAL;
5068 }
5069 r = radeon_gart_table_vram_pin(rdev);
5070 if (r)
5071 return r;
5072 radeon_gart_restore(rdev);
5073 /* Setup TLB control */
5074 WREG32(MC_VM_MX_L1_TLB_CNTL,
5075 (0xA << 7) |
5076 ENABLE_L1_TLB |
5077 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5078 ENABLE_ADVANCED_DRIVER_MODEL |
5079 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5080 /* Setup L2 cache */
5081 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5082 ENABLE_L2_FRAGMENT_PROCESSING |
5083 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5084 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5085 EFFECTIVE_L2_QUEUE_SIZE(7) |
5086 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5087 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5088 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5089 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5090 /* setup context0 */
5091 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5092 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5093 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5094 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5095 (u32)(rdev->dummy_page.addr >> 12));
5096 WREG32(VM_CONTEXT0_CNTL2, 0);
5097 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5098 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5099
5100 WREG32(0x15D4, 0);
5101 WREG32(0x15D8, 0);
5102 WREG32(0x15DC, 0);
5103
5104 /* empty context1-15 */
5105 /* FIXME start with 4G, once using 2 level pt switch to full
5106 * vm size space
5107 */
5108 /* set vm size, must be a multiple of 4 */
5109 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5110 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5111 for (i = 1; i < 16; i++) {
5112 if (i < 8)
5113 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5114 rdev->gart.table_addr >> 12);
5115 else
5116 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5117 rdev->gart.table_addr >> 12);
5118 }
5119
5120 /* enable context1-15 */
5121 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5122 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5123 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5124 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5125 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5126 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5127 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5128 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5129 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5130 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5131 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5132 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5133 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5134 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5135 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5136 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5137
5138 /* TC cache setup ??? */
5139 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5140 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5141 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5142
5143 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5144 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5145 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5146 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5147 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5148
5149 WREG32(TC_CFG_L1_VOLATILE, 0);
5150 WREG32(TC_CFG_L2_VOLATILE, 0);
5151
5152 if (rdev->family == CHIP_KAVERI) {
5153 u32 tmp = RREG32(CHUB_CONTROL);
5154 tmp &= ~BYPASS_VM;
5155 WREG32(CHUB_CONTROL, tmp);
5156 }
5157
5158 /* XXX SH_MEM regs */
5159 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5160 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5161 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5162 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5163 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5164 WREG32(SH_MEM_CONFIG, 0);
5165 WREG32(SH_MEM_APE1_BASE, 1);
5166 WREG32(SH_MEM_APE1_LIMIT, 0);
5167 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5168 /* SDMA GFX */
5169 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5170 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5171 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5172 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5173 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5174 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5175 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5176 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5177
5178 cik_pcie_gart_tlb_flush(rdev);
5179 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5180 (unsigned)(rdev->mc.gtt_size >> 20),
5181 (unsigned long long)rdev->gart.table_addr);
5182 rdev->gart.ready = true;
5183 return 0;
5184}
5185
5186/**
5187 * cik_pcie_gart_disable - gart disable
5188 *
5189 * @rdev: radeon_device pointer
5190 *
5191 * This disables all VM page table (CIK).
5192 */
5193static void cik_pcie_gart_disable(struct radeon_device *rdev)
5194{
5195 /* Disable all tables */
5196 WREG32(VM_CONTEXT0_CNTL, 0);
5197 WREG32(VM_CONTEXT1_CNTL, 0);
5198 /* Setup TLB control */
5199 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5200 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5201 /* Setup L2 cache */
5202 WREG32(VM_L2_CNTL,
5203 ENABLE_L2_FRAGMENT_PROCESSING |
5204 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5205 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5206 EFFECTIVE_L2_QUEUE_SIZE(7) |
5207 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5208 WREG32(VM_L2_CNTL2, 0);
5209 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5210 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5211 radeon_gart_table_vram_unpin(rdev);
5212}
5213
5214/**
5215 * cik_pcie_gart_fini - vm fini callback
5216 *
5217 * @rdev: radeon_device pointer
5218 *
5219 * Tears down the driver GART/VM setup (CIK).
5220 */
5221static void cik_pcie_gart_fini(struct radeon_device *rdev)
5222{
5223 cik_pcie_gart_disable(rdev);
5224 radeon_gart_table_vram_free(rdev);
5225 radeon_gart_fini(rdev);
5226}
5227
5228/* vm parser */
5229/**
5230 * cik_ib_parse - vm ib_parse callback
5231 *
5232 * @rdev: radeon_device pointer
5233 * @ib: indirect buffer pointer
5234 *
5235 * CIK uses hw IB checking so this is a nop (CIK).
5236 */
5237int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5238{
5239 return 0;
5240}
5241
5242/*
5243 * vm
5244 * VMID 0 is the physical GPU addresses as used by the kernel.
5245 * VMIDs 1-15 are used for userspace clients and are handled
5246 * by the radeon vm/hsa code.
5247 */
5248/**
5249 * cik_vm_init - cik vm init callback
5250 *
5251 * @rdev: radeon_device pointer
5252 *
5253 * Inits cik specific vm parameters (number of VMs, base of vram for
5254 * VMIDs 1-15) (CIK).
5255 * Returns 0 for success.
5256 */
5257int cik_vm_init(struct radeon_device *rdev)
5258{
5259 /* number of VMs */
5260 rdev->vm_manager.nvm = 16;
5261 /* base offset of vram pages */
5262 if (rdev->flags & RADEON_IS_IGP) {
5263 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5264 tmp <<= 22;
5265 rdev->vm_manager.vram_base_offset = tmp;
5266 } else
5267 rdev->vm_manager.vram_base_offset = 0;
5268
5269 return 0;
5270}
5271
5272/**
5273 * cik_vm_fini - cik vm fini callback
5274 *
5275 * @rdev: radeon_device pointer
5276 *
5277 * Tear down any asic specific VM setup (CIK).
5278 */
5279void cik_vm_fini(struct radeon_device *rdev)
5280{
5281}
5282
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5283/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5284 * cik_vm_decode_fault - print human readable fault info
5285 *
5286 * @rdev: radeon_device pointer
5287 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5288 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5289 *
5290 * Print human readable fault information (CIK).
5291 */
5292static void cik_vm_decode_fault(struct radeon_device *rdev,
5293 u32 status, u32 addr, u32 mc_client)
5294{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5295 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5296 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5297 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5298 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5299 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5300
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5301 if (rdev->family == CHIP_HAWAII)
5302 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5303 else
5304 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5305
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5306 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5307 protections, vmid, addr,
5308 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5309 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5310}
5311
5312/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5313 * cik_vm_flush - cik vm flush using the CP
5314 *
5315 * @rdev: radeon_device pointer
5316 *
5317 * Update the page table base and flush the VM TLB
5318 * using the CP (CIK).
5319 */
5320void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5321{
5322 struct radeon_ring *ring = &rdev->ring[ridx];
5323
5324 if (vm == NULL)
5325 return;
5326
5327 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5328 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5329 WRITE_DATA_DST_SEL(0)));
5330 if (vm->id < 8) {
5331 radeon_ring_write(ring,
5332 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5333 } else {
5334 radeon_ring_write(ring,
5335 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5336 }
5337 radeon_ring_write(ring, 0);
5338 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5339
5340 /* update SH_MEM_* regs */
5341 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5342 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5343 WRITE_DATA_DST_SEL(0)));
5344 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5345 radeon_ring_write(ring, 0);
5346 radeon_ring_write(ring, VMID(vm->id));
5347
5348 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5349 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5350 WRITE_DATA_DST_SEL(0)));
5351 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5352 radeon_ring_write(ring, 0);
5353
5354 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5355 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5356 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5357 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5358
5359 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5360 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5361 WRITE_DATA_DST_SEL(0)));
5362 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5363 radeon_ring_write(ring, 0);
5364 radeon_ring_write(ring, VMID(0));
5365
5366 /* HDP flush */
5367 /* We should be using the WAIT_REG_MEM packet here like in
5368 * cik_fence_ring_emit(), but it causes the CP to hang in this
5369 * context...
5370 */
5371 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5372 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5373 WRITE_DATA_DST_SEL(0)));
5374 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5375 radeon_ring_write(ring, 0);
5376 radeon_ring_write(ring, 0);
5377
5378 /* bits 0-15 are the VM contexts0-15 */
5379 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5380 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5381 WRITE_DATA_DST_SEL(0)));
5382 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5383 radeon_ring_write(ring, 0);
5384 radeon_ring_write(ring, 1 << vm->id);
5385
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5386 /* compute doesn't have PFP */
5387 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5388 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5389 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5390 radeon_ring_write(ring, 0x0);
5391 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5392}
5393
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5394/*
5395 * RLC
5396 * The RLC is a multi-purpose microengine that handles a
5397 * variety of functions, the most important of which is
5398 * the interrupt controller.
5399 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5400static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5401 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5402{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5403 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5404
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5405 if (enable)
5406 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5407 else
5408 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5409 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5410}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5411
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5412static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5413{
5414 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5415
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5416 tmp = RREG32(RLC_LB_CNTL);
5417 if (enable)
5418 tmp |= LOAD_BALANCE_ENABLE;
5419 else
5420 tmp &= ~LOAD_BALANCE_ENABLE;
5421 WREG32(RLC_LB_CNTL, tmp);
5422}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5423
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5424static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5425{
5426 u32 i, j, k;
5427 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5428
5429 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5430 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5431 cik_select_se_sh(rdev, i, j);
5432 for (k = 0; k < rdev->usec_timeout; k++) {
5433 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5434 break;
5435 udelay(1);
5436 }
5437 }
5438 }
5439 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5440
5441 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5442 for (k = 0; k < rdev->usec_timeout; k++) {
5443 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5444 break;
5445 udelay(1);
5446 }
5447}
5448
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5449static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5450{
5451 u32 tmp;
5452
5453 tmp = RREG32(RLC_CNTL);
5454 if (tmp != rlc)
5455 WREG32(RLC_CNTL, rlc);
5456}
5457
5458static u32 cik_halt_rlc(struct radeon_device *rdev)
5459{
5460 u32 data, orig;
5461
5462 orig = data = RREG32(RLC_CNTL);
5463
5464 if (data & RLC_ENABLE) {
5465 u32 i;
5466
5467 data &= ~RLC_ENABLE;
5468 WREG32(RLC_CNTL, data);
5469
5470 for (i = 0; i < rdev->usec_timeout; i++) {
5471 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5472 break;
5473 udelay(1);
5474 }
5475
5476 cik_wait_for_rlc_serdes(rdev);
5477 }
5478
5479 return orig;
5480}
5481
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5482void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5483{
5484 u32 tmp, i, mask;
5485
5486 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5487 WREG32(RLC_GPR_REG2, tmp);
5488
5489 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5490 for (i = 0; i < rdev->usec_timeout; i++) {
5491 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5492 break;
5493 udelay(1);
5494 }
5495
5496 for (i = 0; i < rdev->usec_timeout; i++) {
5497 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5498 break;
5499 udelay(1);
5500 }
5501}
5502
5503void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5504{
5505 u32 tmp;
5506
5507 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5508 WREG32(RLC_GPR_REG2, tmp);
5509}
5510
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5511/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5512 * cik_rlc_stop - stop the RLC ME
5513 *
5514 * @rdev: radeon_device pointer
5515 *
5516 * Halt the RLC ME (MicroEngine) (CIK).
5517 */
5518static void cik_rlc_stop(struct radeon_device *rdev)
5519{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5520 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5521
5522 cik_enable_gui_idle_interrupt(rdev, false);
5523
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5524 cik_wait_for_rlc_serdes(rdev);
5525}
5526
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5527/**
5528 * cik_rlc_start - start the RLC ME
5529 *
5530 * @rdev: radeon_device pointer
5531 *
5532 * Unhalt the RLC ME (MicroEngine) (CIK).
5533 */
5534static void cik_rlc_start(struct radeon_device *rdev)
5535{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5536 WREG32(RLC_CNTL, RLC_ENABLE);
5537
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5538 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5539
5540 udelay(50);
5541}
5542
5543/**
5544 * cik_rlc_resume - setup the RLC hw
5545 *
5546 * @rdev: radeon_device pointer
5547 *
5548 * Initialize the RLC registers, load the ucode,
5549 * and start the RLC (CIK).
5550 * Returns 0 for success, -EINVAL if the ucode is not available.
5551 */
5552static int cik_rlc_resume(struct radeon_device *rdev)
5553{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5554 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5555 const __be32 *fw_data;
5556
5557 if (!rdev->rlc_fw)
5558 return -EINVAL;
5559
5560 switch (rdev->family) {
5561 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5562 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5563 default:
5564 size = BONAIRE_RLC_UCODE_SIZE;
5565 break;
5566 case CHIP_KAVERI:
5567 size = KV_RLC_UCODE_SIZE;
5568 break;
5569 case CHIP_KABINI:
5570 size = KB_RLC_UCODE_SIZE;
5571 break;
5572 }
5573
5574 cik_rlc_stop(rdev);
5575
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5576 /* disable CG */
5577 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5578 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5579
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5580 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5581
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5582 cik_init_pg(rdev);
5583
5584 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5585
5586 WREG32(RLC_LB_CNTR_INIT, 0);
5587 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5588
5589 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5590 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5591 WREG32(RLC_LB_PARAMS, 0x00600408);
5592 WREG32(RLC_LB_CNTL, 0x80000004);
5593
5594 WREG32(RLC_MC_CNTL, 0);
5595 WREG32(RLC_UCODE_CNTL, 0);
5596
5597 fw_data = (const __be32 *)rdev->rlc_fw->data;
5598 WREG32(RLC_GPM_UCODE_ADDR, 0);
5599 for (i = 0; i < size; i++)
5600 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5601 WREG32(RLC_GPM_UCODE_ADDR, 0);
5602
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5603 /* XXX - find out what chips support lbpw */
5604 cik_enable_lbpw(rdev, false);
5605
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5606 if (rdev->family == CHIP_BONAIRE)
5607 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5608
5609 cik_rlc_start(rdev);
5610
5611 return 0;
5612}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5613
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5614static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5615{
5616 u32 data, orig, tmp, tmp2;
5617
5618 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5619
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5620 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5621 cik_enable_gui_idle_interrupt(rdev, true);
5622
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5623 tmp = cik_halt_rlc(rdev);
5624
5625 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5626 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5627 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5628 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5629 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5630
5631 cik_update_rlc(rdev, tmp);
5632
5633 data |= CGCG_EN | CGLS_EN;
5634 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5635 cik_enable_gui_idle_interrupt(rdev, false);
5636
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5637 RREG32(CB_CGTT_SCLK_CTRL);
5638 RREG32(CB_CGTT_SCLK_CTRL);
5639 RREG32(CB_CGTT_SCLK_CTRL);
5640 RREG32(CB_CGTT_SCLK_CTRL);
5641
5642 data &= ~(CGCG_EN | CGLS_EN);
5643 }
5644
5645 if (orig != data)
5646 WREG32(RLC_CGCG_CGLS_CTRL, data);
5647
5648}
5649
5650static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5651{
5652 u32 data, orig, tmp = 0;
5653
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5654 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5655 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5656 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5657 orig = data = RREG32(CP_MEM_SLP_CNTL);
5658 data |= CP_MEM_LS_EN;
5659 if (orig != data)
5660 WREG32(CP_MEM_SLP_CNTL, data);
5661 }
5662 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5663
5664 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5665 data &= 0xfffffffd;
5666 if (orig != data)
5667 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5668
5669 tmp = cik_halt_rlc(rdev);
5670
5671 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5672 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5673 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5674 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5675 WREG32(RLC_SERDES_WR_CTRL, data);
5676
5677 cik_update_rlc(rdev, tmp);
5678
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5679 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5680 orig = data = RREG32(CGTS_SM_CTRL_REG);
5681 data &= ~SM_MODE_MASK;
5682 data |= SM_MODE(0x2);
5683 data |= SM_MODE_ENABLE;
5684 data &= ~CGTS_OVERRIDE;
5685 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5686 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5687 data &= ~CGTS_LS_OVERRIDE;
5688 data &= ~ON_MONITOR_ADD_MASK;
5689 data |= ON_MONITOR_ADD_EN;
5690 data |= ON_MONITOR_ADD(0x96);
5691 if (orig != data)
5692 WREG32(CGTS_SM_CTRL_REG, data);
5693 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5694 } else {
5695 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5696 data |= 0x00000002;
5697 if (orig != data)
5698 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5699
5700 data = RREG32(RLC_MEM_SLP_CNTL);
5701 if (data & RLC_MEM_LS_EN) {
5702 data &= ~RLC_MEM_LS_EN;
5703 WREG32(RLC_MEM_SLP_CNTL, data);
5704 }
5705
5706 data = RREG32(CP_MEM_SLP_CNTL);
5707 if (data & CP_MEM_LS_EN) {
5708 data &= ~CP_MEM_LS_EN;
5709 WREG32(CP_MEM_SLP_CNTL, data);
5710 }
5711
5712 orig = data = RREG32(CGTS_SM_CTRL_REG);
5713 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5714 if (orig != data)
5715 WREG32(CGTS_SM_CTRL_REG, data);
5716
5717 tmp = cik_halt_rlc(rdev);
5718
5719 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5720 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5721 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5722 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5723 WREG32(RLC_SERDES_WR_CTRL, data);
5724
5725 cik_update_rlc(rdev, tmp);
5726 }
5727}
5728
5729static const u32 mc_cg_registers[] =
5730{
5731 MC_HUB_MISC_HUB_CG,
5732 MC_HUB_MISC_SIP_CG,
5733 MC_HUB_MISC_VM_CG,
5734 MC_XPB_CLK_GAT,
5735 ATC_MISC_CG,
5736 MC_CITF_MISC_WR_CG,
5737 MC_CITF_MISC_RD_CG,
5738 MC_CITF_MISC_VM_CG,
5739 VM_L2_CG,
5740};
5741
5742static void cik_enable_mc_ls(struct radeon_device *rdev,
5743 bool enable)
5744{
5745 int i;
5746 u32 orig, data;
5747
5748 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5749 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5750 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5751 data |= MC_LS_ENABLE;
5752 else
5753 data &= ~MC_LS_ENABLE;
5754 if (data != orig)
5755 WREG32(mc_cg_registers[i], data);
5756 }
5757}
5758
5759static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5760 bool enable)
5761{
5762 int i;
5763 u32 orig, data;
5764
5765 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5766 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5767 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5768 data |= MC_CG_ENABLE;
5769 else
5770 data &= ~MC_CG_ENABLE;
5771 if (data != orig)
5772 WREG32(mc_cg_registers[i], data);
5773 }
5774}
5775
5776static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5777 bool enable)
5778{
5779 u32 orig, data;
5780
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5781 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5782 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5783 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5784 } else {
5785 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5786 data |= 0xff000000;
5787 if (data != orig)
5788 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5789
5790 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5791 data |= 0xff000000;
5792 if (data != orig)
5793 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5794 }
5795}
5796
5797static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5798 bool enable)
5799{
5800 u32 orig, data;
5801
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5802 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5803 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5804 data |= 0x100;
5805 if (orig != data)
5806 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5807
5808 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5809 data |= 0x100;
5810 if (orig != data)
5811 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5812 } else {
5813 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5814 data &= ~0x100;
5815 if (orig != data)
5816 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5817
5818 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5819 data &= ~0x100;
5820 if (orig != data)
5821 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5822 }
5823}
5824
5825static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5826 bool enable)
5827{
5828 u32 orig, data;
5829
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5830 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5831 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5832 data = 0xfff;
5833 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5834
5835 orig = data = RREG32(UVD_CGC_CTRL);
5836 data |= DCM;
5837 if (orig != data)
5838 WREG32(UVD_CGC_CTRL, data);
5839 } else {
5840 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5841 data &= ~0xfff;
5842 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5843
5844 orig = data = RREG32(UVD_CGC_CTRL);
5845 data &= ~DCM;
5846 if (orig != data)
5847 WREG32(UVD_CGC_CTRL, data);
5848 }
5849}
5850
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5851static void cik_enable_bif_mgls(struct radeon_device *rdev,
5852 bool enable)
5853{
5854 u32 orig, data;
5855
5856 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5857
5858 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5859 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5860 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5861 else
5862 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5863 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5864
5865 if (orig != data)
5866 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5867}
5868
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5869static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5870 bool enable)
5871{
5872 u32 orig, data;
5873
5874 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5875
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5876 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5877 data &= ~CLOCK_GATING_DIS;
5878 else
5879 data |= CLOCK_GATING_DIS;
5880
5881 if (orig != data)
5882 WREG32(HDP_HOST_PATH_CNTL, data);
5883}
5884
5885static void cik_enable_hdp_ls(struct radeon_device *rdev,
5886 bool enable)
5887{
5888 u32 orig, data;
5889
5890 orig = data = RREG32(HDP_MEM_POWER_LS);
5891
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5892 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5893 data |= HDP_LS_ENABLE;
5894 else
5895 data &= ~HDP_LS_ENABLE;
5896
5897 if (orig != data)
5898 WREG32(HDP_MEM_POWER_LS, data);
5899}
5900
5901void cik_update_cg(struct radeon_device *rdev,
5902 u32 block, bool enable)
5903{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5904
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5905 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5906 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5907 /* order matters! */
5908 if (enable) {
5909 cik_enable_mgcg(rdev, true);
5910 cik_enable_cgcg(rdev, true);
5911 } else {
5912 cik_enable_cgcg(rdev, false);
5913 cik_enable_mgcg(rdev, false);
5914 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5915 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5916 }
5917
5918 if (block & RADEON_CG_BLOCK_MC) {
5919 if (!(rdev->flags & RADEON_IS_IGP)) {
5920 cik_enable_mc_mgcg(rdev, enable);
5921 cik_enable_mc_ls(rdev, enable);
5922 }
5923 }
5924
5925 if (block & RADEON_CG_BLOCK_SDMA) {
5926 cik_enable_sdma_mgcg(rdev, enable);
5927 cik_enable_sdma_mgls(rdev, enable);
5928 }
5929
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5930 if (block & RADEON_CG_BLOCK_BIF) {
5931 cik_enable_bif_mgls(rdev, enable);
5932 }
5933
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5934 if (block & RADEON_CG_BLOCK_UVD) {
5935 if (rdev->has_uvd)
5936 cik_enable_uvd_mgcg(rdev, enable);
5937 }
5938
5939 if (block & RADEON_CG_BLOCK_HDP) {
5940 cik_enable_hdp_mgcg(rdev, enable);
5941 cik_enable_hdp_ls(rdev, enable);
5942 }
5943}
5944
5945static void cik_init_cg(struct radeon_device *rdev)
5946{
5947
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5948 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5949
5950 if (rdev->has_uvd)
5951 si_init_uvd_internal_cg(rdev);
5952
5953 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5954 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5955 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5956 RADEON_CG_BLOCK_UVD |
5957 RADEON_CG_BLOCK_HDP), true);
5958}
5959
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5960static void cik_fini_cg(struct radeon_device *rdev)
5961{
5962 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5963 RADEON_CG_BLOCK_SDMA |
5964 RADEON_CG_BLOCK_BIF |
5965 RADEON_CG_BLOCK_UVD |
5966 RADEON_CG_BLOCK_HDP), false);
5967
5968 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5969}
5970
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5971static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5972 bool enable)
5973{
5974 u32 data, orig;
5975
5976 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5977 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5978 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5979 else
5980 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5981 if (orig != data)
5982 WREG32(RLC_PG_CNTL, data);
5983}
5984
5985static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5986 bool enable)
5987{
5988 u32 data, orig;
5989
5990 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5991 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5992 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5993 else
5994 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5995 if (orig != data)
5996 WREG32(RLC_PG_CNTL, data);
5997}
5998
5999static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6000{
6001 u32 data, orig;
6002
6003 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6004 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6005 data &= ~DISABLE_CP_PG;
6006 else
6007 data |= DISABLE_CP_PG;
6008 if (orig != data)
6009 WREG32(RLC_PG_CNTL, data);
6010}
6011
6012static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6013{
6014 u32 data, orig;
6015
6016 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6017 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6018 data &= ~DISABLE_GDS_PG;
6019 else
6020 data |= DISABLE_GDS_PG;
6021 if (orig != data)
6022 WREG32(RLC_PG_CNTL, data);
6023}
6024
6025#define CP_ME_TABLE_SIZE 96
6026#define CP_ME_TABLE_OFFSET 2048
6027#define CP_MEC_TABLE_OFFSET 4096
6028
6029void cik_init_cp_pg_table(struct radeon_device *rdev)
6030{
6031 const __be32 *fw_data;
6032 volatile u32 *dst_ptr;
6033 int me, i, max_me = 4;
6034 u32 bo_offset = 0;
6035 u32 table_offset;
6036
6037 if (rdev->family == CHIP_KAVERI)
6038 max_me = 5;
6039
6040 if (rdev->rlc.cp_table_ptr == NULL)
6041 return;
6042
6043 /* write the cp table buffer */
6044 dst_ptr = rdev->rlc.cp_table_ptr;
6045 for (me = 0; me < max_me; me++) {
6046 if (me == 0) {
6047 fw_data = (const __be32 *)rdev->ce_fw->data;
6048 table_offset = CP_ME_TABLE_OFFSET;
6049 } else if (me == 1) {
6050 fw_data = (const __be32 *)rdev->pfp_fw->data;
6051 table_offset = CP_ME_TABLE_OFFSET;
6052 } else if (me == 2) {
6053 fw_data = (const __be32 *)rdev->me_fw->data;
6054 table_offset = CP_ME_TABLE_OFFSET;
6055 } else {
6056 fw_data = (const __be32 *)rdev->mec_fw->data;
6057 table_offset = CP_MEC_TABLE_OFFSET;
6058 }
6059
6060 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6061 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6062 }
6063 bo_offset += CP_ME_TABLE_SIZE;
6064 }
6065}
6066
6067static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6068 bool enable)
6069{
6070 u32 data, orig;
6071
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6072 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6073 orig = data = RREG32(RLC_PG_CNTL);
6074 data |= GFX_PG_ENABLE;
6075 if (orig != data)
6076 WREG32(RLC_PG_CNTL, data);
6077
6078 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6079 data |= AUTO_PG_EN;
6080 if (orig != data)
6081 WREG32(RLC_AUTO_PG_CTRL, data);
6082 } else {
6083 orig = data = RREG32(RLC_PG_CNTL);
6084 data &= ~GFX_PG_ENABLE;
6085 if (orig != data)
6086 WREG32(RLC_PG_CNTL, data);
6087
6088 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6089 data &= ~AUTO_PG_EN;
6090 if (orig != data)
6091 WREG32(RLC_AUTO_PG_CTRL, data);
6092
6093 data = RREG32(DB_RENDER_CONTROL);
6094 }
6095}
6096
6097static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6098{
6099 u32 mask = 0, tmp, tmp1;
6100 int i;
6101
6102 cik_select_se_sh(rdev, se, sh);
6103 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6104 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6106
6107 tmp &= 0xffff0000;
6108
6109 tmp |= tmp1;
6110 tmp >>= 16;
6111
6112 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6113 mask <<= 1;
6114 mask |= 1;
6115 }
6116
6117 return (~tmp) & mask;
6118}
6119
6120static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6121{
6122 u32 i, j, k, active_cu_number = 0;
6123 u32 mask, counter, cu_bitmap;
6124 u32 tmp = 0;
6125
6126 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6127 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6128 mask = 1;
6129 cu_bitmap = 0;
6130 counter = 0;
6131 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6132 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6133 if (counter < 2)
6134 cu_bitmap |= mask;
6135 counter ++;
6136 }
6137 mask <<= 1;
6138 }
6139
6140 active_cu_number += counter;
6141 tmp |= (cu_bitmap << (i * 16 + j * 8));
6142 }
6143 }
6144
6145 WREG32(RLC_PG_AO_CU_MASK, tmp);
6146
6147 tmp = RREG32(RLC_MAX_PG_CU);
6148 tmp &= ~MAX_PU_CU_MASK;
6149 tmp |= MAX_PU_CU(active_cu_number);
6150 WREG32(RLC_MAX_PG_CU, tmp);
6151}
6152
6153static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6154 bool enable)
6155{
6156 u32 data, orig;
6157
6158 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6159 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6160 data |= STATIC_PER_CU_PG_ENABLE;
6161 else
6162 data &= ~STATIC_PER_CU_PG_ENABLE;
6163 if (orig != data)
6164 WREG32(RLC_PG_CNTL, data);
6165}
6166
6167static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6168 bool enable)
6169{
6170 u32 data, orig;
6171
6172 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6173 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6174 data |= DYN_PER_CU_PG_ENABLE;
6175 else
6176 data &= ~DYN_PER_CU_PG_ENABLE;
6177 if (orig != data)
6178 WREG32(RLC_PG_CNTL, data);
6179}
6180
6181#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6182#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6183
6184static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6185{
6186 u32 data, orig;
6187 u32 i;
6188
6189 if (rdev->rlc.cs_data) {
6190 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6191 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6192 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6193 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6194 } else {
6195 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6196 for (i = 0; i < 3; i++)
6197 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6198 }
6199 if (rdev->rlc.reg_list) {
6200 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6201 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6202 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6203 }
6204
6205 orig = data = RREG32(RLC_PG_CNTL);
6206 data |= GFX_PG_SRC;
6207 if (orig != data)
6208 WREG32(RLC_PG_CNTL, data);
6209
6210 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6211 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6212
6213 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6214 data &= ~IDLE_POLL_COUNT_MASK;
6215 data |= IDLE_POLL_COUNT(0x60);
6216 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6217
6218 data = 0x10101010;
6219 WREG32(RLC_PG_DELAY, data);
6220
6221 data = RREG32(RLC_PG_DELAY_2);
6222 data &= ~0xff;
6223 data |= 0x3;
6224 WREG32(RLC_PG_DELAY_2, data);
6225
6226 data = RREG32(RLC_AUTO_PG_CTRL);
6227 data &= ~GRBM_REG_SGIT_MASK;
6228 data |= GRBM_REG_SGIT(0x700);
6229 WREG32(RLC_AUTO_PG_CTRL, data);
6230
6231}
6232
6233static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6234{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6235 cik_enable_gfx_cgpg(rdev, enable);
6236 cik_enable_gfx_static_mgpg(rdev, enable);
6237 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6238}
6239
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6240u32 cik_get_csb_size(struct radeon_device *rdev)
6241{
6242 u32 count = 0;
6243 const struct cs_section_def *sect = NULL;
6244 const struct cs_extent_def *ext = NULL;
6245
6246 if (rdev->rlc.cs_data == NULL)
6247 return 0;
6248
6249 /* begin clear state */
6250 count += 2;
6251 /* context control state */
6252 count += 3;
6253
6254 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6255 for (ext = sect->section; ext->extent != NULL; ++ext) {
6256 if (sect->id == SECT_CONTEXT)
6257 count += 2 + ext->reg_count;
6258 else
6259 return 0;
6260 }
6261 }
6262 /* pa_sc_raster_config/pa_sc_raster_config1 */
6263 count += 4;
6264 /* end clear state */
6265 count += 2;
6266 /* clear state */
6267 count += 2;
6268
6269 return count;
6270}
6271
6272void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6273{
6274 u32 count = 0, i;
6275 const struct cs_section_def *sect = NULL;
6276 const struct cs_extent_def *ext = NULL;
6277
6278 if (rdev->rlc.cs_data == NULL)
6279 return;
6280 if (buffer == NULL)
6281 return;
6282
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6283 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6284 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6285
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6286 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6287 buffer[count++] = cpu_to_le32(0x80000000);
6288 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6289
6290 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6291 for (ext = sect->section; ext->extent != NULL; ++ext) {
6292 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6293 buffer[count++] =
6294 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6295 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6296 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6297 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6298 } else {
6299 return;
6300 }
6301 }
6302 }
6303
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6304 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6305 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6306 switch (rdev->family) {
6307 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6308 buffer[count++] = cpu_to_le32(0x16000012);
6309 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6310 break;
6311 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6312 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6313 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6314 break;
6315 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6316 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6317 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6318 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame^]6319 case CHIP_HAWAII:
6320 buffer[count++] = 0x3a00161a;
6321 buffer[count++] = 0x0000002e;
6322 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6323 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6324 buffer[count++] = cpu_to_le32(0x00000000);
6325 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6326 break;
6327 }
6328
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6329 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6330 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6331
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6332 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6333 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6334}
6335
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6336static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6337{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6338 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6339 cik_enable_sck_slowdown_on_pu(rdev, true);
6340 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6341 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6342 cik_init_gfx_cgpg(rdev);
6343 cik_enable_cp_pg(rdev, true);
6344 cik_enable_gds_pg(rdev, true);
6345 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6346 cik_init_ao_cu_mask(rdev);
6347 cik_update_gfx_pg(rdev, true);
6348 }
6349}
6350
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6351static void cik_fini_pg(struct radeon_device *rdev)
6352{
6353 if (rdev->pg_flags) {
6354 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6355 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6356 cik_enable_cp_pg(rdev, false);
6357 cik_enable_gds_pg(rdev, false);
6358 }
6359 }
6360}
6361
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6362/*
6363 * Interrupts
6364 * Starting with r6xx, interrupts are handled via a ring buffer.
6365 * Ring buffers are areas of GPU accessible memory that the GPU
6366 * writes interrupt vectors into and the host reads vectors out of.
6367 * There is a rptr (read pointer) that determines where the
6368 * host is currently reading, and a wptr (write pointer)
6369 * which determines where the GPU has written. When the
6370 * pointers are equal, the ring is idle. When the GPU
6371 * writes vectors to the ring buffer, it increments the
6372 * wptr. When there is an interrupt, the host then starts
6373 * fetching commands and processing them until the pointers are
6374 * equal again at which point it updates the rptr.
6375 */
6376
6377/**
6378 * cik_enable_interrupts - Enable the interrupt ring buffer
6379 *
6380 * @rdev: radeon_device pointer
6381 *
6382 * Enable the interrupt ring buffer (CIK).
6383 */
6384static void cik_enable_interrupts(struct radeon_device *rdev)
6385{
6386 u32 ih_cntl = RREG32(IH_CNTL);
6387 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6388
6389 ih_cntl |= ENABLE_INTR;
6390 ih_rb_cntl |= IH_RB_ENABLE;
6391 WREG32(IH_CNTL, ih_cntl);
6392 WREG32(IH_RB_CNTL, ih_rb_cntl);
6393 rdev->ih.enabled = true;
6394}
6395
6396/**
6397 * cik_disable_interrupts - Disable the interrupt ring buffer
6398 *
6399 * @rdev: radeon_device pointer
6400 *
6401 * Disable the interrupt ring buffer (CIK).
6402 */
6403static void cik_disable_interrupts(struct radeon_device *rdev)
6404{
6405 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6406 u32 ih_cntl = RREG32(IH_CNTL);
6407
6408 ih_rb_cntl &= ~IH_RB_ENABLE;
6409 ih_cntl &= ~ENABLE_INTR;
6410 WREG32(IH_RB_CNTL, ih_rb_cntl);
6411 WREG32(IH_CNTL, ih_cntl);
6412 /* set rptr, wptr to 0 */
6413 WREG32(IH_RB_RPTR, 0);
6414 WREG32(IH_RB_WPTR, 0);
6415 rdev->ih.enabled = false;
6416 rdev->ih.rptr = 0;
6417}
6418
6419/**
6420 * cik_disable_interrupt_state - Disable all interrupt sources
6421 *
6422 * @rdev: radeon_device pointer
6423 *
6424 * Clear all interrupt enable bits used by the driver (CIK).
6425 */
6426static void cik_disable_interrupt_state(struct radeon_device *rdev)
6427{
6428 u32 tmp;
6429
6430 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6431 tmp = RREG32(CP_INT_CNTL_RING0) &
6432 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6433 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6434 /* sdma */
6435 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6436 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6437 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6438 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6439 /* compute queues */
6440 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6441 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6442 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6443 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6444 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6445 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6446 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6447 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6448 /* grbm */
6449 WREG32(GRBM_INT_CNTL, 0);
6450 /* vline/vblank, etc. */
6451 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6452 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6453 if (rdev->num_crtc >= 4) {
6454 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6455 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6456 }
6457 if (rdev->num_crtc >= 6) {
6458 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6459 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6460 }
6461
6462 /* dac hotplug */
6463 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6464
6465 /* digital hotplug */
6466 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6467 WREG32(DC_HPD1_INT_CONTROL, tmp);
6468 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6469 WREG32(DC_HPD2_INT_CONTROL, tmp);
6470 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6471 WREG32(DC_HPD3_INT_CONTROL, tmp);
6472 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6473 WREG32(DC_HPD4_INT_CONTROL, tmp);
6474 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6475 WREG32(DC_HPD5_INT_CONTROL, tmp);
6476 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6477 WREG32(DC_HPD6_INT_CONTROL, tmp);
6478
6479}
6480
6481/**
6482 * cik_irq_init - init and enable the interrupt ring
6483 *
6484 * @rdev: radeon_device pointer
6485 *
6486 * Allocate a ring buffer for the interrupt controller,
6487 * enable the RLC, disable interrupts, enable the IH
6488 * ring buffer and enable it (CIK).
6489 * Called at device load and reume.
6490 * Returns 0 for success, errors for failure.
6491 */
6492static int cik_irq_init(struct radeon_device *rdev)
6493{
6494 int ret = 0;
6495 int rb_bufsz;
6496 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6497
6498 /* allocate ring */
6499 ret = r600_ih_ring_alloc(rdev);
6500 if (ret)
6501 return ret;
6502
6503 /* disable irqs */
6504 cik_disable_interrupts(rdev);
6505
6506 /* init rlc */
6507 ret = cik_rlc_resume(rdev);
6508 if (ret) {
6509 r600_ih_ring_fini(rdev);
6510 return ret;
6511 }
6512
6513 /* setup interrupt control */
6514 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6515 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6516 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6517 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6518 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6519 */
6520 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6521 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6522 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6523 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6524
6525 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6526 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6527
6528 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6529 IH_WPTR_OVERFLOW_CLEAR |
6530 (rb_bufsz << 1));
6531
6532 if (rdev->wb.enabled)
6533 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6534
6535 /* set the writeback address whether it's enabled or not */
6536 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6537 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6538
6539 WREG32(IH_RB_CNTL, ih_rb_cntl);
6540
6541 /* set rptr, wptr to 0 */
6542 WREG32(IH_RB_RPTR, 0);
6543 WREG32(IH_RB_WPTR, 0);
6544
6545 /* Default settings for IH_CNTL (disabled at first) */
6546 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6547 /* RPTR_REARM only works if msi's are enabled */
6548 if (rdev->msi_enabled)
6549 ih_cntl |= RPTR_REARM;
6550 WREG32(IH_CNTL, ih_cntl);
6551
6552 /* force the active interrupt state to all disabled */
6553 cik_disable_interrupt_state(rdev);
6554
6555 pci_set_master(rdev->pdev);
6556
6557 /* enable irqs */
6558 cik_enable_interrupts(rdev);
6559
6560 return ret;
6561}
6562
6563/**
6564 * cik_irq_set - enable/disable interrupt sources
6565 *
6566 * @rdev: radeon_device pointer
6567 *
6568 * Enable interrupt sources on the GPU (vblanks, hpd,
6569 * etc.) (CIK).
6570 * Returns 0 for success, errors for failure.
6571 */
6572int cik_irq_set(struct radeon_device *rdev)
6573{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6574 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6575 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6576 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6577 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6578 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6579 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6580 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6581 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6582
6583 if (!rdev->irq.installed) {
6584 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6585 return -EINVAL;
6586 }
6587 /* don't enable anything if the ih is disabled */
6588 if (!rdev->ih.enabled) {
6589 cik_disable_interrupts(rdev);
6590 /* force the active interrupt state to all disabled */
6591 cik_disable_interrupt_state(rdev);
6592 return 0;
6593 }
6594
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6595 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6596 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6597 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6598
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6599 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6600 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6601 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6602 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6604 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6605
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6606 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6607 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6608
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6609 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6610 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6611 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6612 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6616 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6617
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6618 if (rdev->flags & RADEON_IS_IGP)
6619 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6620 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6621 else
6622 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6623 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6624
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6625 /* enable CP interrupts on all rings */
6626 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6627 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6628 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6629 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6630 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6631 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6632 DRM_DEBUG("si_irq_set: sw int cp1\n");
6633 if (ring->me == 1) {
6634 switch (ring->pipe) {
6635 case 0:
6636 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6637 break;
6638 case 1:
6639 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6640 break;
6641 case 2:
6642 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6643 break;
6644 case 3:
6645 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6646 break;
6647 default:
6648 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6649 break;
6650 }
6651 } else if (ring->me == 2) {
6652 switch (ring->pipe) {
6653 case 0:
6654 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6655 break;
6656 case 1:
6657 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6658 break;
6659 case 2:
6660 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6661 break;
6662 case 3:
6663 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6664 break;
6665 default:
6666 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6667 break;
6668 }
6669 } else {
6670 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6671 }
6672 }
6673 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6674 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6675 DRM_DEBUG("si_irq_set: sw int cp2\n");
6676 if (ring->me == 1) {
6677 switch (ring->pipe) {
6678 case 0:
6679 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6680 break;
6681 case 1:
6682 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6683 break;
6684 case 2:
6685 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6686 break;
6687 case 3:
6688 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6689 break;
6690 default:
6691 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6692 break;
6693 }
6694 } else if (ring->me == 2) {
6695 switch (ring->pipe) {
6696 case 0:
6697 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6698 break;
6699 case 1:
6700 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6701 break;
6702 case 2:
6703 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6704 break;
6705 case 3:
6706 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6707 break;
6708 default:
6709 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6710 break;
6711 }
6712 } else {
6713 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6714 }
6715 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6716
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6717 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6718 DRM_DEBUG("cik_irq_set: sw int dma\n");
6719 dma_cntl |= TRAP_ENABLE;
6720 }
6721
6722 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6723 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6724 dma_cntl1 |= TRAP_ENABLE;
6725 }
6726
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6727 if (rdev->irq.crtc_vblank_int[0] ||
6728 atomic_read(&rdev->irq.pflip[0])) {
6729 DRM_DEBUG("cik_irq_set: vblank 0\n");
6730 crtc1 |= VBLANK_INTERRUPT_MASK;
6731 }
6732 if (rdev->irq.crtc_vblank_int[1] ||
6733 atomic_read(&rdev->irq.pflip[1])) {
6734 DRM_DEBUG("cik_irq_set: vblank 1\n");
6735 crtc2 |= VBLANK_INTERRUPT_MASK;
6736 }
6737 if (rdev->irq.crtc_vblank_int[2] ||
6738 atomic_read(&rdev->irq.pflip[2])) {
6739 DRM_DEBUG("cik_irq_set: vblank 2\n");
6740 crtc3 |= VBLANK_INTERRUPT_MASK;
6741 }
6742 if (rdev->irq.crtc_vblank_int[3] ||
6743 atomic_read(&rdev->irq.pflip[3])) {
6744 DRM_DEBUG("cik_irq_set: vblank 3\n");
6745 crtc4 |= VBLANK_INTERRUPT_MASK;
6746 }
6747 if (rdev->irq.crtc_vblank_int[4] ||
6748 atomic_read(&rdev->irq.pflip[4])) {
6749 DRM_DEBUG("cik_irq_set: vblank 4\n");
6750 crtc5 |= VBLANK_INTERRUPT_MASK;
6751 }
6752 if (rdev->irq.crtc_vblank_int[5] ||
6753 atomic_read(&rdev->irq.pflip[5])) {
6754 DRM_DEBUG("cik_irq_set: vblank 5\n");
6755 crtc6 |= VBLANK_INTERRUPT_MASK;
6756 }
6757 if (rdev->irq.hpd[0]) {
6758 DRM_DEBUG("cik_irq_set: hpd 1\n");
6759 hpd1 |= DC_HPDx_INT_EN;
6760 }
6761 if (rdev->irq.hpd[1]) {
6762 DRM_DEBUG("cik_irq_set: hpd 2\n");
6763 hpd2 |= DC_HPDx_INT_EN;
6764 }
6765 if (rdev->irq.hpd[2]) {
6766 DRM_DEBUG("cik_irq_set: hpd 3\n");
6767 hpd3 |= DC_HPDx_INT_EN;
6768 }
6769 if (rdev->irq.hpd[3]) {
6770 DRM_DEBUG("cik_irq_set: hpd 4\n");
6771 hpd4 |= DC_HPDx_INT_EN;
6772 }
6773 if (rdev->irq.hpd[4]) {
6774 DRM_DEBUG("cik_irq_set: hpd 5\n");
6775 hpd5 |= DC_HPDx_INT_EN;
6776 }
6777 if (rdev->irq.hpd[5]) {
6778 DRM_DEBUG("cik_irq_set: hpd 6\n");
6779 hpd6 |= DC_HPDx_INT_EN;
6780 }
6781
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6782 if (rdev->irq.dpm_thermal) {
6783 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6784 if (rdev->flags & RADEON_IS_IGP)
6785 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6786 else
6787 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6788 }
6789
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6790 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6791
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6792 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6793 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6794
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6795 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6796 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6797 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6798 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6799 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6800 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6801 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6802 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6803
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6804 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6805
6806 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6807 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6808 if (rdev->num_crtc >= 4) {
6809 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6810 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6811 }
6812 if (rdev->num_crtc >= 6) {
6813 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6814 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6815 }
6816
6817 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6818 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6819 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6820 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6821 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6822 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6823
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6824 if (rdev->flags & RADEON_IS_IGP)
6825 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6826 else
6827 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6828
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6829 return 0;
6830}
6831
6832/**
6833 * cik_irq_ack - ack interrupt sources
6834 *
6835 * @rdev: radeon_device pointer
6836 *
6837 * Ack interrupt sources on the GPU (vblanks, hpd,
6838 * etc.) (CIK). Certain interrupts sources are sw
6839 * generated and do not require an explicit ack.
6840 */
6841static inline void cik_irq_ack(struct radeon_device *rdev)
6842{
6843 u32 tmp;
6844
6845 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6846 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6847 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6848 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6849 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6850 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6851 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6852
6853 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6854 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6855 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6856 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6857 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6858 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6859 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6860 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6861
6862 if (rdev->num_crtc >= 4) {
6863 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6864 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6865 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6866 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6867 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6868 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6869 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6870 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6871 }
6872
6873 if (rdev->num_crtc >= 6) {
6874 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6875 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6876 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6877 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6878 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6879 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6880 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6881 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6882 }
6883
6884 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6885 tmp = RREG32(DC_HPD1_INT_CONTROL);
6886 tmp |= DC_HPDx_INT_ACK;
6887 WREG32(DC_HPD1_INT_CONTROL, tmp);
6888 }
6889 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6890 tmp = RREG32(DC_HPD2_INT_CONTROL);
6891 tmp |= DC_HPDx_INT_ACK;
6892 WREG32(DC_HPD2_INT_CONTROL, tmp);
6893 }
6894 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6895 tmp = RREG32(DC_HPD3_INT_CONTROL);
6896 tmp |= DC_HPDx_INT_ACK;
6897 WREG32(DC_HPD3_INT_CONTROL, tmp);
6898 }
6899 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6900 tmp = RREG32(DC_HPD4_INT_CONTROL);
6901 tmp |= DC_HPDx_INT_ACK;
6902 WREG32(DC_HPD4_INT_CONTROL, tmp);
6903 }
6904 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6905 tmp = RREG32(DC_HPD5_INT_CONTROL);
6906 tmp |= DC_HPDx_INT_ACK;
6907 WREG32(DC_HPD5_INT_CONTROL, tmp);
6908 }
6909 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6910 tmp = RREG32(DC_HPD5_INT_CONTROL);
6911 tmp |= DC_HPDx_INT_ACK;
6912 WREG32(DC_HPD6_INT_CONTROL, tmp);
6913 }
6914}
6915
6916/**
6917 * cik_irq_disable - disable interrupts
6918 *
6919 * @rdev: radeon_device pointer
6920 *
6921 * Disable interrupts on the hw (CIK).
6922 */
6923static void cik_irq_disable(struct radeon_device *rdev)
6924{
6925 cik_disable_interrupts(rdev);
6926 /* Wait and acknowledge irq */
6927 mdelay(1);
6928 cik_irq_ack(rdev);
6929 cik_disable_interrupt_state(rdev);
6930}
6931
6932/**
6933 * cik_irq_disable - disable interrupts for suspend
6934 *
6935 * @rdev: radeon_device pointer
6936 *
6937 * Disable interrupts and stop the RLC (CIK).
6938 * Used for suspend.
6939 */
6940static void cik_irq_suspend(struct radeon_device *rdev)
6941{
6942 cik_irq_disable(rdev);
6943 cik_rlc_stop(rdev);
6944}
6945
6946/**
6947 * cik_irq_fini - tear down interrupt support
6948 *
6949 * @rdev: radeon_device pointer
6950 *
6951 * Disable interrupts on the hw and free the IH ring
6952 * buffer (CIK).
6953 * Used for driver unload.
6954 */
6955static void cik_irq_fini(struct radeon_device *rdev)
6956{
6957 cik_irq_suspend(rdev);
6958 r600_ih_ring_fini(rdev);
6959}
6960
6961/**
6962 * cik_get_ih_wptr - get the IH ring buffer wptr
6963 *
6964 * @rdev: radeon_device pointer
6965 *
6966 * Get the IH ring buffer wptr from either the register
6967 * or the writeback memory buffer (CIK). Also check for
6968 * ring buffer overflow and deal with it.
6969 * Used by cik_irq_process().
6970 * Returns the value of the wptr.
6971 */
6972static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6973{
6974 u32 wptr, tmp;
6975
6976 if (rdev->wb.enabled)
6977 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6978 else
6979 wptr = RREG32(IH_RB_WPTR);
6980
6981 if (wptr & RB_OVERFLOW) {
6982 /* When a ring buffer overflow happen start parsing interrupt
6983 * from the last not overwritten vector (wptr + 16). Hopefully
6984 * this should allow us to catchup.
6985 */
6986 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6987 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6988 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6989 tmp = RREG32(IH_RB_CNTL);
6990 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6991 WREG32(IH_RB_CNTL, tmp);
6992 }
6993 return (wptr & rdev->ih.ptr_mask);
6994}
6995
6996/* CIK IV Ring
6997 * Each IV ring entry is 128 bits:
6998 * [7:0] - interrupt source id
6999 * [31:8] - reserved
7000 * [59:32] - interrupt source data
7001 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7002 * [71:64] - RINGID
7003 * CP:
7004 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7005 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7006 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7007 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7008 * PIPE_ID - ME0 0=3D
7009 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7010 * SDMA:
7011 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7012 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7013 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7014 * [79:72] - VMID
7015 * [95:80] - PASID
7016 * [127:96] - reserved
7017 */
7018/**
7019 * cik_irq_process - interrupt handler
7020 *
7021 * @rdev: radeon_device pointer
7022 *
7023 * Interrupt hander (CIK). Walk the IH ring,
7024 * ack interrupts and schedule work to handle
7025 * interrupt events.
7026 * Returns irq process return code.
7027 */
7028int cik_irq_process(struct radeon_device *rdev)
7029{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7030 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7031 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7032 u32 wptr;
7033 u32 rptr;
7034 u32 src_id, src_data, ring_id;
7035 u8 me_id, pipe_id, queue_id;
7036 u32 ring_index;
7037 bool queue_hotplug = false;
7038 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7039 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7040 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7041
7042 if (!rdev->ih.enabled || rdev->shutdown)
7043 return IRQ_NONE;
7044
7045 wptr = cik_get_ih_wptr(rdev);
7046
7047restart_ih:
7048 /* is somebody else already processing irqs? */
7049 if (atomic_xchg(&rdev->ih.lock, 1))
7050 return IRQ_NONE;
7051
7052 rptr = rdev->ih.rptr;
7053 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7054
7055 /* Order reading of wptr vs. reading of IH ring data */
7056 rmb();
7057
7058 /* display interrupts */
7059 cik_irq_ack(rdev);
7060
7061 while (rptr != wptr) {
7062 /* wptr/rptr are in bytes! */
7063 ring_index = rptr / 4;
7064 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7065 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7066 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7067
7068 switch (src_id) {
7069 case 1: /* D1 vblank/vline */
7070 switch (src_data) {
7071 case 0: /* D1 vblank */
7072 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7073 if (rdev->irq.crtc_vblank_int[0]) {
7074 drm_handle_vblank(rdev->ddev, 0);
7075 rdev->pm.vblank_sync = true;
7076 wake_up(&rdev->irq.vblank_queue);
7077 }
7078 if (atomic_read(&rdev->irq.pflip[0]))
7079 radeon_crtc_handle_flip(rdev, 0);
7080 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7081 DRM_DEBUG("IH: D1 vblank\n");
7082 }
7083 break;
7084 case 1: /* D1 vline */
7085 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7086 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7087 DRM_DEBUG("IH: D1 vline\n");
7088 }
7089 break;
7090 default:
7091 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7092 break;
7093 }
7094 break;
7095 case 2: /* D2 vblank/vline */
7096 switch (src_data) {
7097 case 0: /* D2 vblank */
7098 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7099 if (rdev->irq.crtc_vblank_int[1]) {
7100 drm_handle_vblank(rdev->ddev, 1);
7101 rdev->pm.vblank_sync = true;
7102 wake_up(&rdev->irq.vblank_queue);
7103 }
7104 if (atomic_read(&rdev->irq.pflip[1]))
7105 radeon_crtc_handle_flip(rdev, 1);
7106 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7107 DRM_DEBUG("IH: D2 vblank\n");
7108 }
7109 break;
7110 case 1: /* D2 vline */
7111 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7112 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7113 DRM_DEBUG("IH: D2 vline\n");
7114 }
7115 break;
7116 default:
7117 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7118 break;
7119 }
7120 break;
7121 case 3: /* D3 vblank/vline */
7122 switch (src_data) {
7123 case 0: /* D3 vblank */
7124 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7125 if (rdev->irq.crtc_vblank_int[2]) {
7126 drm_handle_vblank(rdev->ddev, 2);
7127 rdev->pm.vblank_sync = true;
7128 wake_up(&rdev->irq.vblank_queue);
7129 }
7130 if (atomic_read(&rdev->irq.pflip[2]))
7131 radeon_crtc_handle_flip(rdev, 2);
7132 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7133 DRM_DEBUG("IH: D3 vblank\n");
7134 }
7135 break;
7136 case 1: /* D3 vline */
7137 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7138 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7139 DRM_DEBUG("IH: D3 vline\n");
7140 }
7141 break;
7142 default:
7143 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7144 break;
7145 }
7146 break;
7147 case 4: /* D4 vblank/vline */
7148 switch (src_data) {
7149 case 0: /* D4 vblank */
7150 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7151 if (rdev->irq.crtc_vblank_int[3]) {
7152 drm_handle_vblank(rdev->ddev, 3);
7153 rdev->pm.vblank_sync = true;
7154 wake_up(&rdev->irq.vblank_queue);
7155 }
7156 if (atomic_read(&rdev->irq.pflip[3]))
7157 radeon_crtc_handle_flip(rdev, 3);
7158 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7159 DRM_DEBUG("IH: D4 vblank\n");
7160 }
7161 break;
7162 case 1: /* D4 vline */
7163 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7164 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7165 DRM_DEBUG("IH: D4 vline\n");
7166 }
7167 break;
7168 default:
7169 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7170 break;
7171 }
7172 break;
7173 case 5: /* D5 vblank/vline */
7174 switch (src_data) {
7175 case 0: /* D5 vblank */
7176 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7177 if (rdev->irq.crtc_vblank_int[4]) {
7178 drm_handle_vblank(rdev->ddev, 4);
7179 rdev->pm.vblank_sync = true;
7180 wake_up(&rdev->irq.vblank_queue);
7181 }
7182 if (atomic_read(&rdev->irq.pflip[4]))
7183 radeon_crtc_handle_flip(rdev, 4);
7184 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7185 DRM_DEBUG("IH: D5 vblank\n");
7186 }
7187 break;
7188 case 1: /* D5 vline */
7189 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7190 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7191 DRM_DEBUG("IH: D5 vline\n");
7192 }
7193 break;
7194 default:
7195 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7196 break;
7197 }
7198 break;
7199 case 6: /* D6 vblank/vline */
7200 switch (src_data) {
7201 case 0: /* D6 vblank */
7202 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7203 if (rdev->irq.crtc_vblank_int[5]) {
7204 drm_handle_vblank(rdev->ddev, 5);
7205 rdev->pm.vblank_sync = true;
7206 wake_up(&rdev->irq.vblank_queue);
7207 }
7208 if (atomic_read(&rdev->irq.pflip[5]))
7209 radeon_crtc_handle_flip(rdev, 5);
7210 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7211 DRM_DEBUG("IH: D6 vblank\n");
7212 }
7213 break;
7214 case 1: /* D6 vline */
7215 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7216 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7217 DRM_DEBUG("IH: D6 vline\n");
7218 }
7219 break;
7220 default:
7221 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7222 break;
7223 }
7224 break;
7225 case 42: /* HPD hotplug */
7226 switch (src_data) {
7227 case 0:
7228 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7229 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7230 queue_hotplug = true;
7231 DRM_DEBUG("IH: HPD1\n");
7232 }
7233 break;
7234 case 1:
7235 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7236 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7237 queue_hotplug = true;
7238 DRM_DEBUG("IH: HPD2\n");
7239 }
7240 break;
7241 case 2:
7242 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7243 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7244 queue_hotplug = true;
7245 DRM_DEBUG("IH: HPD3\n");
7246 }
7247 break;
7248 case 3:
7249 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7250 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7251 queue_hotplug = true;
7252 DRM_DEBUG("IH: HPD4\n");
7253 }
7254 break;
7255 case 4:
7256 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7257 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7258 queue_hotplug = true;
7259 DRM_DEBUG("IH: HPD5\n");
7260 }
7261 break;
7262 case 5:
7263 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7264 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7265 queue_hotplug = true;
7266 DRM_DEBUG("IH: HPD6\n");
7267 }
7268 break;
7269 default:
7270 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7271 break;
7272 }
7273 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7274 case 124: /* UVD */
7275 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7276 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7277 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7278 case 146:
7279 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7280 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7281 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7282 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7283 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7284 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7285 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7286 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7287 status);
7288 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7289 /* reset addr and status */
7290 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7291 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7292 case 176: /* GFX RB CP_INT */
7293 case 177: /* GFX IB CP_INT */
7294 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7295 break;
7296 case 181: /* CP EOP event */
7297 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7298 /* XXX check the bitfield order! */
7299 me_id = (ring_id & 0x60) >> 5;
7300 pipe_id = (ring_id & 0x18) >> 3;
7301 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7302 switch (me_id) {
7303 case 0:
7304 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7305 break;
7306 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7307 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7308 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7309 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7310 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7311 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7312 break;
7313 }
7314 break;
7315 case 184: /* CP Privileged reg access */
7316 DRM_ERROR("Illegal register access in command stream\n");
7317 /* XXX check the bitfield order! */
7318 me_id = (ring_id & 0x60) >> 5;
7319 pipe_id = (ring_id & 0x18) >> 3;
7320 queue_id = (ring_id & 0x7) >> 0;
7321 switch (me_id) {
7322 case 0:
7323 /* This results in a full GPU reset, but all we need to do is soft
7324 * reset the CP for gfx
7325 */
7326 queue_reset = true;
7327 break;
7328 case 1:
7329 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7330 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7331 break;
7332 case 2:
7333 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7334 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7335 break;
7336 }
7337 break;
7338 case 185: /* CP Privileged inst */
7339 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7340 /* XXX check the bitfield order! */
7341 me_id = (ring_id & 0x60) >> 5;
7342 pipe_id = (ring_id & 0x18) >> 3;
7343 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7344 switch (me_id) {
7345 case 0:
7346 /* This results in a full GPU reset, but all we need to do is soft
7347 * reset the CP for gfx
7348 */
7349 queue_reset = true;
7350 break;
7351 case 1:
7352 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7353 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7354 break;
7355 case 2:
7356 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7357 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7358 break;
7359 }
7360 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7361 case 224: /* SDMA trap event */
7362 /* XXX check the bitfield order! */
7363 me_id = (ring_id & 0x3) >> 0;
7364 queue_id = (ring_id & 0xc) >> 2;
7365 DRM_DEBUG("IH: SDMA trap\n");
7366 switch (me_id) {
7367 case 0:
7368 switch (queue_id) {
7369 case 0:
7370 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7371 break;
7372 case 1:
7373 /* XXX compute */
7374 break;
7375 case 2:
7376 /* XXX compute */
7377 break;
7378 }
7379 break;
7380 case 1:
7381 switch (queue_id) {
7382 case 0:
7383 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7384 break;
7385 case 1:
7386 /* XXX compute */
7387 break;
7388 case 2:
7389 /* XXX compute */
7390 break;
7391 }
7392 break;
7393 }
7394 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7395 case 230: /* thermal low to high */
7396 DRM_DEBUG("IH: thermal low to high\n");
7397 rdev->pm.dpm.thermal.high_to_low = false;
7398 queue_thermal = true;
7399 break;
7400 case 231: /* thermal high to low */
7401 DRM_DEBUG("IH: thermal high to low\n");
7402 rdev->pm.dpm.thermal.high_to_low = true;
7403 queue_thermal = true;
7404 break;
7405 case 233: /* GUI IDLE */
7406 DRM_DEBUG("IH: GUI idle\n");
7407 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7408 case 241: /* SDMA Privileged inst */
7409 case 247: /* SDMA Privileged inst */
7410 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7411 /* XXX check the bitfield order! */
7412 me_id = (ring_id & 0x3) >> 0;
7413 queue_id = (ring_id & 0xc) >> 2;
7414 switch (me_id) {
7415 case 0:
7416 switch (queue_id) {
7417 case 0:
7418 queue_reset = true;
7419 break;
7420 case 1:
7421 /* XXX compute */
7422 queue_reset = true;
7423 break;
7424 case 2:
7425 /* XXX compute */
7426 queue_reset = true;
7427 break;
7428 }
7429 break;
7430 case 1:
7431 switch (queue_id) {
7432 case 0:
7433 queue_reset = true;
7434 break;
7435 case 1:
7436 /* XXX compute */
7437 queue_reset = true;
7438 break;
7439 case 2:
7440 /* XXX compute */
7441 queue_reset = true;
7442 break;
7443 }
7444 break;
7445 }
7446 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7447 default:
7448 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7449 break;
7450 }
7451
7452 /* wptr/rptr are in bytes! */
7453 rptr += 16;
7454 rptr &= rdev->ih.ptr_mask;
7455 }
7456 if (queue_hotplug)
7457 schedule_work(&rdev->hotplug_work);
7458 if (queue_reset)
7459 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7460 if (queue_thermal)
7461 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7462 rdev->ih.rptr = rptr;
7463 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7464 atomic_set(&rdev->ih.lock, 0);
7465
7466 /* make sure wptr hasn't changed while processing */
7467 wptr = cik_get_ih_wptr(rdev);
7468 if (wptr != rptr)
7469 goto restart_ih;
7470
7471 return IRQ_HANDLED;
7472}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7473
7474/*
7475 * startup/shutdown callbacks
7476 */
7477/**
7478 * cik_startup - program the asic to a functional state
7479 *
7480 * @rdev: radeon_device pointer
7481 *
7482 * Programs the asic to a functional state (CIK).
7483 * Called by cik_init() and cik_resume().
7484 * Returns 0 for success, error for failure.
7485 */
7486static int cik_startup(struct radeon_device *rdev)
7487{
7488 struct radeon_ring *ring;
7489 int r;
7490
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7491 /* enable pcie gen2/3 link */
7492 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7493 /* enable aspm */
7494 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7495
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7496 /* scratch needs to be initialized before MC */
7497 r = r600_vram_scratch_init(rdev);
7498 if (r)
7499 return r;
7500
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7501 cik_mc_program(rdev);
7502
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7503 if (rdev->flags & RADEON_IS_IGP) {
7504 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7505 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7506 r = cik_init_microcode(rdev);
7507 if (r) {
7508 DRM_ERROR("Failed to load firmware!\n");
7509 return r;
7510 }
7511 }
7512 } else {
7513 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7514 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7515 !rdev->mc_fw) {
7516 r = cik_init_microcode(rdev);
7517 if (r) {
7518 DRM_ERROR("Failed to load firmware!\n");
7519 return r;
7520 }
7521 }
7522
7523 r = ci_mc_load_microcode(rdev);
7524 if (r) {
7525 DRM_ERROR("Failed to load MC firmware!\n");
7526 return r;
7527 }
7528 }
7529
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7530 r = cik_pcie_gart_enable(rdev);
7531 if (r)
7532 return r;
7533 cik_gpu_init(rdev);
7534
7535 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7536 if (rdev->flags & RADEON_IS_IGP) {
7537 if (rdev->family == CHIP_KAVERI) {
7538 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7539 rdev->rlc.reg_list_size =
7540 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7541 } else {
7542 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7543 rdev->rlc.reg_list_size =
7544 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7545 }
7546 }
7547 rdev->rlc.cs_data = ci_cs_data;
7548 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7549 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7550 if (r) {
7551 DRM_ERROR("Failed to init rlc BOs!\n");
7552 return r;
7553 }
7554
7555 /* allocate wb buffer */
7556 r = radeon_wb_init(rdev);
7557 if (r)
7558 return r;
7559
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7560 /* allocate mec buffers */
7561 r = cik_mec_init(rdev);
7562 if (r) {
7563 DRM_ERROR("Failed to init MEC BOs!\n");
7564 return r;
7565 }
7566
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7567 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7568 if (r) {
7569 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7570 return r;
7571 }
7572
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7573 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7574 if (r) {
7575 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7576 return r;
7577 }
7578
7579 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7580 if (r) {
7581 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7582 return r;
7583 }
7584
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7585 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7586 if (r) {
7587 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7588 return r;
7589 }
7590
7591 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7592 if (r) {
7593 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7594 return r;
7595 }
7596
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7597 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7598 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7599 r = uvd_v4_2_resume(rdev);
7600 if (!r) {
7601 r = radeon_fence_driver_start_ring(rdev,
7602 R600_RING_TYPE_UVD_INDEX);
7603 if (r)
7604 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7605 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7606 }
7607 if (r)
7608 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7609
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7610 /* Enable IRQ */
7611 if (!rdev->irq.installed) {
7612 r = radeon_irq_kms_init(rdev);
7613 if (r)
7614 return r;
7615 }
7616
7617 r = cik_irq_init(rdev);
7618 if (r) {
7619 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7620 radeon_irq_kms_fini(rdev);
7621 return r;
7622 }
7623 cik_irq_set(rdev);
7624
7625 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7626 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7627 CP_RB0_RPTR, CP_RB0_WPTR,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7628 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7629 if (r)
7630 return r;
7631
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7632 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7633 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7634 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7635 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7636 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7637 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7638 if (r)
7639 return r;
7640 ring->me = 1; /* first MEC */
7641 ring->pipe = 0; /* first pipe */
7642 ring->queue = 0; /* first queue */
7643 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7644
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7645 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7646 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7647 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7648 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7649 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7650 if (r)
7651 return r;
7652 /* dGPU only have 1 MEC */
7653 ring->me = 1; /* first MEC */
7654 ring->pipe = 0; /* first pipe */
7655 ring->queue = 1; /* second queue */
7656 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7657
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7658 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7659 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7660 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7661 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7662 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7663 if (r)
7664 return r;
7665
7666 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7667 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7668 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7669 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7670 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7671 if (r)
7672 return r;
7673
7674 r = cik_cp_resume(rdev);
7675 if (r)
7676 return r;
7677
7678 r = cik_sdma_resume(rdev);
7679 if (r)
7680 return r;
7681
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7682 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7683 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7684 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7685 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7686 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7687 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7688 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7689 if (r)
7690 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7691 }
7692
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7693 r = radeon_ib_pool_init(rdev);
7694 if (r) {
7695 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7696 return r;
7697 }
7698
7699 r = radeon_vm_manager_init(rdev);
7700 if (r) {
7701 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7702 return r;
7703 }
7704
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7705 r = dce6_audio_init(rdev);
7706 if (r)
7707 return r;
7708
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7709 return 0;
7710}
7711
7712/**
7713 * cik_resume - resume the asic to a functional state
7714 *
7715 * @rdev: radeon_device pointer
7716 *
7717 * Programs the asic to a functional state (CIK).
7718 * Called at resume.
7719 * Returns 0 for success, error for failure.
7720 */
7721int cik_resume(struct radeon_device *rdev)
7722{
7723 int r;
7724
7725 /* post card */
7726 atom_asic_init(rdev->mode_info.atom_context);
7727
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7728 /* init golden registers */
7729 cik_init_golden_registers(rdev);
7730
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7731 rdev->accel_working = true;
7732 r = cik_startup(rdev);
7733 if (r) {
7734 DRM_ERROR("cik startup failed on resume\n");
7735 rdev->accel_working = false;
7736 return r;
7737 }
7738
7739 return r;
7740
7741}
7742
7743/**
7744 * cik_suspend - suspend the asic
7745 *
7746 * @rdev: radeon_device pointer
7747 *
7748 * Bring the chip into a state suitable for suspend (CIK).
7749 * Called at suspend.
7750 * Returns 0 for success.
7751 */
7752int cik_suspend(struct radeon_device *rdev)
7753{
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7754 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7755 radeon_vm_manager_fini(rdev);
7756 cik_cp_enable(rdev, false);
7757 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7758 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7759 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7760 cik_fini_pg(rdev);
7761 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7762 cik_irq_suspend(rdev);
7763 radeon_wb_disable(rdev);
7764 cik_pcie_gart_disable(rdev);
7765 return 0;
7766}
7767
7768/* Plan is to move initialization in that function and use
7769 * helper function so that radeon_device_init pretty much
7770 * do nothing more than calling asic specific function. This
7771 * should also allow to remove a bunch of callback function
7772 * like vram_info.
7773 */
7774/**
7775 * cik_init - asic specific driver and hw init
7776 *
7777 * @rdev: radeon_device pointer
7778 *
7779 * Setup asic specific driver variables and program the hw
7780 * to a functional state (CIK).
7781 * Called at driver startup.
7782 * Returns 0 for success, errors for failure.
7783 */
7784int cik_init(struct radeon_device *rdev)
7785{
7786 struct radeon_ring *ring;
7787 int r;
7788
7789 /* Read BIOS */
7790 if (!radeon_get_bios(rdev)) {
7791 if (ASIC_IS_AVIVO(rdev))
7792 return -EINVAL;
7793 }
7794 /* Must be an ATOMBIOS */
7795 if (!rdev->is_atom_bios) {
7796 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7797 return -EINVAL;
7798 }
7799 r = radeon_atombios_init(rdev);
7800 if (r)
7801 return r;
7802
7803 /* Post card if necessary */
7804 if (!radeon_card_posted(rdev)) {
7805 if (!rdev->bios) {
7806 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7807 return -EINVAL;
7808 }
7809 DRM_INFO("GPU not posted. posting now...\n");
7810 atom_asic_init(rdev->mode_info.atom_context);
7811 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7812 /* init golden registers */
7813 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7814 /* Initialize scratch registers */
7815 cik_scratch_init(rdev);
7816 /* Initialize surface registers */
7817 radeon_surface_init(rdev);
7818 /* Initialize clocks */
7819 radeon_get_clock_info(rdev->ddev);
7820
7821 /* Fence driver */
7822 r = radeon_fence_driver_init(rdev);
7823 if (r)
7824 return r;
7825
7826 /* initialize memory controller */
7827 r = cik_mc_init(rdev);
7828 if (r)
7829 return r;
7830 /* Memory manager */
7831 r = radeon_bo_init(rdev);
7832 if (r)
7833 return r;
7834
7835 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7836 ring->ring_obj = NULL;
7837 r600_ring_init(rdev, ring, 1024 * 1024);
7838
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7839 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7840 ring->ring_obj = NULL;
7841 r600_ring_init(rdev, ring, 1024 * 1024);
7842 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7843 if (r)
7844 return r;
7845
7846 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7847 ring->ring_obj = NULL;
7848 r600_ring_init(rdev, ring, 1024 * 1024);
7849 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7850 if (r)
7851 return r;
7852
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7853 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7854 ring->ring_obj = NULL;
7855 r600_ring_init(rdev, ring, 256 * 1024);
7856
7857 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7858 ring->ring_obj = NULL;
7859 r600_ring_init(rdev, ring, 256 * 1024);
7860
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7861 r = radeon_uvd_init(rdev);
7862 if (!r) {
7863 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7864 ring->ring_obj = NULL;
7865 r600_ring_init(rdev, ring, 4096);
7866 }
7867
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7868 rdev->ih.ring_obj = NULL;
7869 r600_ih_ring_init(rdev, 64 * 1024);
7870
7871 r = r600_pcie_gart_init(rdev);
7872 if (r)
7873 return r;
7874
7875 rdev->accel_working = true;
7876 r = cik_startup(rdev);
7877 if (r) {
7878 dev_err(rdev->dev, "disabling GPU acceleration\n");
7879 cik_cp_fini(rdev);
7880 cik_sdma_fini(rdev);
7881 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7882 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7883 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7884 radeon_wb_fini(rdev);
7885 radeon_ib_pool_fini(rdev);
7886 radeon_vm_manager_fini(rdev);
7887 radeon_irq_kms_fini(rdev);
7888 cik_pcie_gart_fini(rdev);
7889 rdev->accel_working = false;
7890 }
7891
7892 /* Don't start up if the MC ucode is missing.
7893 * The default clocks and voltages before the MC ucode
7894 * is loaded are not suffient for advanced operations.
7895 */
7896 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7897 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7898 return -EINVAL;
7899 }
7900
7901 return 0;
7902}
7903
7904/**
7905 * cik_fini - asic specific driver and hw fini
7906 *
7907 * @rdev: radeon_device pointer
7908 *
7909 * Tear down the asic specific driver variables and program the hw
7910 * to an idle state (CIK).
7911 * Called at driver unload.
7912 */
7913void cik_fini(struct radeon_device *rdev)
7914{
7915 cik_cp_fini(rdev);
7916 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7917 cik_fini_pg(rdev);
7918 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7919 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7920 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7921 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7922 radeon_wb_fini(rdev);
7923 radeon_vm_manager_fini(rdev);
7924 radeon_ib_pool_fini(rdev);
7925 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7926 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7927 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7928 cik_pcie_gart_fini(rdev);
7929 r600_vram_scratch_fini(rdev);
7930 radeon_gem_fini(rdev);
7931 radeon_fence_driver_fini(rdev);
7932 radeon_bo_fini(rdev);
7933 radeon_atombios_fini(rdev);
7934 kfree(rdev->bios);
7935 rdev->bios = NULL;
7936}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7937
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7938void dce8_program_fmt(struct drm_encoder *encoder)
7939{
7940 struct drm_device *dev = encoder->dev;
7941 struct radeon_device *rdev = dev->dev_private;
7942 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7943 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7944 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7945 int bpc = 0;
7946 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7947 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7948
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7949 if (connector) {
7950 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7951 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7952 dither = radeon_connector->dither;
7953 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7954
7955 /* LVDS/eDP FMT is set up by atom */
7956 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7957 return;
7958
7959 /* not needed for analog */
7960 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7961 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7962 return;
7963
7964 if (bpc == 0)
7965 return;
7966
7967 switch (bpc) {
7968 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7969 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7970 /* XXX sort out optimal dither settings */
7971 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7972 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7973 else
7974 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7975 break;
7976 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7977 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7978 /* XXX sort out optimal dither settings */
7979 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7980 FMT_RGB_RANDOM_ENABLE |
7981 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7982 else
7983 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7984 break;
7985 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7986 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7987 /* XXX sort out optimal dither settings */
7988 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7989 FMT_RGB_RANDOM_ENABLE |
7990 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7991 else
7992 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7993 break;
7994 default:
7995 /* not needed */
7996 break;
7997 }
7998
7999 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8000}
8001
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8002/* display watermark setup */
8003/**
8004 * dce8_line_buffer_adjust - Set up the line buffer
8005 *
8006 * @rdev: radeon_device pointer
8007 * @radeon_crtc: the selected display controller
8008 * @mode: the current display mode on the selected display
8009 * controller
8010 *
8011 * Setup up the line buffer allocation for
8012 * the selected display controller (CIK).
8013 * Returns the line buffer size in pixels.
8014 */
8015static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8016 struct radeon_crtc *radeon_crtc,
8017 struct drm_display_mode *mode)
8018{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8019 u32 tmp, buffer_alloc, i;
8020 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8021 /*
8022 * Line Buffer Setup
8023 * There are 6 line buffers, one for each display controllers.
8024 * There are 3 partitions per LB. Select the number of partitions
8025 * to enable based on the display width. For display widths larger
8026 * than 4096, you need use to use 2 display controllers and combine
8027 * them using the stereo blender.
8028 */
8029 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8030 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8031 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8032 buffer_alloc = 2;
8033 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8034 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8035 buffer_alloc = 2;
8036 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8037 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8038 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8039 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8040 DRM_DEBUG_KMS("Mode too big for LB!\n");
8041 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8042 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8043 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8044 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8045 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8046 buffer_alloc = 0;
8047 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8048
8049 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8050 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8051
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8052 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8053 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8054 for (i = 0; i < rdev->usec_timeout; i++) {
8055 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8056 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8057 break;
8058 udelay(1);
8059 }
8060
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8061 if (radeon_crtc->base.enabled && mode) {
8062 switch (tmp) {
8063 case 0:
8064 default:
8065 return 4096 * 2;
8066 case 1:
8067 return 1920 * 2;
8068 case 2:
8069 return 2560 * 2;
8070 }
8071 }
8072
8073 /* controller not enabled, so no lb used */
8074 return 0;
8075}
8076
8077/**
8078 * cik_get_number_of_dram_channels - get the number of dram channels
8079 *
8080 * @rdev: radeon_device pointer
8081 *
8082 * Look up the number of video ram channels (CIK).
8083 * Used for display watermark bandwidth calculations
8084 * Returns the number of dram channels
8085 */
8086static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8087{
8088 u32 tmp = RREG32(MC_SHARED_CHMAP);
8089
8090 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8091 case 0:
8092 default:
8093 return 1;
8094 case 1:
8095 return 2;
8096 case 2:
8097 return 4;
8098 case 3:
8099 return 8;
8100 case 4:
8101 return 3;
8102 case 5:
8103 return 6;
8104 case 6:
8105 return 10;
8106 case 7:
8107 return 12;
8108 case 8:
8109 return 16;
8110 }
8111}
8112
8113struct dce8_wm_params {
8114 u32 dram_channels; /* number of dram channels */
8115 u32 yclk; /* bandwidth per dram data pin in kHz */
8116 u32 sclk; /* engine clock in kHz */
8117 u32 disp_clk; /* display clock in kHz */
8118 u32 src_width; /* viewport width */
8119 u32 active_time; /* active display time in ns */
8120 u32 blank_time; /* blank time in ns */
8121 bool interlaced; /* mode is interlaced */
8122 fixed20_12 vsc; /* vertical scale ratio */
8123 u32 num_heads; /* number of active crtcs */
8124 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8125 u32 lb_size; /* line buffer allocated to pipe */
8126 u32 vtaps; /* vertical scaler taps */
8127};
8128
8129/**
8130 * dce8_dram_bandwidth - get the dram bandwidth
8131 *
8132 * @wm: watermark calculation data
8133 *
8134 * Calculate the raw dram bandwidth (CIK).
8135 * Used for display watermark bandwidth calculations
8136 * Returns the dram bandwidth in MBytes/s
8137 */
8138static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8139{
8140 /* Calculate raw DRAM Bandwidth */
8141 fixed20_12 dram_efficiency; /* 0.7 */
8142 fixed20_12 yclk, dram_channels, bandwidth;
8143 fixed20_12 a;
8144
8145 a.full = dfixed_const(1000);
8146 yclk.full = dfixed_const(wm->yclk);
8147 yclk.full = dfixed_div(yclk, a);
8148 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8149 a.full = dfixed_const(10);
8150 dram_efficiency.full = dfixed_const(7);
8151 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8152 bandwidth.full = dfixed_mul(dram_channels, yclk);
8153 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8154
8155 return dfixed_trunc(bandwidth);
8156}
8157
8158/**
8159 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8160 *
8161 * @wm: watermark calculation data
8162 *
8163 * Calculate the dram bandwidth used for display (CIK).
8164 * Used for display watermark bandwidth calculations
8165 * Returns the dram bandwidth for display in MBytes/s
8166 */
8167static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8168{
8169 /* Calculate DRAM Bandwidth and the part allocated to display. */
8170 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8171 fixed20_12 yclk, dram_channels, bandwidth;
8172 fixed20_12 a;
8173
8174 a.full = dfixed_const(1000);
8175 yclk.full = dfixed_const(wm->yclk);
8176 yclk.full = dfixed_div(yclk, a);
8177 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8178 a.full = dfixed_const(10);
8179 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8180 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8181 bandwidth.full = dfixed_mul(dram_channels, yclk);
8182 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8183
8184 return dfixed_trunc(bandwidth);
8185}
8186
8187/**
8188 * dce8_data_return_bandwidth - get the data return bandwidth
8189 *
8190 * @wm: watermark calculation data
8191 *
8192 * Calculate the data return bandwidth used for display (CIK).
8193 * Used for display watermark bandwidth calculations
8194 * Returns the data return bandwidth in MBytes/s
8195 */
8196static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8197{
8198 /* Calculate the display Data return Bandwidth */
8199 fixed20_12 return_efficiency; /* 0.8 */
8200 fixed20_12 sclk, bandwidth;
8201 fixed20_12 a;
8202
8203 a.full = dfixed_const(1000);
8204 sclk.full = dfixed_const(wm->sclk);
8205 sclk.full = dfixed_div(sclk, a);
8206 a.full = dfixed_const(10);
8207 return_efficiency.full = dfixed_const(8);
8208 return_efficiency.full = dfixed_div(return_efficiency, a);
8209 a.full = dfixed_const(32);
8210 bandwidth.full = dfixed_mul(a, sclk);
8211 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8212
8213 return dfixed_trunc(bandwidth);
8214}
8215
8216/**
8217 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8218 *
8219 * @wm: watermark calculation data
8220 *
8221 * Calculate the dmif bandwidth used for display (CIK).
8222 * Used for display watermark bandwidth calculations
8223 * Returns the dmif bandwidth in MBytes/s
8224 */
8225static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8226{
8227 /* Calculate the DMIF Request Bandwidth */
8228 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8229 fixed20_12 disp_clk, bandwidth;
8230 fixed20_12 a, b;
8231
8232 a.full = dfixed_const(1000);
8233 disp_clk.full = dfixed_const(wm->disp_clk);
8234 disp_clk.full = dfixed_div(disp_clk, a);
8235 a.full = dfixed_const(32);
8236 b.full = dfixed_mul(a, disp_clk);
8237
8238 a.full = dfixed_const(10);
8239 disp_clk_request_efficiency.full = dfixed_const(8);
8240 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8241
8242 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8243
8244 return dfixed_trunc(bandwidth);
8245}
8246
8247/**
8248 * dce8_available_bandwidth - get the min available bandwidth
8249 *
8250 * @wm: watermark calculation data
8251 *
8252 * Calculate the min available bandwidth used for display (CIK).
8253 * Used for display watermark bandwidth calculations
8254 * Returns the min available bandwidth in MBytes/s
8255 */
8256static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8257{
8258 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8259 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8260 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8261 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8262
8263 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8264}
8265
8266/**
8267 * dce8_average_bandwidth - get the average available bandwidth
8268 *
8269 * @wm: watermark calculation data
8270 *
8271 * Calculate the average available bandwidth used for display (CIK).
8272 * Used for display watermark bandwidth calculations
8273 * Returns the average available bandwidth in MBytes/s
8274 */
8275static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8276{
8277 /* Calculate the display mode Average Bandwidth
8278 * DisplayMode should contain the source and destination dimensions,
8279 * timing, etc.
8280 */
8281 fixed20_12 bpp;
8282 fixed20_12 line_time;
8283 fixed20_12 src_width;
8284 fixed20_12 bandwidth;
8285 fixed20_12 a;
8286
8287 a.full = dfixed_const(1000);
8288 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8289 line_time.full = dfixed_div(line_time, a);
8290 bpp.full = dfixed_const(wm->bytes_per_pixel);
8291 src_width.full = dfixed_const(wm->src_width);
8292 bandwidth.full = dfixed_mul(src_width, bpp);
8293 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8294 bandwidth.full = dfixed_div(bandwidth, line_time);
8295
8296 return dfixed_trunc(bandwidth);
8297}
8298
8299/**
8300 * dce8_latency_watermark - get the latency watermark
8301 *
8302 * @wm: watermark calculation data
8303 *
8304 * Calculate the latency watermark (CIK).
8305 * Used for display watermark bandwidth calculations
8306 * Returns the latency watermark in ns
8307 */
8308static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8309{
8310 /* First calculate the latency in ns */
8311 u32 mc_latency = 2000; /* 2000 ns. */
8312 u32 available_bandwidth = dce8_available_bandwidth(wm);
8313 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8314 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8315 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8316 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8317 (wm->num_heads * cursor_line_pair_return_time);
8318 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8319 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8320 u32 tmp, dmif_size = 12288;
8321 fixed20_12 a, b, c;
8322
8323 if (wm->num_heads == 0)
8324 return 0;
8325
8326 a.full = dfixed_const(2);
8327 b.full = dfixed_const(1);
8328 if ((wm->vsc.full > a.full) ||
8329 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8330 (wm->vtaps >= 5) ||
8331 ((wm->vsc.full >= a.full) && wm->interlaced))
8332 max_src_lines_per_dst_line = 4;
8333 else
8334 max_src_lines_per_dst_line = 2;
8335
8336 a.full = dfixed_const(available_bandwidth);
8337 b.full = dfixed_const(wm->num_heads);
8338 a.full = dfixed_div(a, b);
8339
8340 b.full = dfixed_const(mc_latency + 512);
8341 c.full = dfixed_const(wm->disp_clk);
8342 b.full = dfixed_div(b, c);
8343
8344 c.full = dfixed_const(dmif_size);
8345 b.full = dfixed_div(c, b);
8346
8347 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8348
8349 b.full = dfixed_const(1000);
8350 c.full = dfixed_const(wm->disp_clk);
8351 b.full = dfixed_div(c, b);
8352 c.full = dfixed_const(wm->bytes_per_pixel);
8353 b.full = dfixed_mul(b, c);
8354
8355 lb_fill_bw = min(tmp, dfixed_trunc(b));
8356
8357 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8358 b.full = dfixed_const(1000);
8359 c.full = dfixed_const(lb_fill_bw);
8360 b.full = dfixed_div(c, b);
8361 a.full = dfixed_div(a, b);
8362 line_fill_time = dfixed_trunc(a);
8363
8364 if (line_fill_time < wm->active_time)
8365 return latency;
8366 else
8367 return latency + (line_fill_time - wm->active_time);
8368
8369}
8370
8371/**
8372 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8373 * average and available dram bandwidth
8374 *
8375 * @wm: watermark calculation data
8376 *
8377 * Check if the display average bandwidth fits in the display
8378 * dram bandwidth (CIK).
8379 * Used for display watermark bandwidth calculations
8380 * Returns true if the display fits, false if not.
8381 */
8382static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8383{
8384 if (dce8_average_bandwidth(wm) <=
8385 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8386 return true;
8387 else
8388 return false;
8389}
8390
8391/**
8392 * dce8_average_bandwidth_vs_available_bandwidth - check
8393 * average and available bandwidth
8394 *
8395 * @wm: watermark calculation data
8396 *
8397 * Check if the display average bandwidth fits in the display
8398 * available bandwidth (CIK).
8399 * Used for display watermark bandwidth calculations
8400 * Returns true if the display fits, false if not.
8401 */
8402static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8403{
8404 if (dce8_average_bandwidth(wm) <=
8405 (dce8_available_bandwidth(wm) / wm->num_heads))
8406 return true;
8407 else
8408 return false;
8409}
8410
8411/**
8412 * dce8_check_latency_hiding - check latency hiding
8413 *
8414 * @wm: watermark calculation data
8415 *
8416 * Check latency hiding (CIK).
8417 * Used for display watermark bandwidth calculations
8418 * Returns true if the display fits, false if not.
8419 */
8420static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8421{
8422 u32 lb_partitions = wm->lb_size / wm->src_width;
8423 u32 line_time = wm->active_time + wm->blank_time;
8424 u32 latency_tolerant_lines;
8425 u32 latency_hiding;
8426 fixed20_12 a;
8427
8428 a.full = dfixed_const(1);
8429 if (wm->vsc.full > a.full)
8430 latency_tolerant_lines = 1;
8431 else {
8432 if (lb_partitions <= (wm->vtaps + 1))
8433 latency_tolerant_lines = 1;
8434 else
8435 latency_tolerant_lines = 2;
8436 }
8437
8438 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8439
8440 if (dce8_latency_watermark(wm) <= latency_hiding)
8441 return true;
8442 else
8443 return false;
8444}
8445
8446/**
8447 * dce8_program_watermarks - program display watermarks
8448 *
8449 * @rdev: radeon_device pointer
8450 * @radeon_crtc: the selected display controller
8451 * @lb_size: line buffer size
8452 * @num_heads: number of display controllers in use
8453 *
8454 * Calculate and program the display watermarks for the
8455 * selected display controller (CIK).
8456 */
8457static void dce8_program_watermarks(struct radeon_device *rdev,
8458 struct radeon_crtc *radeon_crtc,
8459 u32 lb_size, u32 num_heads)
8460{
8461 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8462 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8463 u32 pixel_period;
8464 u32 line_time = 0;
8465 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8466 u32 tmp, wm_mask;
8467
8468 if (radeon_crtc->base.enabled && num_heads && mode) {
8469 pixel_period = 1000000 / (u32)mode->clock;
8470 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8471
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8472 /* watermark for high clocks */
8473 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8474 rdev->pm.dpm_enabled) {
8475 wm_high.yclk =
8476 radeon_dpm_get_mclk(rdev, false) * 10;
8477 wm_high.sclk =
8478 radeon_dpm_get_sclk(rdev, false) * 10;
8479 } else {
8480 wm_high.yclk = rdev->pm.current_mclk * 10;
8481 wm_high.sclk = rdev->pm.current_sclk * 10;
8482 }
8483
8484 wm_high.disp_clk = mode->clock;
8485 wm_high.src_width = mode->crtc_hdisplay;
8486 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8487 wm_high.blank_time = line_time - wm_high.active_time;
8488 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8489 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8490 wm_high.interlaced = true;
8491 wm_high.vsc = radeon_crtc->vsc;
8492 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8493 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8494 wm_high.vtaps = 2;
8495 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8496 wm_high.lb_size = lb_size;
8497 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8498 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8499
8500 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8501 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8502
8503 /* possibly force display priority to high */
8504 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8505 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8506 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8507 !dce8_check_latency_hiding(&wm_high) ||
8508 (rdev->disp_priority == 2)) {
8509 DRM_DEBUG_KMS("force priority to high\n");
8510 }
8511
8512 /* watermark for low clocks */
8513 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8514 rdev->pm.dpm_enabled) {
8515 wm_low.yclk =
8516 radeon_dpm_get_mclk(rdev, true) * 10;
8517 wm_low.sclk =
8518 radeon_dpm_get_sclk(rdev, true) * 10;
8519 } else {
8520 wm_low.yclk = rdev->pm.current_mclk * 10;
8521 wm_low.sclk = rdev->pm.current_sclk * 10;
8522 }
8523
8524 wm_low.disp_clk = mode->clock;
8525 wm_low.src_width = mode->crtc_hdisplay;
8526 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8527 wm_low.blank_time = line_time - wm_low.active_time;
8528 wm_low.interlaced = false;
8529 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8530 wm_low.interlaced = true;
8531 wm_low.vsc = radeon_crtc->vsc;
8532 wm_low.vtaps = 1;
8533 if (radeon_crtc->rmx_type != RMX_OFF)
8534 wm_low.vtaps = 2;
8535 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8536 wm_low.lb_size = lb_size;
8537 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8538 wm_low.num_heads = num_heads;
8539
8540 /* set for low clocks */
8541 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8542
8543 /* possibly force display priority to high */
8544 /* should really do this at mode validation time... */
8545 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8546 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8547 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8548 (rdev->disp_priority == 2)) {
8549 DRM_DEBUG_KMS("force priority to high\n");
8550 }
8551 }
8552
8553 /* select wm A */
8554 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8555 tmp = wm_mask;
8556 tmp &= ~LATENCY_WATERMARK_MASK(3);
8557 tmp |= LATENCY_WATERMARK_MASK(1);
8558 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8559 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8560 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8561 LATENCY_HIGH_WATERMARK(line_time)));
8562 /* select wm B */
8563 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8564 tmp &= ~LATENCY_WATERMARK_MASK(3);
8565 tmp |= LATENCY_WATERMARK_MASK(2);
8566 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8567 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8568 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8569 LATENCY_HIGH_WATERMARK(line_time)));
8570 /* restore original selection */
8571 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8572
8573 /* save values for DPM */
8574 radeon_crtc->line_time = line_time;
8575 radeon_crtc->wm_high = latency_watermark_a;
8576 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8577}
8578
8579/**
8580 * dce8_bandwidth_update - program display watermarks
8581 *
8582 * @rdev: radeon_device pointer
8583 *
8584 * Calculate and program the display watermarks and line
8585 * buffer allocation (CIK).
8586 */
8587void dce8_bandwidth_update(struct radeon_device *rdev)
8588{
8589 struct drm_display_mode *mode = NULL;
8590 u32 num_heads = 0, lb_size;
8591 int i;
8592
8593 radeon_update_display_priority(rdev);
8594
8595 for (i = 0; i < rdev->num_crtc; i++) {
8596 if (rdev->mode_info.crtcs[i]->base.enabled)
8597 num_heads++;
8598 }
8599 for (i = 0; i < rdev->num_crtc; i++) {
8600 mode = &rdev->mode_info.crtcs[i]->base.mode;
8601 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8602 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8603 }
8604}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8605
8606/**
8607 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8608 *
8609 * @rdev: radeon_device pointer
8610 *
8611 * Fetches a GPU clock counter snapshot (SI).
8612 * Returns the 64 bit clock counter snapshot.
8613 */
8614uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8615{
8616 uint64_t clock;
8617
8618 mutex_lock(&rdev->gpu_clock_mutex);
8619 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8620 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8621 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8622 mutex_unlock(&rdev->gpu_clock_mutex);
8623 return clock;
8624}
8625
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8626static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8627 u32 cntl_reg, u32 status_reg)
8628{
8629 int r, i;
8630 struct atom_clock_dividers dividers;
8631 uint32_t tmp;
8632
8633 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8634 clock, false, &dividers);
8635 if (r)
8636 return r;
8637
8638 tmp = RREG32_SMC(cntl_reg);
8639 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8640 tmp |= dividers.post_divider;
8641 WREG32_SMC(cntl_reg, tmp);
8642
8643 for (i = 0; i < 100; i++) {
8644 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8645 break;
8646 mdelay(10);
8647 }
8648 if (i == 100)
8649 return -ETIMEDOUT;
8650
8651 return 0;
8652}
8653
8654int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8655{
8656 int r = 0;
8657
8658 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8659 if (r)
8660 return r;
8661
8662 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8663 return r;
8664}
8665
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8666static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8667{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8668 struct pci_dev *root = rdev->pdev->bus->self;
8669 int bridge_pos, gpu_pos;
8670 u32 speed_cntl, mask, current_data_rate;
8671 int ret, i;
8672 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8673
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8674 if (radeon_pcie_gen2 == 0)
8675 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8676
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8677 if (rdev->flags & RADEON_IS_IGP)
8678 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8679
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8680 if (!(rdev->flags & RADEON_IS_PCIE))
8681 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8682
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8683 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8684 if (ret != 0)
8685 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8686
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8687 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8688 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8689
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8690 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8691 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8692 LC_CURRENT_DATA_RATE_SHIFT;
8693 if (mask & DRM_PCIE_SPEED_80) {
8694 if (current_data_rate == 2) {
8695 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8696 return;
8697 }
8698 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8699 } else if (mask & DRM_PCIE_SPEED_50) {
8700 if (current_data_rate == 1) {
8701 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8702 return;
8703 }
8704 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8705 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8706
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8707 bridge_pos = pci_pcie_cap(root);
8708 if (!bridge_pos)
8709 return;
8710
8711 gpu_pos = pci_pcie_cap(rdev->pdev);
8712 if (!gpu_pos)
8713 return;
8714
8715 if (mask & DRM_PCIE_SPEED_80) {
8716 /* re-try equalization if gen3 is not already enabled */
8717 if (current_data_rate != 2) {
8718 u16 bridge_cfg, gpu_cfg;
8719 u16 bridge_cfg2, gpu_cfg2;
8720 u32 max_lw, current_lw, tmp;
8721
8722 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8723 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8724
8725 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8726 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8727
8728 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8729 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8730
8731 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8732 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8733 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8734
8735 if (current_lw < max_lw) {
8736 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8737 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8738 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8739 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8740 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8741 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8742 }
8743 }
8744
8745 for (i = 0; i < 10; i++) {
8746 /* check status */
8747 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8748 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8749 break;
8750
8751 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8752 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8753
8754 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8755 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8756
8757 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8758 tmp |= LC_SET_QUIESCE;
8759 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8760
8761 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8762 tmp |= LC_REDO_EQ;
8763 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8764
8765 mdelay(100);
8766
8767 /* linkctl */
8768 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8769 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8770 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8771 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8772
8773 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8774 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8775 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8776 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8777
8778 /* linkctl2 */
8779 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8780 tmp16 &= ~((1 << 4) | (7 << 9));
8781 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8782 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8783
8784 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8785 tmp16 &= ~((1 << 4) | (7 << 9));
8786 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8787 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8788
8789 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8790 tmp &= ~LC_SET_QUIESCE;
8791 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8792 }
8793 }
8794 }
8795
8796 /* set the link speed */
8797 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8798 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8799 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8800
8801 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8802 tmp16 &= ~0xf;
8803 if (mask & DRM_PCIE_SPEED_80)
8804 tmp16 |= 3; /* gen3 */
8805 else if (mask & DRM_PCIE_SPEED_50)
8806 tmp16 |= 2; /* gen2 */
8807 else
8808 tmp16 |= 1; /* gen1 */
8809 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8810
8811 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8812 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8813 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8814
8815 for (i = 0; i < rdev->usec_timeout; i++) {
8816 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8817 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8818 break;
8819 udelay(1);
8820 }
8821}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]8822
8823static void cik_program_aspm(struct radeon_device *rdev)
8824{
8825 u32 data, orig;
8826 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8827 bool disable_clkreq = false;
8828
8829 if (radeon_aspm == 0)
8830 return;
8831
8832 /* XXX double check IGPs */
8833 if (rdev->flags & RADEON_IS_IGP)
8834 return;
8835
8836 if (!(rdev->flags & RADEON_IS_PCIE))
8837 return;
8838
8839 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8840 data &= ~LC_XMIT_N_FTS_MASK;
8841 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8842 if (orig != data)
8843 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8844
8845 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8846 data |= LC_GO_TO_RECOVERY;
8847 if (orig != data)
8848 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8849
8850 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8851 data |= P_IGNORE_EDB_ERR;
8852 if (orig != data)
8853 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8854
8855 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8856 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8857 data |= LC_PMI_TO_L1_DIS;
8858 if (!disable_l0s)
8859 data |= LC_L0S_INACTIVITY(7);
8860
8861 if (!disable_l1) {
8862 data |= LC_L1_INACTIVITY(7);
8863 data &= ~LC_PMI_TO_L1_DIS;
8864 if (orig != data)
8865 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8866
8867 if (!disable_plloff_in_l1) {
8868 bool clk_req_support;
8869
8870 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8871 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8872 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8873 if (orig != data)
8874 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8875
8876 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8877 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8878 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8879 if (orig != data)
8880 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8881
8882 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8883 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8884 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8885 if (orig != data)
8886 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8887
8888 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8889 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8890 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8891 if (orig != data)
8892 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8893
8894 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8895 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8896 data |= LC_DYN_LANES_PWR_STATE(3);
8897 if (orig != data)
8898 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8899
8900 if (!disable_clkreq) {
8901 struct pci_dev *root = rdev->pdev->bus->self;
8902 u32 lnkcap;
8903
8904 clk_req_support = false;
8905 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8906 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8907 clk_req_support = true;
8908 } else {
8909 clk_req_support = false;
8910 }
8911
8912 if (clk_req_support) {
8913 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8914 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8915 if (orig != data)
8916 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8917
8918 orig = data = RREG32_SMC(THM_CLK_CNTL);
8919 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8920 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8921 if (orig != data)
8922 WREG32_SMC(THM_CLK_CNTL, data);
8923
8924 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8925 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8926 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8927 if (orig != data)
8928 WREG32_SMC(MISC_CLK_CTRL, data);
8929
8930 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8931 data &= ~BCLK_AS_XCLK;
8932 if (orig != data)
8933 WREG32_SMC(CG_CLKPIN_CNTL, data);
8934
8935 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8936 data &= ~FORCE_BIF_REFCLK_EN;
8937 if (orig != data)
8938 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8939
8940 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8941 data &= ~MPLL_CLKOUT_SEL_MASK;
8942 data |= MPLL_CLKOUT_SEL(4);
8943 if (orig != data)
8944 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8945 }
8946 }
8947 } else {
8948 if (orig != data)
8949 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8950 }
8951
8952 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8953 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8954 if (orig != data)
8955 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8956
8957 if (!disable_l0s) {
8958 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8959 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8960 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8961 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8962 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8963 data &= ~LC_L0S_INACTIVITY_MASK;
8964 if (orig != data)
8965 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8966 }
8967 }
8968 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8969}