Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.19 / 5d2590673f3e5c0c0ecf496fb48c73e3ee3ee573 / . / drivers / gpu / drm / radeon / cik.c
blob: e8ec15dfe5f8d94dc242b7478fd1c9f1ee58ddd4 [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]78static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]80static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]87
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]124
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]130 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]131 u32 r;
132
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]151}
152
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1532 default:
1533 break;
1534 }
1535}
1536
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1563 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1564 *
1565 * Returns the value in the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1566 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1567 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1569{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1572 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1582 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1586 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1587 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1589{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1592 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1594 }
1595}
1596
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]1700int ci_mc_load_microcode(struct radeon_device *rdev)
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1802 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1904 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1925 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1932 }
1933
1934out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1952 }
1953 return err;
1954}
1955
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1994 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1995
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2432 }
2433 } else if (num_pipe_configs == 4) {
2434 if (num_rbs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2437 case 0:
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2442 break;
2443 case 1:
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2448 break;
2449 case 2:
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2454 break;
2455 case 3:
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2460 break;
2461 case 4:
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2466 break;
2467 case 5:
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2470 break;
2471 case 6:
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2476 break;
2477 case 7:
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2482 break;
2483 case 8:
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2486 break;
2487 case 9:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2490 break;
2491 case 10:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2496 break;
2497 case 11:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2502 break;
2503 case 12:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2508 break;
2509 case 13:
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2512 break;
2513 case 14:
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518 break;
2519 case 16:
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2524 break;
2525 case 17:
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 break;
2531 case 27:
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2534 break;
2535 case 28:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 29:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 30:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 default:
2554 gb_tile_moden = 0;
2555 break;
2556 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2559 }
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2563 case 0:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2568 break;
2569 case 1:
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2574 break;
2575 case 2:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 break;
2581 case 3:
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2586 break;
2587 case 4:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2592 break;
2593 case 5:
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2596 break;
2597 case 6:
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2602 break;
2603 case 7:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2608 break;
2609 case 8:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2612 break;
2613 case 9:
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2616 break;
2617 case 10:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 11:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 12:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 13:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2638 break;
2639 case 14:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 16:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2650 break;
2651 case 17:
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2656 break;
2657 case 27:
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2660 break;
2661 case 28:
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2666 break;
2667 case 29:
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 break;
2673 case 30:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 default:
2680 gb_tile_moden = 0;
2681 break;
2682 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2685 }
2686 }
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2689 case 0:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2694 break;
2695 case 1:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 2:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 3:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 4:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 5:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 6:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 8:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 9:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 10:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 11:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2754 break;
2755 case 12:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 break;
2761 case 13:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2766 break;
2767 case 14:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2772 break;
2773 default:
2774 gb_tile_moden = 0;
2775 break;
2776 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779 }
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2783 case 0:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2788 break;
2789 case 1:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2794 break;
2795 case 2:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 3:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2806 break;
2807 case 4:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2812 break;
2813 case 5:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2816 break;
2817 case 6:
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2822 break;
2823 case 7:
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2828 break;
2829 case 8:
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2831 break;
2832 case 9:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2835 break;
2836 case 10:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 11:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 12:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 13:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2857 break;
2858 case 14:
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 break;
2864 case 16:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 17:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 27:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2879 break;
2880 case 28:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 29:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 30:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 break;
2898 default:
2899 gb_tile_moden = 0;
2900 break;
2901 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2904 }
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2907 case 0:
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2912 break;
2913 case 1:
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2918 break;
2919 case 2:
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2924 break;
2925 case 3:
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2930 break;
2931 case 4:
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2936 break;
2937 case 5:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 6:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2948 break;
2949 case 8:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 9:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 10:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 11:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 12:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2978 break;
2979 case 13:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 14:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2990 break;
2991 default:
2992 gb_tile_moden = 0;
2993 break;
2994 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2997 }
2998 } else
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3000}
3001
3002/**
3003 * cik_select_se_sh - select which SE, SH to address
3004 *
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3008 *
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3012 */
3013static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3015{
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3017
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3024 else
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3027}
3028
3029/**
3030 * cik_create_bitmask - create a bitmask
3031 *
3032 * @bit_width: length of the mask
3033 *
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3036 */
3037static u32 cik_create_bitmask(u32 bit_width)
3038{
3039 u32 i, mask = 0;
3040
3041 for (i = 0; i < bit_width; i++) {
3042 mask <<= 1;
3043 mask |= 1;
3044 }
3045 return mask;
3046}
3047
3048/**
3049 * cik_select_se_sh - select which SE, SH to address
3050 *
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3055 *
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3058 */
3059static u32 cik_get_rb_disabled(struct radeon_device *rdev,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3060 u32 max_rb_num_per_se,
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3061 u32 sh_per_se)
3062{
3063 u32 data, mask;
3064
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3066 if (data & 1)
3067 data &= BACKEND_DISABLE_MASK;
3068 else
3069 data = 0;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3071
3072 data >>= BACKEND_DISABLE_SHIFT;
3073
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3074 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3075
3076 return data & mask;
3077}
3078
3079/**
3080 * cik_setup_rb - setup the RBs on the asic
3081 *
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3086 *
3087 * Configures per-SE/SH RB registers (CIK).
3088 */
3089static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3091 u32 max_rb_num_per_se)
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3092{
3093 int i, j;
3094 u32 data, mask;
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3097
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3101 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3104 else
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3106 }
3107 }
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3109
3110 mask = 1;
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3111 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3114 mask <<= 1;
3115 }
3116
Marek Olšák439a1cf2013-12-22 02:18:01 +0100[diff] [blame]3117 rdev->config.cik.backend_enable_mask = enabled_rbs;
3118
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3119 for (i = 0; i < se_num; i++) {
3120 cik_select_se_sh(rdev, i, 0xffffffff);
3121 data = 0;
3122 for (j = 0; j < sh_per_se; j++) {
3123 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3124 case 0:
3125 if (j == 0)
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3127 else
3128 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3129 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3130 case 1:
3131 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3132 break;
3133 case 2:
3134 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3135 break;
3136 case 3:
3137 default:
3138 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3139 break;
3140 }
3141 enabled_rbs >>= 2;
3142 }
3143 WREG32(PA_SC_RASTER_CONFIG, data);
3144 }
3145 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3146}
3147
3148/**
3149 * cik_gpu_init - setup the 3D engine
3150 *
3151 * @rdev: radeon_device pointer
3152 *
3153 * Configures the 3D engine and tiling configuration
3154 * registers so that the 3D engine is usable.
3155 */
3156static void cik_gpu_init(struct radeon_device *rdev)
3157{
3158 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3159 u32 mc_shared_chmap, mc_arb_ramcfg;
3160 u32 hdp_host_path_cntl;
3161 u32 tmp;
3162 int i, j;
3163
3164 switch (rdev->family) {
3165 case CHIP_BONAIRE:
3166 rdev->config.cik.max_shader_engines = 2;
3167 rdev->config.cik.max_tile_pipes = 4;
3168 rdev->config.cik.max_cu_per_sh = 7;
3169 rdev->config.cik.max_sh_per_se = 1;
3170 rdev->config.cik.max_backends_per_se = 2;
3171 rdev->config.cik.max_texture_channel_caches = 4;
3172 rdev->config.cik.max_gprs = 256;
3173 rdev->config.cik.max_gs_threads = 32;
3174 rdev->config.cik.max_hw_contexts = 8;
3175
3176 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3177 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3178 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3181 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3182 case CHIP_HAWAII:
3183 rdev->config.cik.max_shader_engines = 4;
3184 rdev->config.cik.max_tile_pipes = 16;
3185 rdev->config.cik.max_cu_per_sh = 11;
3186 rdev->config.cik.max_sh_per_se = 1;
3187 rdev->config.cik.max_backends_per_se = 4;
3188 rdev->config.cik.max_texture_channel_caches = 16;
3189 rdev->config.cik.max_gprs = 256;
3190 rdev->config.cik.max_gs_threads = 32;
3191 rdev->config.cik.max_hw_contexts = 8;
3192
3193 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3194 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3195 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3198 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3199 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3200 rdev->config.cik.max_shader_engines = 1;
3201 rdev->config.cik.max_tile_pipes = 4;
3202 if ((rdev->pdev->device == 0x1304) ||
3203 (rdev->pdev->device == 0x1305) ||
3204 (rdev->pdev->device == 0x130C) ||
3205 (rdev->pdev->device == 0x130F) ||
3206 (rdev->pdev->device == 0x1310) ||
3207 (rdev->pdev->device == 0x1311) ||
3208 (rdev->pdev->device == 0x131C)) {
3209 rdev->config.cik.max_cu_per_sh = 8;
3210 rdev->config.cik.max_backends_per_se = 2;
3211 } else if ((rdev->pdev->device == 0x1309) ||
3212 (rdev->pdev->device == 0x130A) ||
3213 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3214 (rdev->pdev->device == 0x1313) ||
3215 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3216 rdev->config.cik.max_cu_per_sh = 6;
3217 rdev->config.cik.max_backends_per_se = 2;
3218 } else if ((rdev->pdev->device == 0x1306) ||
3219 (rdev->pdev->device == 0x1307) ||
3220 (rdev->pdev->device == 0x130B) ||
3221 (rdev->pdev->device == 0x130E) ||
3222 (rdev->pdev->device == 0x1315) ||
3223 (rdev->pdev->device == 0x131B)) {
3224 rdev->config.cik.max_cu_per_sh = 4;
3225 rdev->config.cik.max_backends_per_se = 1;
3226 } else {
3227 rdev->config.cik.max_cu_per_sh = 3;
3228 rdev->config.cik.max_backends_per_se = 1;
3229 }
3230 rdev->config.cik.max_sh_per_se = 1;
3231 rdev->config.cik.max_texture_channel_caches = 4;
3232 rdev->config.cik.max_gprs = 256;
3233 rdev->config.cik.max_gs_threads = 16;
3234 rdev->config.cik.max_hw_contexts = 8;
3235
3236 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3237 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3238 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3240 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3241 break;
3242 case CHIP_KABINI:
3243 default:
3244 rdev->config.cik.max_shader_engines = 1;
3245 rdev->config.cik.max_tile_pipes = 2;
3246 rdev->config.cik.max_cu_per_sh = 2;
3247 rdev->config.cik.max_sh_per_se = 1;
3248 rdev->config.cik.max_backends_per_se = 1;
3249 rdev->config.cik.max_texture_channel_caches = 2;
3250 rdev->config.cik.max_gprs = 256;
3251 rdev->config.cik.max_gs_threads = 16;
3252 rdev->config.cik.max_hw_contexts = 8;
3253
3254 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3255 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3256 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3257 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3258 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3259 break;
3260 }
3261
3262 /* Initialize HDP */
3263 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3264 WREG32((0x2c14 + j), 0x00000000);
3265 WREG32((0x2c18 + j), 0x00000000);
3266 WREG32((0x2c1c + j), 0x00000000);
3267 WREG32((0x2c20 + j), 0x00000000);
3268 WREG32((0x2c24 + j), 0x00000000);
3269 }
3270
3271 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3272
3273 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3274
3275 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3276 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3277
3278 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3279 rdev->config.cik.mem_max_burst_length_bytes = 256;
3280 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3281 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3282 if (rdev->config.cik.mem_row_size_in_kb > 4)
3283 rdev->config.cik.mem_row_size_in_kb = 4;
3284 /* XXX use MC settings? */
3285 rdev->config.cik.shader_engine_tile_size = 32;
3286 rdev->config.cik.num_gpus = 1;
3287 rdev->config.cik.multi_gpu_tile_size = 64;
3288
3289 /* fix up row size */
3290 gb_addr_config &= ~ROW_SIZE_MASK;
3291 switch (rdev->config.cik.mem_row_size_in_kb) {
3292 case 1:
3293 default:
3294 gb_addr_config |= ROW_SIZE(0);
3295 break;
3296 case 2:
3297 gb_addr_config |= ROW_SIZE(1);
3298 break;
3299 case 4:
3300 gb_addr_config |= ROW_SIZE(2);
3301 break;
3302 }
3303
3304 /* setup tiling info dword. gb_addr_config is not adequate since it does
3305 * not have bank info, so create a custom tiling dword.
3306 * bits 3:0 num_pipes
3307 * bits 7:4 num_banks
3308 * bits 11:8 group_size
3309 * bits 15:12 row_size
3310 */
3311 rdev->config.cik.tile_config = 0;
3312 switch (rdev->config.cik.num_tile_pipes) {
3313 case 1:
3314 rdev->config.cik.tile_config |= (0 << 0);
3315 break;
3316 case 2:
3317 rdev->config.cik.tile_config |= (1 << 0);
3318 break;
3319 case 4:
3320 rdev->config.cik.tile_config |= (2 << 0);
3321 break;
3322 case 8:
3323 default:
3324 /* XXX what about 12? */
3325 rdev->config.cik.tile_config |= (3 << 0);
3326 break;
3327 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3328 rdev->config.cik.tile_config |=
3329 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3332 rdev->config.cik.tile_config |=
3333 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3334
3335 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3336 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3337 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3338 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3339 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3340 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3341 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3342 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3343
3344 cik_tiling_mode_table_init(rdev);
3345
3346 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3347 rdev->config.cik.max_sh_per_se,
3348 rdev->config.cik.max_backends_per_se);
3349
3350 /* set HW defaults for 3D engine */
3351 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3352
3353 WREG32(SX_DEBUG_1, 0x20);
3354
3355 WREG32(TA_CNTL_AUX, 0x00010000);
3356
3357 tmp = RREG32(SPI_CONFIG_CNTL);
3358 tmp |= 0x03000000;
3359 WREG32(SPI_CONFIG_CNTL, tmp);
3360
3361 WREG32(SQ_CONFIG, 1);
3362
3363 WREG32(DB_DEBUG, 0);
3364
3365 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3366 tmp |= 0x00000400;
3367 WREG32(DB_DEBUG2, tmp);
3368
3369 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3370 tmp |= 0x00020200;
3371 WREG32(DB_DEBUG3, tmp);
3372
3373 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3374 tmp |= 0x00018208;
3375 WREG32(CB_HW_CONTROL, tmp);
3376
3377 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3378
3379 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3380 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3381 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3382 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3383
3384 WREG32(VGT_NUM_INSTANCES, 1);
3385
3386 WREG32(CP_PERFMON_CNTL, 0);
3387
3388 WREG32(SQ_CONFIG, 0);
3389
3390 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3391 FORCE_EOV_MAX_REZ_CNT(255)));
3392
3393 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3394 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3395
3396 WREG32(VGT_GS_VERTEX_REUSE, 16);
3397 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3398
3399 tmp = RREG32(HDP_MISC_CNTL);
3400 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3401 WREG32(HDP_MISC_CNTL, tmp);
3402
3403 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3404 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3405
3406 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3407 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3408
3409 udelay(50);
3410}
3411
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3412/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3413 * GPU scratch registers helpers function.
3414 */
3415/**
3416 * cik_scratch_init - setup driver info for CP scratch regs
3417 *
3418 * @rdev: radeon_device pointer
3419 *
3420 * Set up the number and offset of the CP scratch registers.
3421 * NOTE: use of CP scratch registers is a legacy inferface and
3422 * is not used by default on newer asics (r6xx+). On newer asics,
3423 * memory buffers are used for fences rather than scratch regs.
3424 */
3425static void cik_scratch_init(struct radeon_device *rdev)
3426{
3427 int i;
3428
3429 rdev->scratch.num_reg = 7;
3430 rdev->scratch.reg_base = SCRATCH_REG0;
3431 for (i = 0; i < rdev->scratch.num_reg; i++) {
3432 rdev->scratch.free[i] = true;
3433 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3434 }
3435}
3436
3437/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3438 * cik_ring_test - basic gfx ring test
3439 *
3440 * @rdev: radeon_device pointer
3441 * @ring: radeon_ring structure holding ring information
3442 *
3443 * Allocate a scratch register and write to it using the gfx ring (CIK).
3444 * Provides a basic gfx ring test to verify that the ring is working.
3445 * Used by cik_cp_gfx_resume();
3446 * Returns 0 on success, error on failure.
3447 */
3448int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3449{
3450 uint32_t scratch;
3451 uint32_t tmp = 0;
3452 unsigned i;
3453 int r;
3454
3455 r = radeon_scratch_get(rdev, &scratch);
3456 if (r) {
3457 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3458 return r;
3459 }
3460 WREG32(scratch, 0xCAFEDEAD);
3461 r = radeon_ring_lock(rdev, ring, 3);
3462 if (r) {
3463 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3464 radeon_scratch_free(rdev, scratch);
3465 return r;
3466 }
3467 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3468 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3469 radeon_ring_write(ring, 0xDEADBEEF);
3470 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3471
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3472 for (i = 0; i < rdev->usec_timeout; i++) {
3473 tmp = RREG32(scratch);
3474 if (tmp == 0xDEADBEEF)
3475 break;
3476 DRM_UDELAY(1);
3477 }
3478 if (i < rdev->usec_timeout) {
3479 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3480 } else {
3481 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3482 ring->idx, scratch, tmp);
3483 r = -EINVAL;
3484 }
3485 radeon_scratch_free(rdev, scratch);
3486 return r;
3487}
3488
3489/**
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3490 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3491 *
3492 * @rdev: radeon_device pointer
3493 * @ridx: radeon ring index
3494 *
3495 * Emits an hdp flush on the cp.
3496 */
3497static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3498 int ridx)
3499{
3500 struct radeon_ring *ring = &rdev->ring[ridx];
Alex Deucher5d259062014-01-09 16:51:56 -0500[diff] [blame^]3501 u32 ref_and_mask;
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3502
Alex Deucher5d259062014-01-09 16:51:56 -0500[diff] [blame^]3503 switch (ring->idx) {
3504 case CAYMAN_RING_TYPE_CP1_INDEX:
3505 case CAYMAN_RING_TYPE_CP2_INDEX:
3506 default:
3507 switch (ring->me) {
3508 case 0:
3509 ref_and_mask = CP2 << ring->pipe;
3510 break;
3511 case 1:
3512 ref_and_mask = CP6 << ring->pipe;
3513 break;
3514 default:
3515 return;
3516 }
3517 break;
3518 case RADEON_RING_TYPE_GFX_INDEX:
3519 ref_and_mask = CP0;
3520 break;
3521 }
3522
3523 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3524 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3525 WAIT_REG_MEM_FUNCTION(3) | /* == */
3526 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3527 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3528 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3529 radeon_ring_write(ring, ref_and_mask);
3530 radeon_ring_write(ring, ref_and_mask);
3531 radeon_ring_write(ring, 0x20); /* poll interval */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3532}
3533
3534/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3535 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3536 *
3537 * @rdev: radeon_device pointer
3538 * @fence: radeon fence object
3539 *
3540 * Emits a fence sequnce number on the gfx ring and flushes
3541 * GPU caches.
3542 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3543void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3544 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3545{
3546 struct radeon_ring *ring = &rdev->ring[fence->ring];
3547 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3548
3549 /* EVENT_WRITE_EOP - flush caches, send int */
3550 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3551 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3552 EOP_TC_ACTION_EN |
3553 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3554 EVENT_INDEX(5)));
3555 radeon_ring_write(ring, addr & 0xfffffffc);
3556 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3557 radeon_ring_write(ring, fence->seq);
3558 radeon_ring_write(ring, 0);
3559 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3560 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3561}
3562
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3563/**
3564 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3565 *
3566 * @rdev: radeon_device pointer
3567 * @fence: radeon fence object
3568 *
3569 * Emits a fence sequnce number on the compute ring and flushes
3570 * GPU caches.
3571 */
3572void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3573 struct radeon_fence *fence)
3574{
3575 struct radeon_ring *ring = &rdev->ring[fence->ring];
3576 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3577
3578 /* RELEASE_MEM - flush caches, send int */
3579 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3580 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3581 EOP_TC_ACTION_EN |
3582 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3583 EVENT_INDEX(5)));
3584 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3585 radeon_ring_write(ring, addr & 0xfffffffc);
3586 radeon_ring_write(ring, upper_32_bits(addr));
3587 radeon_ring_write(ring, fence->seq);
3588 radeon_ring_write(ring, 0);
3589 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]3590 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3591}
3592
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3593bool cik_semaphore_ring_emit(struct radeon_device *rdev,
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3594 struct radeon_ring *ring,
3595 struct radeon_semaphore *semaphore,
3596 bool emit_wait)
3597{
3598 uint64_t addr = semaphore->gpu_addr;
3599 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3600
3601 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3602 radeon_ring_write(ring, addr & 0xffffffff);
3603 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3604
3605 return true;
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3606}
3607
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3608/**
3609 * cik_copy_cpdma - copy pages using the CP DMA engine
3610 *
3611 * @rdev: radeon_device pointer
3612 * @src_offset: src GPU address
3613 * @dst_offset: dst GPU address
3614 * @num_gpu_pages: number of GPU pages to xfer
3615 * @fence: radeon fence object
3616 *
3617 * Copy GPU paging using the CP DMA engine (CIK+).
3618 * Used by the radeon ttm implementation to move pages if
3619 * registered as the asic copy callback.
3620 */
3621int cik_copy_cpdma(struct radeon_device *rdev,
3622 uint64_t src_offset, uint64_t dst_offset,
3623 unsigned num_gpu_pages,
3624 struct radeon_fence **fence)
3625{
3626 struct radeon_semaphore *sem = NULL;
3627 int ring_index = rdev->asic->copy.blit_ring_index;
3628 struct radeon_ring *ring = &rdev->ring[ring_index];
3629 u32 size_in_bytes, cur_size_in_bytes, control;
3630 int i, num_loops;
3631 int r = 0;
3632
3633 r = radeon_semaphore_create(rdev, &sem);
3634 if (r) {
3635 DRM_ERROR("radeon: moving bo (%d).\n", r);
3636 return r;
3637 }
3638
3639 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3640 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3641 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3642 if (r) {
3643 DRM_ERROR("radeon: moving bo (%d).\n", r);
3644 radeon_semaphore_free(rdev, &sem, NULL);
3645 return r;
3646 }
3647
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3648 radeon_semaphore_sync_to(sem, *fence);
3649 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3650
3651 for (i = 0; i < num_loops; i++) {
3652 cur_size_in_bytes = size_in_bytes;
3653 if (cur_size_in_bytes > 0x1fffff)
3654 cur_size_in_bytes = 0x1fffff;
3655 size_in_bytes -= cur_size_in_bytes;
3656 control = 0;
3657 if (size_in_bytes == 0)
3658 control |= PACKET3_DMA_DATA_CP_SYNC;
3659 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3660 radeon_ring_write(ring, control);
3661 radeon_ring_write(ring, lower_32_bits(src_offset));
3662 radeon_ring_write(ring, upper_32_bits(src_offset));
3663 radeon_ring_write(ring, lower_32_bits(dst_offset));
3664 radeon_ring_write(ring, upper_32_bits(dst_offset));
3665 radeon_ring_write(ring, cur_size_in_bytes);
3666 src_offset += cur_size_in_bytes;
3667 dst_offset += cur_size_in_bytes;
3668 }
3669
3670 r = radeon_fence_emit(rdev, fence, ring->idx);
3671 if (r) {
3672 radeon_ring_unlock_undo(rdev, ring);
3673 return r;
3674 }
3675
3676 radeon_ring_unlock_commit(rdev, ring);
3677 radeon_semaphore_free(rdev, &sem, *fence);
3678
3679 return r;
3680}
3681
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3682/*
3683 * IB stuff
3684 */
3685/**
3686 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3687 *
3688 * @rdev: radeon_device pointer
3689 * @ib: radeon indirect buffer object
3690 *
3691 * Emits an DE (drawing engine) or CE (constant engine) IB
3692 * on the gfx ring. IBs are usually generated by userspace
3693 * acceleration drivers and submitted to the kernel for
3694 * sheduling on the ring. This function schedules the IB
3695 * on the gfx ring for execution by the GPU.
3696 */
3697void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3698{
3699 struct radeon_ring *ring = &rdev->ring[ib->ring];
3700 u32 header, control = INDIRECT_BUFFER_VALID;
3701
3702 if (ib->is_const_ib) {
3703 /* set switch buffer packet before const IB */
3704 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3705 radeon_ring_write(ring, 0);
3706
3707 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3708 } else {
3709 u32 next_rptr;
3710 if (ring->rptr_save_reg) {
3711 next_rptr = ring->wptr + 3 + 4;
3712 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3713 radeon_ring_write(ring, ((ring->rptr_save_reg -
3714 PACKET3_SET_UCONFIG_REG_START) >> 2));
3715 radeon_ring_write(ring, next_rptr);
3716 } else if (rdev->wb.enabled) {
3717 next_rptr = ring->wptr + 5 + 4;
3718 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3719 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3720 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3721 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3722 radeon_ring_write(ring, next_rptr);
3723 }
3724
3725 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3726 }
3727
3728 control |= ib->length_dw |
3729 (ib->vm ? (ib->vm->id << 24) : 0);
3730
3731 radeon_ring_write(ring, header);
3732 radeon_ring_write(ring,
3733#ifdef __BIG_ENDIAN
3734 (2 << 0) |
3735#endif
3736 (ib->gpu_addr & 0xFFFFFFFC));
3737 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3738 radeon_ring_write(ring, control);
3739}
3740
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3741/**
3742 * cik_ib_test - basic gfx ring IB test
3743 *
3744 * @rdev: radeon_device pointer
3745 * @ring: radeon_ring structure holding ring information
3746 *
3747 * Allocate an IB and execute it on the gfx ring (CIK).
3748 * Provides a basic gfx ring test to verify that IBs are working.
3749 * Returns 0 on success, error on failure.
3750 */
3751int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3752{
3753 struct radeon_ib ib;
3754 uint32_t scratch;
3755 uint32_t tmp = 0;
3756 unsigned i;
3757 int r;
3758
3759 r = radeon_scratch_get(rdev, &scratch);
3760 if (r) {
3761 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3762 return r;
3763 }
3764 WREG32(scratch, 0xCAFEDEAD);
3765 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3766 if (r) {
3767 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3768 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3769 return r;
3770 }
3771 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3772 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3773 ib.ptr[2] = 0xDEADBEEF;
3774 ib.length_dw = 3;
3775 r = radeon_ib_schedule(rdev, &ib, NULL);
3776 if (r) {
3777 radeon_scratch_free(rdev, scratch);
3778 radeon_ib_free(rdev, &ib);
3779 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3780 return r;
3781 }
3782 r = radeon_fence_wait(ib.fence, false);
3783 if (r) {
3784 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3785 radeon_scratch_free(rdev, scratch);
3786 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3787 return r;
3788 }
3789 for (i = 0; i < rdev->usec_timeout; i++) {
3790 tmp = RREG32(scratch);
3791 if (tmp == 0xDEADBEEF)
3792 break;
3793 DRM_UDELAY(1);
3794 }
3795 if (i < rdev->usec_timeout) {
3796 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3797 } else {
3798 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3799 scratch, tmp);
3800 r = -EINVAL;
3801 }
3802 radeon_scratch_free(rdev, scratch);
3803 radeon_ib_free(rdev, &ib);
3804 return r;
3805}
3806
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3807/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3808 * CP.
3809 * On CIK, gfx and compute now have independant command processors.
3810 *
3811 * GFX
3812 * Gfx consists of a single ring and can process both gfx jobs and
3813 * compute jobs. The gfx CP consists of three microengines (ME):
3814 * PFP - Pre-Fetch Parser
3815 * ME - Micro Engine
3816 * CE - Constant Engine
3817 * The PFP and ME make up what is considered the Drawing Engine (DE).
3818 * The CE is an asynchronous engine used for updating buffer desciptors
3819 * used by the DE so that they can be loaded into cache in parallel
3820 * while the DE is processing state update packets.
3821 *
3822 * Compute
3823 * The compute CP consists of two microengines (ME):
3824 * MEC1 - Compute MicroEngine 1
3825 * MEC2 - Compute MicroEngine 2
3826 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3827 * The queues are exposed to userspace and are programmed directly
3828 * by the compute runtime.
3829 */
3830/**
3831 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3832 *
3833 * @rdev: radeon_device pointer
3834 * @enable: enable or disable the MEs
3835 *
3836 * Halts or unhalts the gfx MEs.
3837 */
3838static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3839{
3840 if (enable)
3841 WREG32(CP_ME_CNTL, 0);
3842 else {
3843 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3844 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3845 }
3846 udelay(50);
3847}
3848
3849/**
3850 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3851 *
3852 * @rdev: radeon_device pointer
3853 *
3854 * Loads the gfx PFP, ME, and CE ucode.
3855 * Returns 0 for success, -EINVAL if the ucode is not available.
3856 */
3857static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3858{
3859 const __be32 *fw_data;
3860 int i;
3861
3862 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3863 return -EINVAL;
3864
3865 cik_cp_gfx_enable(rdev, false);
3866
3867 /* PFP */
3868 fw_data = (const __be32 *)rdev->pfp_fw->data;
3869 WREG32(CP_PFP_UCODE_ADDR, 0);
3870 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3871 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3872 WREG32(CP_PFP_UCODE_ADDR, 0);
3873
3874 /* CE */
3875 fw_data = (const __be32 *)rdev->ce_fw->data;
3876 WREG32(CP_CE_UCODE_ADDR, 0);
3877 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3878 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3879 WREG32(CP_CE_UCODE_ADDR, 0);
3880
3881 /* ME */
3882 fw_data = (const __be32 *)rdev->me_fw->data;
3883 WREG32(CP_ME_RAM_WADDR, 0);
3884 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3885 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3886 WREG32(CP_ME_RAM_WADDR, 0);
3887
3888 WREG32(CP_PFP_UCODE_ADDR, 0);
3889 WREG32(CP_CE_UCODE_ADDR, 0);
3890 WREG32(CP_ME_RAM_WADDR, 0);
3891 WREG32(CP_ME_RAM_RADDR, 0);
3892 return 0;
3893}
3894
3895/**
3896 * cik_cp_gfx_start - start the gfx ring
3897 *
3898 * @rdev: radeon_device pointer
3899 *
3900 * Enables the ring and loads the clear state context and other
3901 * packets required to init the ring.
3902 * Returns 0 for success, error for failure.
3903 */
3904static int cik_cp_gfx_start(struct radeon_device *rdev)
3905{
3906 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3907 int r, i;
3908
3909 /* init the CP */
3910 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3911 WREG32(CP_ENDIAN_SWAP, 0);
3912 WREG32(CP_DEVICE_ID, 1);
3913
3914 cik_cp_gfx_enable(rdev, true);
3915
3916 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3917 if (r) {
3918 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3919 return r;
3920 }
3921
3922 /* init the CE partitions. CE only used for gfx on CIK */
3923 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3924 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3925 radeon_ring_write(ring, 0xc000);
3926 radeon_ring_write(ring, 0xc000);
3927
3928 /* setup clear context state */
3929 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3930 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3931
3932 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3933 radeon_ring_write(ring, 0x80000000);
3934 radeon_ring_write(ring, 0x80000000);
3935
3936 for (i = 0; i < cik_default_size; i++)
3937 radeon_ring_write(ring, cik_default_state[i]);
3938
3939 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3940 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3941
3942 /* set clear context state */
3943 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3944 radeon_ring_write(ring, 0);
3945
3946 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3947 radeon_ring_write(ring, 0x00000316);
3948 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3949 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3950
3951 radeon_ring_unlock_commit(rdev, ring);
3952
3953 return 0;
3954}
3955
3956/**
3957 * cik_cp_gfx_fini - stop the gfx ring
3958 *
3959 * @rdev: radeon_device pointer
3960 *
3961 * Stop the gfx ring and tear down the driver ring
3962 * info.
3963 */
3964static void cik_cp_gfx_fini(struct radeon_device *rdev)
3965{
3966 cik_cp_gfx_enable(rdev, false);
3967 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3968}
3969
3970/**
3971 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3972 *
3973 * @rdev: radeon_device pointer
3974 *
3975 * Program the location and size of the gfx ring buffer
3976 * and test it to make sure it's working.
3977 * Returns 0 for success, error for failure.
3978 */
3979static int cik_cp_gfx_resume(struct radeon_device *rdev)
3980{
3981 struct radeon_ring *ring;
3982 u32 tmp;
3983 u32 rb_bufsz;
3984 u64 rb_addr;
3985 int r;
3986
3987 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3988 if (rdev->family != CHIP_HAWAII)
3989 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3990
3991 /* Set the write pointer delay */
3992 WREG32(CP_RB_WPTR_DELAY, 0);
3993
3994 /* set the RB to use vmid 0 */
3995 WREG32(CP_RB_VMID, 0);
3996
3997 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3998
3999 /* ring 0 - compute and gfx */
4000 /* Set ring buffer size */
4001 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4002 rb_bufsz = order_base_2(ring->ring_size / 8);
4003 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4004#ifdef __BIG_ENDIAN
4005 tmp |= BUF_SWAP_32BIT;
4006#endif
4007 WREG32(CP_RB0_CNTL, tmp);
4008
4009 /* Initialize the ring buffer's read and write pointers */
4010 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4011 ring->wptr = 0;
4012 WREG32(CP_RB0_WPTR, ring->wptr);
4013
4014 /* set the wb address wether it's enabled or not */
4015 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4016 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4017
4018 /* scratch register shadowing is no longer supported */
4019 WREG32(SCRATCH_UMSK, 0);
4020
4021 if (!rdev->wb.enabled)
4022 tmp |= RB_NO_UPDATE;
4023
4024 mdelay(1);
4025 WREG32(CP_RB0_CNTL, tmp);
4026
4027 rb_addr = ring->gpu_addr >> 8;
4028 WREG32(CP_RB0_BASE, rb_addr);
4029 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4030
4031 ring->rptr = RREG32(CP_RB0_RPTR);
4032
4033 /* start the ring */
4034 cik_cp_gfx_start(rdev);
4035 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4036 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4037 if (r) {
4038 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4039 return r;
4040 }
4041 return 0;
4042}
4043
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4044u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4045 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4046{
4047 u32 rptr;
4048
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4049 if (rdev->wb.enabled)
4050 rptr = rdev->wb.wb[ring->rptr_offs/4];
4051 else
4052 rptr = RREG32(CP_RB0_RPTR);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4053
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4054 return rptr;
4055}
4056
4057u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4058 struct radeon_ring *ring)
4059{
4060 u32 wptr;
4061
4062 wptr = RREG32(CP_RB0_WPTR);
4063
4064 return wptr;
4065}
4066
4067void cik_gfx_set_wptr(struct radeon_device *rdev,
4068 struct radeon_ring *ring)
4069{
4070 WREG32(CP_RB0_WPTR, ring->wptr);
4071 (void)RREG32(CP_RB0_WPTR);
4072}
4073
4074u32 cik_compute_get_rptr(struct radeon_device *rdev,
4075 struct radeon_ring *ring)
4076{
4077 u32 rptr;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4078
4079 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4080 rptr = rdev->wb.wb[ring->rptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4081 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4082 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4083 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4084 rptr = RREG32(CP_HQD_PQ_RPTR);
4085 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4086 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4087 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4088
4089 return rptr;
4090}
4091
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4092u32 cik_compute_get_wptr(struct radeon_device *rdev,
4093 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4094{
4095 u32 wptr;
4096
4097 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4098 /* XXX check if swapping is necessary on BE */
4099 wptr = rdev->wb.wb[ring->wptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4100 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4101 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4102 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4103 wptr = RREG32(CP_HQD_PQ_WPTR);
4104 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4105 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4106 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4107
4108 return wptr;
4109}
4110
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4111void cik_compute_set_wptr(struct radeon_device *rdev,
4112 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4113{
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4114 /* XXX check if swapping is necessary on BE */
4115 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4116 WDOORBELL32(ring->doorbell_index, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4117}
4118
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4119/**
4120 * cik_cp_compute_enable - enable/disable the compute CP MEs
4121 *
4122 * @rdev: radeon_device pointer
4123 * @enable: enable or disable the MEs
4124 *
4125 * Halts or unhalts the compute MEs.
4126 */
4127static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4128{
4129 if (enable)
4130 WREG32(CP_MEC_CNTL, 0);
4131 else
4132 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4133 udelay(50);
4134}
4135
4136/**
4137 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4138 *
4139 * @rdev: radeon_device pointer
4140 *
4141 * Loads the compute MEC1&2 ucode.
4142 * Returns 0 for success, -EINVAL if the ucode is not available.
4143 */
4144static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4145{
4146 const __be32 *fw_data;
4147 int i;
4148
4149 if (!rdev->mec_fw)
4150 return -EINVAL;
4151
4152 cik_cp_compute_enable(rdev, false);
4153
4154 /* MEC1 */
4155 fw_data = (const __be32 *)rdev->mec_fw->data;
4156 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4157 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4158 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4159 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4160
4161 if (rdev->family == CHIP_KAVERI) {
4162 /* MEC2 */
4163 fw_data = (const __be32 *)rdev->mec_fw->data;
4164 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4165 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4166 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4167 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4168 }
4169
4170 return 0;
4171}
4172
4173/**
4174 * cik_cp_compute_start - start the compute queues
4175 *
4176 * @rdev: radeon_device pointer
4177 *
4178 * Enable the compute queues.
4179 * Returns 0 for success, error for failure.
4180 */
4181static int cik_cp_compute_start(struct radeon_device *rdev)
4182{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4183 cik_cp_compute_enable(rdev, true);
4184
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4185 return 0;
4186}
4187
4188/**
4189 * cik_cp_compute_fini - stop the compute queues
4190 *
4191 * @rdev: radeon_device pointer
4192 *
4193 * Stop the compute queues and tear down the driver queue
4194 * info.
4195 */
4196static void cik_cp_compute_fini(struct radeon_device *rdev)
4197{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4198 int i, idx, r;
4199
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4200 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4201
4202 for (i = 0; i < 2; i++) {
4203 if (i == 0)
4204 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4205 else
4206 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4207
4208 if (rdev->ring[idx].mqd_obj) {
4209 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4210 if (unlikely(r != 0))
4211 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4212
4213 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4214 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4215
4216 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4217 rdev->ring[idx].mqd_obj = NULL;
4218 }
4219 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4220}
4221
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4222static void cik_mec_fini(struct radeon_device *rdev)
4223{
4224 int r;
4225
4226 if (rdev->mec.hpd_eop_obj) {
4227 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4228 if (unlikely(r != 0))
4229 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4230 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4231 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4232
4233 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4234 rdev->mec.hpd_eop_obj = NULL;
4235 }
4236}
4237
4238#define MEC_HPD_SIZE 2048
4239
4240static int cik_mec_init(struct radeon_device *rdev)
4241{
4242 int r;
4243 u32 *hpd;
4244
4245 /*
4246 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4247 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4248 */
4249 if (rdev->family == CHIP_KAVERI)
4250 rdev->mec.num_mec = 2;
4251 else
4252 rdev->mec.num_mec = 1;
4253 rdev->mec.num_pipe = 4;
4254 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4255
4256 if (rdev->mec.hpd_eop_obj == NULL) {
4257 r = radeon_bo_create(rdev,
4258 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4259 PAGE_SIZE, true,
4260 RADEON_GEM_DOMAIN_GTT, NULL,
4261 &rdev->mec.hpd_eop_obj);
4262 if (r) {
4263 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4264 return r;
4265 }
4266 }
4267
4268 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4269 if (unlikely(r != 0)) {
4270 cik_mec_fini(rdev);
4271 return r;
4272 }
4273 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4274 &rdev->mec.hpd_eop_gpu_addr);
4275 if (r) {
4276 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4277 cik_mec_fini(rdev);
4278 return r;
4279 }
4280 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4281 if (r) {
4282 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4283 cik_mec_fini(rdev);
4284 return r;
4285 }
4286
4287 /* clear memory. Not sure if this is required or not */
4288 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4289
4290 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4291 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4292
4293 return 0;
4294}
4295
4296struct hqd_registers
4297{
4298 u32 cp_mqd_base_addr;
4299 u32 cp_mqd_base_addr_hi;
4300 u32 cp_hqd_active;
4301 u32 cp_hqd_vmid;
4302 u32 cp_hqd_persistent_state;
4303 u32 cp_hqd_pipe_priority;
4304 u32 cp_hqd_queue_priority;
4305 u32 cp_hqd_quantum;
4306 u32 cp_hqd_pq_base;
4307 u32 cp_hqd_pq_base_hi;
4308 u32 cp_hqd_pq_rptr;
4309 u32 cp_hqd_pq_rptr_report_addr;
4310 u32 cp_hqd_pq_rptr_report_addr_hi;
4311 u32 cp_hqd_pq_wptr_poll_addr;
4312 u32 cp_hqd_pq_wptr_poll_addr_hi;
4313 u32 cp_hqd_pq_doorbell_control;
4314 u32 cp_hqd_pq_wptr;
4315 u32 cp_hqd_pq_control;
4316 u32 cp_hqd_ib_base_addr;
4317 u32 cp_hqd_ib_base_addr_hi;
4318 u32 cp_hqd_ib_rptr;
4319 u32 cp_hqd_ib_control;
4320 u32 cp_hqd_iq_timer;
4321 u32 cp_hqd_iq_rptr;
4322 u32 cp_hqd_dequeue_request;
4323 u32 cp_hqd_dma_offload;
4324 u32 cp_hqd_sema_cmd;
4325 u32 cp_hqd_msg_type;
4326 u32 cp_hqd_atomic0_preop_lo;
4327 u32 cp_hqd_atomic0_preop_hi;
4328 u32 cp_hqd_atomic1_preop_lo;
4329 u32 cp_hqd_atomic1_preop_hi;
4330 u32 cp_hqd_hq_scheduler0;
4331 u32 cp_hqd_hq_scheduler1;
4332 u32 cp_mqd_control;
4333};
4334
4335struct bonaire_mqd
4336{
4337 u32 header;
4338 u32 dispatch_initiator;
4339 u32 dimensions[3];
4340 u32 start_idx[3];
4341 u32 num_threads[3];
4342 u32 pipeline_stat_enable;
4343 u32 perf_counter_enable;
4344 u32 pgm[2];
4345 u32 tba[2];
4346 u32 tma[2];
4347 u32 pgm_rsrc[2];
4348 u32 vmid;
4349 u32 resource_limits;
4350 u32 static_thread_mgmt01[2];
4351 u32 tmp_ring_size;
4352 u32 static_thread_mgmt23[2];
4353 u32 restart[3];
4354 u32 thread_trace_enable;
4355 u32 reserved1;
4356 u32 user_data[16];
4357 u32 vgtcs_invoke_count[2];
4358 struct hqd_registers queue_state;
4359 u32 dequeue_cntr;
4360 u32 interrupt_queue[64];
4361};
4362
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4363/**
4364 * cik_cp_compute_resume - setup the compute queue registers
4365 *
4366 * @rdev: radeon_device pointer
4367 *
4368 * Program the compute queues and test them to make sure they
4369 * are working.
4370 * Returns 0 for success, error for failure.
4371 */
4372static int cik_cp_compute_resume(struct radeon_device *rdev)
4373{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4374 int r, i, idx;
4375 u32 tmp;
4376 bool use_doorbell = true;
4377 u64 hqd_gpu_addr;
4378 u64 mqd_gpu_addr;
4379 u64 eop_gpu_addr;
4380 u64 wb_gpu_addr;
4381 u32 *buf;
4382 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4383
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4384 r = cik_cp_compute_start(rdev);
4385 if (r)
4386 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4387
4388 /* fix up chicken bits */
4389 tmp = RREG32(CP_CPF_DEBUG);
4390 tmp |= (1 << 23);
4391 WREG32(CP_CPF_DEBUG, tmp);
4392
4393 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4394 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4395 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4396 int me = (i < 4) ? 1 : 2;
4397 int pipe = (i < 4) ? i : (i - 4);
4398
4399 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4400
4401 cik_srbm_select(rdev, me, pipe, 0, 0);
4402
4403 /* write the EOP addr */
4404 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4405 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4406
4407 /* set the VMID assigned */
4408 WREG32(CP_HPD_EOP_VMID, 0);
4409
4410 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4411 tmp = RREG32(CP_HPD_EOP_CONTROL);
4412 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4413 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4414 WREG32(CP_HPD_EOP_CONTROL, tmp);
4415 }
4416 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4417 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4418
4419 /* init the queues. Just two for now. */
4420 for (i = 0; i < 2; i++) {
4421 if (i == 0)
4422 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4423 else
4424 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4425
4426 if (rdev->ring[idx].mqd_obj == NULL) {
4427 r = radeon_bo_create(rdev,
4428 sizeof(struct bonaire_mqd),
4429 PAGE_SIZE, true,
4430 RADEON_GEM_DOMAIN_GTT, NULL,
4431 &rdev->ring[idx].mqd_obj);
4432 if (r) {
4433 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4434 return r;
4435 }
4436 }
4437
4438 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4439 if (unlikely(r != 0)) {
4440 cik_cp_compute_fini(rdev);
4441 return r;
4442 }
4443 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4444 &mqd_gpu_addr);
4445 if (r) {
4446 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4447 cik_cp_compute_fini(rdev);
4448 return r;
4449 }
4450 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4451 if (r) {
4452 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4453 cik_cp_compute_fini(rdev);
4454 return r;
4455 }
4456
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4457 /* init the mqd struct */
4458 memset(buf, 0, sizeof(struct bonaire_mqd));
4459
4460 mqd = (struct bonaire_mqd *)buf;
4461 mqd->header = 0xC0310800;
4462 mqd->static_thread_mgmt01[0] = 0xffffffff;
4463 mqd->static_thread_mgmt01[1] = 0xffffffff;
4464 mqd->static_thread_mgmt23[0] = 0xffffffff;
4465 mqd->static_thread_mgmt23[1] = 0xffffffff;
4466
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4467 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4468 cik_srbm_select(rdev, rdev->ring[idx].me,
4469 rdev->ring[idx].pipe,
4470 rdev->ring[idx].queue, 0);
4471
4472 /* disable wptr polling */
4473 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4474 tmp &= ~WPTR_POLL_EN;
4475 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4476
4477 /* enable doorbell? */
4478 mqd->queue_state.cp_hqd_pq_doorbell_control =
4479 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4480 if (use_doorbell)
4481 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4482 else
4483 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4484 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4485 mqd->queue_state.cp_hqd_pq_doorbell_control);
4486
4487 /* disable the queue if it's active */
4488 mqd->queue_state.cp_hqd_dequeue_request = 0;
4489 mqd->queue_state.cp_hqd_pq_rptr = 0;
4490 mqd->queue_state.cp_hqd_pq_wptr= 0;
4491 if (RREG32(CP_HQD_ACTIVE) & 1) {
4492 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4493 for (i = 0; i < rdev->usec_timeout; i++) {
4494 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4495 break;
4496 udelay(1);
4497 }
4498 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4499 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4500 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4501 }
4502
4503 /* set the pointer to the MQD */
4504 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4505 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4506 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4507 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4508 /* set MQD vmid to 0 */
4509 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4510 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4511 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4512
4513 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4514 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4515 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4516 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4517 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4518 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4519
4520 /* set up the HQD, this is similar to CP_RB0_CNTL */
4521 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4522 mqd->queue_state.cp_hqd_pq_control &=
4523 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4524
4525 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4526 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4527 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4528 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4529#ifdef __BIG_ENDIAN
4530 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4531#endif
4532 mqd->queue_state.cp_hqd_pq_control &=
4533 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4534 mqd->queue_state.cp_hqd_pq_control |=
4535 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4536 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4537
4538 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4539 if (i == 0)
4540 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4541 else
4542 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4543 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4544 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4545 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4546 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4547 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4548
4549 /* set the wb address wether it's enabled or not */
4550 if (i == 0)
4551 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4552 else
4553 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4554 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4555 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4556 upper_32_bits(wb_gpu_addr) & 0xffff;
4557 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4558 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4559 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4560 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4561
4562 /* enable the doorbell if requested */
4563 if (use_doorbell) {
4564 mqd->queue_state.cp_hqd_pq_doorbell_control =
4565 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4566 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4567 mqd->queue_state.cp_hqd_pq_doorbell_control |=
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4568 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4569 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4570 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4571 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4572
4573 } else {
4574 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4575 }
4576 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4577 mqd->queue_state.cp_hqd_pq_doorbell_control);
4578
4579 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4580 rdev->ring[idx].wptr = 0;
4581 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4582 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4583 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4584 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4585
4586 /* set the vmid for the queue */
4587 mqd->queue_state.cp_hqd_vmid = 0;
4588 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4589
4590 /* activate the queue */
4591 mqd->queue_state.cp_hqd_active = 1;
4592 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4593
4594 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4595 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4596
4597 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4598 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4599
4600 rdev->ring[idx].ready = true;
4601 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4602 if (r)
4603 rdev->ring[idx].ready = false;
4604 }
4605
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4606 return 0;
4607}
4608
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4609static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4610{
4611 cik_cp_gfx_enable(rdev, enable);
4612 cik_cp_compute_enable(rdev, enable);
4613}
4614
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4615static int cik_cp_load_microcode(struct radeon_device *rdev)
4616{
4617 int r;
4618
4619 r = cik_cp_gfx_load_microcode(rdev);
4620 if (r)
4621 return r;
4622 r = cik_cp_compute_load_microcode(rdev);
4623 if (r)
4624 return r;
4625
4626 return 0;
4627}
4628
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4629static void cik_cp_fini(struct radeon_device *rdev)
4630{
4631 cik_cp_gfx_fini(rdev);
4632 cik_cp_compute_fini(rdev);
4633}
4634
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4635static int cik_cp_resume(struct radeon_device *rdev)
4636{
4637 int r;
4638
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4639 cik_enable_gui_idle_interrupt(rdev, false);
4640
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4641 r = cik_cp_load_microcode(rdev);
4642 if (r)
4643 return r;
4644
4645 r = cik_cp_gfx_resume(rdev);
4646 if (r)
4647 return r;
4648 r = cik_cp_compute_resume(rdev);
4649 if (r)
4650 return r;
4651
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4652 cik_enable_gui_idle_interrupt(rdev, true);
4653
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4654 return 0;
4655}
4656
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4657static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4658{
4659 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4660 RREG32(GRBM_STATUS));
4661 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4662 RREG32(GRBM_STATUS2));
4663 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4664 RREG32(GRBM_STATUS_SE0));
4665 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4666 RREG32(GRBM_STATUS_SE1));
4667 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4668 RREG32(GRBM_STATUS_SE2));
4669 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4670 RREG32(GRBM_STATUS_SE3));
4671 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4672 RREG32(SRBM_STATUS));
4673 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4674 RREG32(SRBM_STATUS2));
4675 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4676 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4677 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4678 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4679 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4680 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4681 RREG32(CP_STALLED_STAT1));
4682 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4683 RREG32(CP_STALLED_STAT2));
4684 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4685 RREG32(CP_STALLED_STAT3));
4686 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4687 RREG32(CP_CPF_BUSY_STAT));
4688 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4689 RREG32(CP_CPF_STALLED_STAT1));
4690 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4691 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4692 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4693 RREG32(CP_CPC_STALLED_STAT1));
4694 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4695}
4696
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4697/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4698 * cik_gpu_check_soft_reset - check which blocks are busy
4699 *
4700 * @rdev: radeon_device pointer
4701 *
4702 * Check which blocks are busy and return the relevant reset
4703 * mask to be used by cik_gpu_soft_reset().
4704 * Returns a mask of the blocks to be reset.
4705 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4706u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4707{
4708 u32 reset_mask = 0;
4709 u32 tmp;
4710
4711 /* GRBM_STATUS */
4712 tmp = RREG32(GRBM_STATUS);
4713 if (tmp & (PA_BUSY | SC_BUSY |
4714 BCI_BUSY | SX_BUSY |
4715 TA_BUSY | VGT_BUSY |
4716 DB_BUSY | CB_BUSY |
4717 GDS_BUSY | SPI_BUSY |
4718 IA_BUSY | IA_BUSY_NO_DMA))
4719 reset_mask |= RADEON_RESET_GFX;
4720
4721 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4722 reset_mask |= RADEON_RESET_CP;
4723
4724 /* GRBM_STATUS2 */
4725 tmp = RREG32(GRBM_STATUS2);
4726 if (tmp & RLC_BUSY)
4727 reset_mask |= RADEON_RESET_RLC;
4728
4729 /* SDMA0_STATUS_REG */
4730 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4731 if (!(tmp & SDMA_IDLE))
4732 reset_mask |= RADEON_RESET_DMA;
4733
4734 /* SDMA1_STATUS_REG */
4735 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4736 if (!(tmp & SDMA_IDLE))
4737 reset_mask |= RADEON_RESET_DMA1;
4738
4739 /* SRBM_STATUS2 */
4740 tmp = RREG32(SRBM_STATUS2);
4741 if (tmp & SDMA_BUSY)
4742 reset_mask |= RADEON_RESET_DMA;
4743
4744 if (tmp & SDMA1_BUSY)
4745 reset_mask |= RADEON_RESET_DMA1;
4746
4747 /* SRBM_STATUS */
4748 tmp = RREG32(SRBM_STATUS);
4749
4750 if (tmp & IH_BUSY)
4751 reset_mask |= RADEON_RESET_IH;
4752
4753 if (tmp & SEM_BUSY)
4754 reset_mask |= RADEON_RESET_SEM;
4755
4756 if (tmp & GRBM_RQ_PENDING)
4757 reset_mask |= RADEON_RESET_GRBM;
4758
4759 if (tmp & VMC_BUSY)
4760 reset_mask |= RADEON_RESET_VMC;
4761
4762 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4763 MCC_BUSY | MCD_BUSY))
4764 reset_mask |= RADEON_RESET_MC;
4765
4766 if (evergreen_is_display_hung(rdev))
4767 reset_mask |= RADEON_RESET_DISPLAY;
4768
4769 /* Skip MC reset as it's mostly likely not hung, just busy */
4770 if (reset_mask & RADEON_RESET_MC) {
4771 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4772 reset_mask &= ~RADEON_RESET_MC;
4773 }
4774
4775 return reset_mask;
4776}
4777
4778/**
4779 * cik_gpu_soft_reset - soft reset GPU
4780 *
4781 * @rdev: radeon_device pointer
4782 * @reset_mask: mask of which blocks to reset
4783 *
4784 * Soft reset the blocks specified in @reset_mask.
4785 */
4786static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4787{
4788 struct evergreen_mc_save save;
4789 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4790 u32 tmp;
4791
4792 if (reset_mask == 0)
4793 return;
4794
4795 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4796
4797 cik_print_gpu_status_regs(rdev);
4798 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4799 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4800 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4801 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4802
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4803 /* disable CG/PG */
4804 cik_fini_pg(rdev);
4805 cik_fini_cg(rdev);
4806
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4807 /* stop the rlc */
4808 cik_rlc_stop(rdev);
4809
4810 /* Disable GFX parsing/prefetching */
4811 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4812
4813 /* Disable MEC parsing/prefetching */
4814 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4815
4816 if (reset_mask & RADEON_RESET_DMA) {
4817 /* sdma0 */
4818 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4819 tmp |= SDMA_HALT;
4820 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4821 }
4822 if (reset_mask & RADEON_RESET_DMA1) {
4823 /* sdma1 */
4824 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4825 tmp |= SDMA_HALT;
4826 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4827 }
4828
4829 evergreen_mc_stop(rdev, &save);
4830 if (evergreen_mc_wait_for_idle(rdev)) {
4831 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4832 }
4833
4834 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4835 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4836
4837 if (reset_mask & RADEON_RESET_CP) {
4838 grbm_soft_reset |= SOFT_RESET_CP;
4839
4840 srbm_soft_reset |= SOFT_RESET_GRBM;
4841 }
4842
4843 if (reset_mask & RADEON_RESET_DMA)
4844 srbm_soft_reset |= SOFT_RESET_SDMA;
4845
4846 if (reset_mask & RADEON_RESET_DMA1)
4847 srbm_soft_reset |= SOFT_RESET_SDMA1;
4848
4849 if (reset_mask & RADEON_RESET_DISPLAY)
4850 srbm_soft_reset |= SOFT_RESET_DC;
4851
4852 if (reset_mask & RADEON_RESET_RLC)
4853 grbm_soft_reset |= SOFT_RESET_RLC;
4854
4855 if (reset_mask & RADEON_RESET_SEM)
4856 srbm_soft_reset |= SOFT_RESET_SEM;
4857
4858 if (reset_mask & RADEON_RESET_IH)
4859 srbm_soft_reset |= SOFT_RESET_IH;
4860
4861 if (reset_mask & RADEON_RESET_GRBM)
4862 srbm_soft_reset |= SOFT_RESET_GRBM;
4863
4864 if (reset_mask & RADEON_RESET_VMC)
4865 srbm_soft_reset |= SOFT_RESET_VMC;
4866
4867 if (!(rdev->flags & RADEON_IS_IGP)) {
4868 if (reset_mask & RADEON_RESET_MC)
4869 srbm_soft_reset |= SOFT_RESET_MC;
4870 }
4871
4872 if (grbm_soft_reset) {
4873 tmp = RREG32(GRBM_SOFT_RESET);
4874 tmp |= grbm_soft_reset;
4875 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4876 WREG32(GRBM_SOFT_RESET, tmp);
4877 tmp = RREG32(GRBM_SOFT_RESET);
4878
4879 udelay(50);
4880
4881 tmp &= ~grbm_soft_reset;
4882 WREG32(GRBM_SOFT_RESET, tmp);
4883 tmp = RREG32(GRBM_SOFT_RESET);
4884 }
4885
4886 if (srbm_soft_reset) {
4887 tmp = RREG32(SRBM_SOFT_RESET);
4888 tmp |= srbm_soft_reset;
4889 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4890 WREG32(SRBM_SOFT_RESET, tmp);
4891 tmp = RREG32(SRBM_SOFT_RESET);
4892
4893 udelay(50);
4894
4895 tmp &= ~srbm_soft_reset;
4896 WREG32(SRBM_SOFT_RESET, tmp);
4897 tmp = RREG32(SRBM_SOFT_RESET);
4898 }
4899
4900 /* Wait a little for things to settle down */
4901 udelay(50);
4902
4903 evergreen_mc_resume(rdev, &save);
4904 udelay(50);
4905
4906 cik_print_gpu_status_regs(rdev);
4907}
4908
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]4909struct kv_reset_save_regs {
4910 u32 gmcon_reng_execute;
4911 u32 gmcon_misc;
4912 u32 gmcon_misc3;
4913};
4914
4915static void kv_save_regs_for_reset(struct radeon_device *rdev,
4916 struct kv_reset_save_regs *save)
4917{
4918 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4919 save->gmcon_misc = RREG32(GMCON_MISC);
4920 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4921
4922 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4923 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4924 STCTRL_STUTTER_EN));
4925}
4926
4927static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4928 struct kv_reset_save_regs *save)
4929{
4930 int i;
4931
4932 WREG32(GMCON_PGFSM_WRITE, 0);
4933 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4934
4935 for (i = 0; i < 5; i++)
4936 WREG32(GMCON_PGFSM_WRITE, 0);
4937
4938 WREG32(GMCON_PGFSM_WRITE, 0);
4939 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4940
4941 for (i = 0; i < 5; i++)
4942 WREG32(GMCON_PGFSM_WRITE, 0);
4943
4944 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4945 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4946
4947 for (i = 0; i < 5; i++)
4948 WREG32(GMCON_PGFSM_WRITE, 0);
4949
4950 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4951 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4952
4953 for (i = 0; i < 5; i++)
4954 WREG32(GMCON_PGFSM_WRITE, 0);
4955
4956 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4957 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4958
4959 for (i = 0; i < 5; i++)
4960 WREG32(GMCON_PGFSM_WRITE, 0);
4961
4962 WREG32(GMCON_PGFSM_WRITE, 0);
4963 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
4964
4965 for (i = 0; i < 5; i++)
4966 WREG32(GMCON_PGFSM_WRITE, 0);
4967
4968 WREG32(GMCON_PGFSM_WRITE, 0x420000);
4969 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
4970
4971 for (i = 0; i < 5; i++)
4972 WREG32(GMCON_PGFSM_WRITE, 0);
4973
4974 WREG32(GMCON_PGFSM_WRITE, 0x120202);
4975 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
4976
4977 for (i = 0; i < 5; i++)
4978 WREG32(GMCON_PGFSM_WRITE, 0);
4979
4980 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
4981 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
4982
4983 for (i = 0; i < 5; i++)
4984 WREG32(GMCON_PGFSM_WRITE, 0);
4985
4986 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
4987 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
4988
4989 for (i = 0; i < 5; i++)
4990 WREG32(GMCON_PGFSM_WRITE, 0);
4991
4992 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
4993 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
4994
4995 WREG32(GMCON_MISC3, save->gmcon_misc3);
4996 WREG32(GMCON_MISC, save->gmcon_misc);
4997 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
4998}
4999
5000static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5001{
5002 struct evergreen_mc_save save;
5003 struct kv_reset_save_regs kv_save = { 0 };
5004 u32 tmp, i;
5005
5006 dev_info(rdev->dev, "GPU pci config reset\n");
5007
5008 /* disable dpm? */
5009
5010 /* disable cg/pg */
5011 cik_fini_pg(rdev);
5012 cik_fini_cg(rdev);
5013
5014 /* Disable GFX parsing/prefetching */
5015 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5016
5017 /* Disable MEC parsing/prefetching */
5018 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5019
5020 /* sdma0 */
5021 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5022 tmp |= SDMA_HALT;
5023 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5024 /* sdma1 */
5025 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5026 tmp |= SDMA_HALT;
5027 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5028 /* XXX other engines? */
5029
5030 /* halt the rlc, disable cp internal ints */
5031 cik_rlc_stop(rdev);
5032
5033 udelay(50);
5034
5035 /* disable mem access */
5036 evergreen_mc_stop(rdev, &save);
5037 if (evergreen_mc_wait_for_idle(rdev)) {
5038 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5039 }
5040
5041 if (rdev->flags & RADEON_IS_IGP)
5042 kv_save_regs_for_reset(rdev, &kv_save);
5043
5044 /* disable BM */
5045 pci_clear_master(rdev->pdev);
5046 /* reset */
5047 radeon_pci_config_reset(rdev);
5048
5049 udelay(100);
5050
5051 /* wait for asic to come out of reset */
5052 for (i = 0; i < rdev->usec_timeout; i++) {
5053 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5054 break;
5055 udelay(1);
5056 }
5057
5058 /* does asic init need to be run first??? */
5059 if (rdev->flags & RADEON_IS_IGP)
5060 kv_restore_regs_for_reset(rdev, &kv_save);
5061}
5062
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5063/**
5064 * cik_asic_reset - soft reset GPU
5065 *
5066 * @rdev: radeon_device pointer
5067 *
5068 * Look up which blocks are hung and attempt
5069 * to reset them.
5070 * Returns 0 for success.
5071 */
5072int cik_asic_reset(struct radeon_device *rdev)
5073{
5074 u32 reset_mask;
5075
5076 reset_mask = cik_gpu_check_soft_reset(rdev);
5077
5078 if (reset_mask)
5079 r600_set_bios_scratch_engine_hung(rdev, true);
5080
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]5081 /* try soft reset */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5082 cik_gpu_soft_reset(rdev, reset_mask);
5083
5084 reset_mask = cik_gpu_check_soft_reset(rdev);
5085
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame]5086 /* try pci config reset */
5087 if (reset_mask && radeon_hard_reset)
5088 cik_gpu_pci_config_reset(rdev);
5089
5090 reset_mask = cik_gpu_check_soft_reset(rdev);
5091
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5092 if (!reset_mask)
5093 r600_set_bios_scratch_engine_hung(rdev, false);
5094
5095 return 0;
5096}
5097
5098/**
5099 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5100 *
5101 * @rdev: radeon_device pointer
5102 * @ring: radeon_ring structure holding ring information
5103 *
5104 * Check if the 3D engine is locked up (CIK).
5105 * Returns true if the engine is locked, false if not.
5106 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5107bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5108{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5109 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5110
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5111 if (!(reset_mask & (RADEON_RESET_GFX |
5112 RADEON_RESET_COMPUTE |
5113 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5114 radeon_ring_lockup_update(ring);
5115 return false;
5116 }
5117 /* force CP activities */
5118 radeon_ring_force_activity(rdev, ring);
5119 return radeon_ring_test_lockup(rdev, ring);
5120}
5121
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5122/* MC */
5123/**
5124 * cik_mc_program - program the GPU memory controller
5125 *
5126 * @rdev: radeon_device pointer
5127 *
5128 * Set the location of vram, gart, and AGP in the GPU's
5129 * physical address space (CIK).
5130 */
5131static void cik_mc_program(struct radeon_device *rdev)
5132{
5133 struct evergreen_mc_save save;
5134 u32 tmp;
5135 int i, j;
5136
5137 /* Initialize HDP */
5138 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5139 WREG32((0x2c14 + j), 0x00000000);
5140 WREG32((0x2c18 + j), 0x00000000);
5141 WREG32((0x2c1c + j), 0x00000000);
5142 WREG32((0x2c20 + j), 0x00000000);
5143 WREG32((0x2c24 + j), 0x00000000);
5144 }
5145 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5146
5147 evergreen_mc_stop(rdev, &save);
5148 if (radeon_mc_wait_for_idle(rdev)) {
5149 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5150 }
5151 /* Lockout access through VGA aperture*/
5152 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5153 /* Update configuration */
5154 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5155 rdev->mc.vram_start >> 12);
5156 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5157 rdev->mc.vram_end >> 12);
5158 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5159 rdev->vram_scratch.gpu_addr >> 12);
5160 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5161 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5162 WREG32(MC_VM_FB_LOCATION, tmp);
5163 /* XXX double check these! */
5164 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5165 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5166 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5167 WREG32(MC_VM_AGP_BASE, 0);
5168 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5169 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5170 if (radeon_mc_wait_for_idle(rdev)) {
5171 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5172 }
5173 evergreen_mc_resume(rdev, &save);
5174 /* we need to own VRAM, so turn off the VGA renderer here
5175 * to stop it overwriting our objects */
5176 rv515_vga_render_disable(rdev);
5177}
5178
5179/**
5180 * cik_mc_init - initialize the memory controller driver params
5181 *
5182 * @rdev: radeon_device pointer
5183 *
5184 * Look up the amount of vram, vram width, and decide how to place
5185 * vram and gart within the GPU's physical address space (CIK).
5186 * Returns 0 for success.
5187 */
5188static int cik_mc_init(struct radeon_device *rdev)
5189{
5190 u32 tmp;
5191 int chansize, numchan;
5192
5193 /* Get VRAM informations */
5194 rdev->mc.vram_is_ddr = true;
5195 tmp = RREG32(MC_ARB_RAMCFG);
5196 if (tmp & CHANSIZE_MASK) {
5197 chansize = 64;
5198 } else {
5199 chansize = 32;
5200 }
5201 tmp = RREG32(MC_SHARED_CHMAP);
5202 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5203 case 0:
5204 default:
5205 numchan = 1;
5206 break;
5207 case 1:
5208 numchan = 2;
5209 break;
5210 case 2:
5211 numchan = 4;
5212 break;
5213 case 3:
5214 numchan = 8;
5215 break;
5216 case 4:
5217 numchan = 3;
5218 break;
5219 case 5:
5220 numchan = 6;
5221 break;
5222 case 6:
5223 numchan = 10;
5224 break;
5225 case 7:
5226 numchan = 12;
5227 break;
5228 case 8:
5229 numchan = 16;
5230 break;
5231 }
5232 rdev->mc.vram_width = numchan * chansize;
5233 /* Could aper size report 0 ? */
5234 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5235 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5236 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5237 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5238 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5239 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5240 si_vram_gtt_location(rdev, &rdev->mc);
5241 radeon_update_bandwidth_info(rdev);
5242
5243 return 0;
5244}
5245
5246/*
5247 * GART
5248 * VMID 0 is the physical GPU addresses as used by the kernel.
5249 * VMIDs 1-15 are used for userspace clients and are handled
5250 * by the radeon vm/hsa code.
5251 */
5252/**
5253 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5254 *
5255 * @rdev: radeon_device pointer
5256 *
5257 * Flush the TLB for the VMID 0 page table (CIK).
5258 */
5259void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5260{
5261 /* flush hdp cache */
5262 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5263
5264 /* bits 0-15 are the VM contexts0-15 */
5265 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5266}
5267
5268/**
5269 * cik_pcie_gart_enable - gart enable
5270 *
5271 * @rdev: radeon_device pointer
5272 *
5273 * This sets up the TLBs, programs the page tables for VMID0,
5274 * sets up the hw for VMIDs 1-15 which are allocated on
5275 * demand, and sets up the global locations for the LDS, GDS,
5276 * and GPUVM for FSA64 clients (CIK).
5277 * Returns 0 for success, errors for failure.
5278 */
5279static int cik_pcie_gart_enable(struct radeon_device *rdev)
5280{
5281 int r, i;
5282
5283 if (rdev->gart.robj == NULL) {
5284 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5285 return -EINVAL;
5286 }
5287 r = radeon_gart_table_vram_pin(rdev);
5288 if (r)
5289 return r;
5290 radeon_gart_restore(rdev);
5291 /* Setup TLB control */
5292 WREG32(MC_VM_MX_L1_TLB_CNTL,
5293 (0xA << 7) |
5294 ENABLE_L1_TLB |
5295 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5296 ENABLE_ADVANCED_DRIVER_MODEL |
5297 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5298 /* Setup L2 cache */
5299 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5300 ENABLE_L2_FRAGMENT_PROCESSING |
5301 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5302 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5303 EFFECTIVE_L2_QUEUE_SIZE(7) |
5304 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5305 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5306 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5307 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5308 /* setup context0 */
5309 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5310 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5311 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5312 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5313 (u32)(rdev->dummy_page.addr >> 12));
5314 WREG32(VM_CONTEXT0_CNTL2, 0);
5315 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5316 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5317
5318 WREG32(0x15D4, 0);
5319 WREG32(0x15D8, 0);
5320 WREG32(0x15DC, 0);
5321
5322 /* empty context1-15 */
5323 /* FIXME start with 4G, once using 2 level pt switch to full
5324 * vm size space
5325 */
5326 /* set vm size, must be a multiple of 4 */
5327 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5328 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5329 for (i = 1; i < 16; i++) {
5330 if (i < 8)
5331 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5332 rdev->gart.table_addr >> 12);
5333 else
5334 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5335 rdev->gart.table_addr >> 12);
5336 }
5337
5338 /* enable context1-15 */
5339 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5340 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5341 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5342 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5343 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5344 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5345 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5346 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5347 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5348 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5349 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5350 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5351 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5352 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5353 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5354 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5355
5356 /* TC cache setup ??? */
5357 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5358 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5359 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5360
5361 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5362 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5363 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5364 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5365 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5366
5367 WREG32(TC_CFG_L1_VOLATILE, 0);
5368 WREG32(TC_CFG_L2_VOLATILE, 0);
5369
5370 if (rdev->family == CHIP_KAVERI) {
5371 u32 tmp = RREG32(CHUB_CONTROL);
5372 tmp &= ~BYPASS_VM;
5373 WREG32(CHUB_CONTROL, tmp);
5374 }
5375
5376 /* XXX SH_MEM regs */
5377 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5378 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5379 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5380 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5381 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5382 WREG32(SH_MEM_CONFIG, 0);
5383 WREG32(SH_MEM_APE1_BASE, 1);
5384 WREG32(SH_MEM_APE1_LIMIT, 0);
5385 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5386 /* SDMA GFX */
5387 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5388 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5389 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5390 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5391 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5392 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5393 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5394 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5395
5396 cik_pcie_gart_tlb_flush(rdev);
5397 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5398 (unsigned)(rdev->mc.gtt_size >> 20),
5399 (unsigned long long)rdev->gart.table_addr);
5400 rdev->gart.ready = true;
5401 return 0;
5402}
5403
5404/**
5405 * cik_pcie_gart_disable - gart disable
5406 *
5407 * @rdev: radeon_device pointer
5408 *
5409 * This disables all VM page table (CIK).
5410 */
5411static void cik_pcie_gart_disable(struct radeon_device *rdev)
5412{
5413 /* Disable all tables */
5414 WREG32(VM_CONTEXT0_CNTL, 0);
5415 WREG32(VM_CONTEXT1_CNTL, 0);
5416 /* Setup TLB control */
5417 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5418 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5419 /* Setup L2 cache */
5420 WREG32(VM_L2_CNTL,
5421 ENABLE_L2_FRAGMENT_PROCESSING |
5422 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5423 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5424 EFFECTIVE_L2_QUEUE_SIZE(7) |
5425 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5426 WREG32(VM_L2_CNTL2, 0);
5427 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5428 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5429 radeon_gart_table_vram_unpin(rdev);
5430}
5431
5432/**
5433 * cik_pcie_gart_fini - vm fini callback
5434 *
5435 * @rdev: radeon_device pointer
5436 *
5437 * Tears down the driver GART/VM setup (CIK).
5438 */
5439static void cik_pcie_gart_fini(struct radeon_device *rdev)
5440{
5441 cik_pcie_gart_disable(rdev);
5442 radeon_gart_table_vram_free(rdev);
5443 radeon_gart_fini(rdev);
5444}
5445
5446/* vm parser */
5447/**
5448 * cik_ib_parse - vm ib_parse callback
5449 *
5450 * @rdev: radeon_device pointer
5451 * @ib: indirect buffer pointer
5452 *
5453 * CIK uses hw IB checking so this is a nop (CIK).
5454 */
5455int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5456{
5457 return 0;
5458}
5459
5460/*
5461 * vm
5462 * VMID 0 is the physical GPU addresses as used by the kernel.
5463 * VMIDs 1-15 are used for userspace clients and are handled
5464 * by the radeon vm/hsa code.
5465 */
5466/**
5467 * cik_vm_init - cik vm init callback
5468 *
5469 * @rdev: radeon_device pointer
5470 *
5471 * Inits cik specific vm parameters (number of VMs, base of vram for
5472 * VMIDs 1-15) (CIK).
5473 * Returns 0 for success.
5474 */
5475int cik_vm_init(struct radeon_device *rdev)
5476{
5477 /* number of VMs */
5478 rdev->vm_manager.nvm = 16;
5479 /* base offset of vram pages */
5480 if (rdev->flags & RADEON_IS_IGP) {
5481 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5482 tmp <<= 22;
5483 rdev->vm_manager.vram_base_offset = tmp;
5484 } else
5485 rdev->vm_manager.vram_base_offset = 0;
5486
5487 return 0;
5488}
5489
5490/**
5491 * cik_vm_fini - cik vm fini callback
5492 *
5493 * @rdev: radeon_device pointer
5494 *
5495 * Tear down any asic specific VM setup (CIK).
5496 */
5497void cik_vm_fini(struct radeon_device *rdev)
5498{
5499}
5500
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5501/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5502 * cik_vm_decode_fault - print human readable fault info
5503 *
5504 * @rdev: radeon_device pointer
5505 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5506 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5507 *
5508 * Print human readable fault information (CIK).
5509 */
5510static void cik_vm_decode_fault(struct radeon_device *rdev,
5511 u32 status, u32 addr, u32 mc_client)
5512{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5513 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5514 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5515 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5516 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5517 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5518
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5519 if (rdev->family == CHIP_HAWAII)
5520 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5521 else
5522 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5523
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5524 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5525 protections, vmid, addr,
5526 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5527 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5528}
5529
5530/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5531 * cik_vm_flush - cik vm flush using the CP
5532 *
5533 * @rdev: radeon_device pointer
5534 *
5535 * Update the page table base and flush the VM TLB
5536 * using the CP (CIK).
5537 */
5538void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5539{
5540 struct radeon_ring *ring = &rdev->ring[ridx];
5541
5542 if (vm == NULL)
5543 return;
5544
5545 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5546 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5547 WRITE_DATA_DST_SEL(0)));
5548 if (vm->id < 8) {
5549 radeon_ring_write(ring,
5550 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5551 } else {
5552 radeon_ring_write(ring,
5553 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5554 }
5555 radeon_ring_write(ring, 0);
5556 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5557
5558 /* update SH_MEM_* regs */
5559 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5560 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5561 WRITE_DATA_DST_SEL(0)));
5562 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5563 radeon_ring_write(ring, 0);
5564 radeon_ring_write(ring, VMID(vm->id));
5565
5566 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5567 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5568 WRITE_DATA_DST_SEL(0)));
5569 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5570 radeon_ring_write(ring, 0);
5571
5572 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5573 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5574 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5575 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5576
5577 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5578 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5579 WRITE_DATA_DST_SEL(0)));
5580 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5581 radeon_ring_write(ring, 0);
5582 radeon_ring_write(ring, VMID(0));
5583
5584 /* HDP flush */
Alex Deucher780f5dd2014-01-09 16:18:11 -0500[diff] [blame]5585 cik_hdp_flush_cp_ring_emit(rdev, ridx);
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5586
5587 /* bits 0-15 are the VM contexts0-15 */
5588 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5589 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5590 WRITE_DATA_DST_SEL(0)));
5591 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5592 radeon_ring_write(ring, 0);
5593 radeon_ring_write(ring, 1 << vm->id);
5594
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5595 /* compute doesn't have PFP */
5596 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5597 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5598 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5599 radeon_ring_write(ring, 0x0);
5600 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5601}
5602
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5603/*
5604 * RLC
5605 * The RLC is a multi-purpose microengine that handles a
5606 * variety of functions, the most important of which is
5607 * the interrupt controller.
5608 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5609static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5610 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5611{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5612 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5613
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5614 if (enable)
5615 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5616 else
5617 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5618 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5619}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5620
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5621static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5622{
5623 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5624
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5625 tmp = RREG32(RLC_LB_CNTL);
5626 if (enable)
5627 tmp |= LOAD_BALANCE_ENABLE;
5628 else
5629 tmp &= ~LOAD_BALANCE_ENABLE;
5630 WREG32(RLC_LB_CNTL, tmp);
5631}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5632
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5633static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5634{
5635 u32 i, j, k;
5636 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5637
5638 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5639 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5640 cik_select_se_sh(rdev, i, j);
5641 for (k = 0; k < rdev->usec_timeout; k++) {
5642 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5643 break;
5644 udelay(1);
5645 }
5646 }
5647 }
5648 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5649
5650 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5651 for (k = 0; k < rdev->usec_timeout; k++) {
5652 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5653 break;
5654 udelay(1);
5655 }
5656}
5657
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5658static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5659{
5660 u32 tmp;
5661
5662 tmp = RREG32(RLC_CNTL);
5663 if (tmp != rlc)
5664 WREG32(RLC_CNTL, rlc);
5665}
5666
5667static u32 cik_halt_rlc(struct radeon_device *rdev)
5668{
5669 u32 data, orig;
5670
5671 orig = data = RREG32(RLC_CNTL);
5672
5673 if (data & RLC_ENABLE) {
5674 u32 i;
5675
5676 data &= ~RLC_ENABLE;
5677 WREG32(RLC_CNTL, data);
5678
5679 for (i = 0; i < rdev->usec_timeout; i++) {
5680 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5681 break;
5682 udelay(1);
5683 }
5684
5685 cik_wait_for_rlc_serdes(rdev);
5686 }
5687
5688 return orig;
5689}
5690
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5691void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5692{
5693 u32 tmp, i, mask;
5694
5695 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5696 WREG32(RLC_GPR_REG2, tmp);
5697
5698 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5699 for (i = 0; i < rdev->usec_timeout; i++) {
5700 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5701 break;
5702 udelay(1);
5703 }
5704
5705 for (i = 0; i < rdev->usec_timeout; i++) {
5706 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5707 break;
5708 udelay(1);
5709 }
5710}
5711
5712void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5713{
5714 u32 tmp;
5715
5716 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5717 WREG32(RLC_GPR_REG2, tmp);
5718}
5719
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5720/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5721 * cik_rlc_stop - stop the RLC ME
5722 *
5723 * @rdev: radeon_device pointer
5724 *
5725 * Halt the RLC ME (MicroEngine) (CIK).
5726 */
5727static void cik_rlc_stop(struct radeon_device *rdev)
5728{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5729 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5730
5731 cik_enable_gui_idle_interrupt(rdev, false);
5732
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5733 cik_wait_for_rlc_serdes(rdev);
5734}
5735
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5736/**
5737 * cik_rlc_start - start the RLC ME
5738 *
5739 * @rdev: radeon_device pointer
5740 *
5741 * Unhalt the RLC ME (MicroEngine) (CIK).
5742 */
5743static void cik_rlc_start(struct radeon_device *rdev)
5744{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5745 WREG32(RLC_CNTL, RLC_ENABLE);
5746
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5747 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5748
5749 udelay(50);
5750}
5751
5752/**
5753 * cik_rlc_resume - setup the RLC hw
5754 *
5755 * @rdev: radeon_device pointer
5756 *
5757 * Initialize the RLC registers, load the ucode,
5758 * and start the RLC (CIK).
5759 * Returns 0 for success, -EINVAL if the ucode is not available.
5760 */
5761static int cik_rlc_resume(struct radeon_device *rdev)
5762{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5763 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5764 const __be32 *fw_data;
5765
5766 if (!rdev->rlc_fw)
5767 return -EINVAL;
5768
5769 switch (rdev->family) {
5770 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5771 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5772 default:
5773 size = BONAIRE_RLC_UCODE_SIZE;
5774 break;
5775 case CHIP_KAVERI:
5776 size = KV_RLC_UCODE_SIZE;
5777 break;
5778 case CHIP_KABINI:
5779 size = KB_RLC_UCODE_SIZE;
5780 break;
5781 }
5782
5783 cik_rlc_stop(rdev);
5784
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5785 /* disable CG */
5786 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5787 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5788
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5789 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5790
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5791 cik_init_pg(rdev);
5792
5793 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5794
5795 WREG32(RLC_LB_CNTR_INIT, 0);
5796 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5797
5798 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5799 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5800 WREG32(RLC_LB_PARAMS, 0x00600408);
5801 WREG32(RLC_LB_CNTL, 0x80000004);
5802
5803 WREG32(RLC_MC_CNTL, 0);
5804 WREG32(RLC_UCODE_CNTL, 0);
5805
5806 fw_data = (const __be32 *)rdev->rlc_fw->data;
5807 WREG32(RLC_GPM_UCODE_ADDR, 0);
5808 for (i = 0; i < size; i++)
5809 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5810 WREG32(RLC_GPM_UCODE_ADDR, 0);
5811
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5812 /* XXX - find out what chips support lbpw */
5813 cik_enable_lbpw(rdev, false);
5814
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5815 if (rdev->family == CHIP_BONAIRE)
5816 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5817
5818 cik_rlc_start(rdev);
5819
5820 return 0;
5821}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5822
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5823static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5824{
5825 u32 data, orig, tmp, tmp2;
5826
5827 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5828
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5829 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5830 cik_enable_gui_idle_interrupt(rdev, true);
5831
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5832 tmp = cik_halt_rlc(rdev);
5833
5834 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5835 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5836 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5837 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5838 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5839
5840 cik_update_rlc(rdev, tmp);
5841
5842 data |= CGCG_EN | CGLS_EN;
5843 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5844 cik_enable_gui_idle_interrupt(rdev, false);
5845
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5846 RREG32(CB_CGTT_SCLK_CTRL);
5847 RREG32(CB_CGTT_SCLK_CTRL);
5848 RREG32(CB_CGTT_SCLK_CTRL);
5849 RREG32(CB_CGTT_SCLK_CTRL);
5850
5851 data &= ~(CGCG_EN | CGLS_EN);
5852 }
5853
5854 if (orig != data)
5855 WREG32(RLC_CGCG_CGLS_CTRL, data);
5856
5857}
5858
5859static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5860{
5861 u32 data, orig, tmp = 0;
5862
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5863 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5864 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5865 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5866 orig = data = RREG32(CP_MEM_SLP_CNTL);
5867 data |= CP_MEM_LS_EN;
5868 if (orig != data)
5869 WREG32(CP_MEM_SLP_CNTL, data);
5870 }
5871 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5872
5873 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5874 data &= 0xfffffffd;
5875 if (orig != data)
5876 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5877
5878 tmp = cik_halt_rlc(rdev);
5879
5880 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5881 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5882 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5883 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5884 WREG32(RLC_SERDES_WR_CTRL, data);
5885
5886 cik_update_rlc(rdev, tmp);
5887
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5888 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5889 orig = data = RREG32(CGTS_SM_CTRL_REG);
5890 data &= ~SM_MODE_MASK;
5891 data |= SM_MODE(0x2);
5892 data |= SM_MODE_ENABLE;
5893 data &= ~CGTS_OVERRIDE;
5894 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5895 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5896 data &= ~CGTS_LS_OVERRIDE;
5897 data &= ~ON_MONITOR_ADD_MASK;
5898 data |= ON_MONITOR_ADD_EN;
5899 data |= ON_MONITOR_ADD(0x96);
5900 if (orig != data)
5901 WREG32(CGTS_SM_CTRL_REG, data);
5902 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5903 } else {
5904 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5905 data |= 0x00000002;
5906 if (orig != data)
5907 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5908
5909 data = RREG32(RLC_MEM_SLP_CNTL);
5910 if (data & RLC_MEM_LS_EN) {
5911 data &= ~RLC_MEM_LS_EN;
5912 WREG32(RLC_MEM_SLP_CNTL, data);
5913 }
5914
5915 data = RREG32(CP_MEM_SLP_CNTL);
5916 if (data & CP_MEM_LS_EN) {
5917 data &= ~CP_MEM_LS_EN;
5918 WREG32(CP_MEM_SLP_CNTL, data);
5919 }
5920
5921 orig = data = RREG32(CGTS_SM_CTRL_REG);
5922 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5923 if (orig != data)
5924 WREG32(CGTS_SM_CTRL_REG, data);
5925
5926 tmp = cik_halt_rlc(rdev);
5927
5928 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5929 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5930 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5931 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5932 WREG32(RLC_SERDES_WR_CTRL, data);
5933
5934 cik_update_rlc(rdev, tmp);
5935 }
5936}
5937
5938static const u32 mc_cg_registers[] =
5939{
5940 MC_HUB_MISC_HUB_CG,
5941 MC_HUB_MISC_SIP_CG,
5942 MC_HUB_MISC_VM_CG,
5943 MC_XPB_CLK_GAT,
5944 ATC_MISC_CG,
5945 MC_CITF_MISC_WR_CG,
5946 MC_CITF_MISC_RD_CG,
5947 MC_CITF_MISC_VM_CG,
5948 VM_L2_CG,
5949};
5950
5951static void cik_enable_mc_ls(struct radeon_device *rdev,
5952 bool enable)
5953{
5954 int i;
5955 u32 orig, data;
5956
5957 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5958 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5959 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5960 data |= MC_LS_ENABLE;
5961 else
5962 data &= ~MC_LS_ENABLE;
5963 if (data != orig)
5964 WREG32(mc_cg_registers[i], data);
5965 }
5966}
5967
5968static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5969 bool enable)
5970{
5971 int i;
5972 u32 orig, data;
5973
5974 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5975 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5976 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5977 data |= MC_CG_ENABLE;
5978 else
5979 data &= ~MC_CG_ENABLE;
5980 if (data != orig)
5981 WREG32(mc_cg_registers[i], data);
5982 }
5983}
5984
5985static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5986 bool enable)
5987{
5988 u32 orig, data;
5989
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5990 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5991 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5992 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5993 } else {
5994 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5995 data |= 0xff000000;
5996 if (data != orig)
5997 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5998
5999 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6000 data |= 0xff000000;
6001 if (data != orig)
6002 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6003 }
6004}
6005
6006static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6007 bool enable)
6008{
6009 u32 orig, data;
6010
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6011 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6012 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6013 data |= 0x100;
6014 if (orig != data)
6015 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6016
6017 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6018 data |= 0x100;
6019 if (orig != data)
6020 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6021 } else {
6022 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6023 data &= ~0x100;
6024 if (orig != data)
6025 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6026
6027 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6028 data &= ~0x100;
6029 if (orig != data)
6030 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6031 }
6032}
6033
6034static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6035 bool enable)
6036{
6037 u32 orig, data;
6038
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6039 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6040 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6041 data = 0xfff;
6042 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6043
6044 orig = data = RREG32(UVD_CGC_CTRL);
6045 data |= DCM;
6046 if (orig != data)
6047 WREG32(UVD_CGC_CTRL, data);
6048 } else {
6049 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6050 data &= ~0xfff;
6051 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6052
6053 orig = data = RREG32(UVD_CGC_CTRL);
6054 data &= ~DCM;
6055 if (orig != data)
6056 WREG32(UVD_CGC_CTRL, data);
6057 }
6058}
6059
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6060static void cik_enable_bif_mgls(struct radeon_device *rdev,
6061 bool enable)
6062{
6063 u32 orig, data;
6064
6065 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6066
6067 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6068 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6069 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6070 else
6071 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6072 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6073
6074 if (orig != data)
6075 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6076}
6077
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6078static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6079 bool enable)
6080{
6081 u32 orig, data;
6082
6083 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6084
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6085 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6086 data &= ~CLOCK_GATING_DIS;
6087 else
6088 data |= CLOCK_GATING_DIS;
6089
6090 if (orig != data)
6091 WREG32(HDP_HOST_PATH_CNTL, data);
6092}
6093
6094static void cik_enable_hdp_ls(struct radeon_device *rdev,
6095 bool enable)
6096{
6097 u32 orig, data;
6098
6099 orig = data = RREG32(HDP_MEM_POWER_LS);
6100
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6101 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6102 data |= HDP_LS_ENABLE;
6103 else
6104 data &= ~HDP_LS_ENABLE;
6105
6106 if (orig != data)
6107 WREG32(HDP_MEM_POWER_LS, data);
6108}
6109
6110void cik_update_cg(struct radeon_device *rdev,
6111 u32 block, bool enable)
6112{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6113
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6114 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6115 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6116 /* order matters! */
6117 if (enable) {
6118 cik_enable_mgcg(rdev, true);
6119 cik_enable_cgcg(rdev, true);
6120 } else {
6121 cik_enable_cgcg(rdev, false);
6122 cik_enable_mgcg(rdev, false);
6123 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6124 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6125 }
6126
6127 if (block & RADEON_CG_BLOCK_MC) {
6128 if (!(rdev->flags & RADEON_IS_IGP)) {
6129 cik_enable_mc_mgcg(rdev, enable);
6130 cik_enable_mc_ls(rdev, enable);
6131 }
6132 }
6133
6134 if (block & RADEON_CG_BLOCK_SDMA) {
6135 cik_enable_sdma_mgcg(rdev, enable);
6136 cik_enable_sdma_mgls(rdev, enable);
6137 }
6138
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6139 if (block & RADEON_CG_BLOCK_BIF) {
6140 cik_enable_bif_mgls(rdev, enable);
6141 }
6142
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6143 if (block & RADEON_CG_BLOCK_UVD) {
6144 if (rdev->has_uvd)
6145 cik_enable_uvd_mgcg(rdev, enable);
6146 }
6147
6148 if (block & RADEON_CG_BLOCK_HDP) {
6149 cik_enable_hdp_mgcg(rdev, enable);
6150 cik_enable_hdp_ls(rdev, enable);
6151 }
6152}
6153
6154static void cik_init_cg(struct radeon_device *rdev)
6155{
6156
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]6157 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6158
6159 if (rdev->has_uvd)
6160 si_init_uvd_internal_cg(rdev);
6161
6162 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6163 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6164 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6165 RADEON_CG_BLOCK_UVD |
6166 RADEON_CG_BLOCK_HDP), true);
6167}
6168
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6169static void cik_fini_cg(struct radeon_device *rdev)
6170{
6171 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6172 RADEON_CG_BLOCK_SDMA |
6173 RADEON_CG_BLOCK_BIF |
6174 RADEON_CG_BLOCK_UVD |
6175 RADEON_CG_BLOCK_HDP), false);
6176
6177 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6178}
6179
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6180static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6181 bool enable)
6182{
6183 u32 data, orig;
6184
6185 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6186 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6187 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6188 else
6189 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6190 if (orig != data)
6191 WREG32(RLC_PG_CNTL, data);
6192}
6193
6194static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6195 bool enable)
6196{
6197 u32 data, orig;
6198
6199 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6200 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6201 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6202 else
6203 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6204 if (orig != data)
6205 WREG32(RLC_PG_CNTL, data);
6206}
6207
6208static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6209{
6210 u32 data, orig;
6211
6212 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6213 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6214 data &= ~DISABLE_CP_PG;
6215 else
6216 data |= DISABLE_CP_PG;
6217 if (orig != data)
6218 WREG32(RLC_PG_CNTL, data);
6219}
6220
6221static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6222{
6223 u32 data, orig;
6224
6225 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6226 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6227 data &= ~DISABLE_GDS_PG;
6228 else
6229 data |= DISABLE_GDS_PG;
6230 if (orig != data)
6231 WREG32(RLC_PG_CNTL, data);
6232}
6233
6234#define CP_ME_TABLE_SIZE 96
6235#define CP_ME_TABLE_OFFSET 2048
6236#define CP_MEC_TABLE_OFFSET 4096
6237
6238void cik_init_cp_pg_table(struct radeon_device *rdev)
6239{
6240 const __be32 *fw_data;
6241 volatile u32 *dst_ptr;
6242 int me, i, max_me = 4;
6243 u32 bo_offset = 0;
6244 u32 table_offset;
6245
6246 if (rdev->family == CHIP_KAVERI)
6247 max_me = 5;
6248
6249 if (rdev->rlc.cp_table_ptr == NULL)
6250 return;
6251
6252 /* write the cp table buffer */
6253 dst_ptr = rdev->rlc.cp_table_ptr;
6254 for (me = 0; me < max_me; me++) {
6255 if (me == 0) {
6256 fw_data = (const __be32 *)rdev->ce_fw->data;
6257 table_offset = CP_ME_TABLE_OFFSET;
6258 } else if (me == 1) {
6259 fw_data = (const __be32 *)rdev->pfp_fw->data;
6260 table_offset = CP_ME_TABLE_OFFSET;
6261 } else if (me == 2) {
6262 fw_data = (const __be32 *)rdev->me_fw->data;
6263 table_offset = CP_ME_TABLE_OFFSET;
6264 } else {
6265 fw_data = (const __be32 *)rdev->mec_fw->data;
6266 table_offset = CP_MEC_TABLE_OFFSET;
6267 }
6268
6269 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6270 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6271 }
6272 bo_offset += CP_ME_TABLE_SIZE;
6273 }
6274}
6275
6276static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6277 bool enable)
6278{
6279 u32 data, orig;
6280
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6281 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6282 orig = data = RREG32(RLC_PG_CNTL);
6283 data |= GFX_PG_ENABLE;
6284 if (orig != data)
6285 WREG32(RLC_PG_CNTL, data);
6286
6287 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6288 data |= AUTO_PG_EN;
6289 if (orig != data)
6290 WREG32(RLC_AUTO_PG_CTRL, data);
6291 } else {
6292 orig = data = RREG32(RLC_PG_CNTL);
6293 data &= ~GFX_PG_ENABLE;
6294 if (orig != data)
6295 WREG32(RLC_PG_CNTL, data);
6296
6297 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6298 data &= ~AUTO_PG_EN;
6299 if (orig != data)
6300 WREG32(RLC_AUTO_PG_CTRL, data);
6301
6302 data = RREG32(DB_RENDER_CONTROL);
6303 }
6304}
6305
6306static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6307{
6308 u32 mask = 0, tmp, tmp1;
6309 int i;
6310
6311 cik_select_se_sh(rdev, se, sh);
6312 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6313 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6314 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6315
6316 tmp &= 0xffff0000;
6317
6318 tmp |= tmp1;
6319 tmp >>= 16;
6320
6321 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6322 mask <<= 1;
6323 mask |= 1;
6324 }
6325
6326 return (~tmp) & mask;
6327}
6328
6329static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6330{
6331 u32 i, j, k, active_cu_number = 0;
6332 u32 mask, counter, cu_bitmap;
6333 u32 tmp = 0;
6334
6335 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6336 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6337 mask = 1;
6338 cu_bitmap = 0;
6339 counter = 0;
6340 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6341 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6342 if (counter < 2)
6343 cu_bitmap |= mask;
6344 counter ++;
6345 }
6346 mask <<= 1;
6347 }
6348
6349 active_cu_number += counter;
6350 tmp |= (cu_bitmap << (i * 16 + j * 8));
6351 }
6352 }
6353
6354 WREG32(RLC_PG_AO_CU_MASK, tmp);
6355
6356 tmp = RREG32(RLC_MAX_PG_CU);
6357 tmp &= ~MAX_PU_CU_MASK;
6358 tmp |= MAX_PU_CU(active_cu_number);
6359 WREG32(RLC_MAX_PG_CU, tmp);
6360}
6361
6362static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6363 bool enable)
6364{
6365 u32 data, orig;
6366
6367 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6368 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6369 data |= STATIC_PER_CU_PG_ENABLE;
6370 else
6371 data &= ~STATIC_PER_CU_PG_ENABLE;
6372 if (orig != data)
6373 WREG32(RLC_PG_CNTL, data);
6374}
6375
6376static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6377 bool enable)
6378{
6379 u32 data, orig;
6380
6381 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6382 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6383 data |= DYN_PER_CU_PG_ENABLE;
6384 else
6385 data &= ~DYN_PER_CU_PG_ENABLE;
6386 if (orig != data)
6387 WREG32(RLC_PG_CNTL, data);
6388}
6389
6390#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6391#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6392
6393static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6394{
6395 u32 data, orig;
6396 u32 i;
6397
6398 if (rdev->rlc.cs_data) {
6399 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6400 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6401 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6402 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6403 } else {
6404 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6405 for (i = 0; i < 3; i++)
6406 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6407 }
6408 if (rdev->rlc.reg_list) {
6409 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6410 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6411 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6412 }
6413
6414 orig = data = RREG32(RLC_PG_CNTL);
6415 data |= GFX_PG_SRC;
6416 if (orig != data)
6417 WREG32(RLC_PG_CNTL, data);
6418
6419 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6420 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6421
6422 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6423 data &= ~IDLE_POLL_COUNT_MASK;
6424 data |= IDLE_POLL_COUNT(0x60);
6425 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6426
6427 data = 0x10101010;
6428 WREG32(RLC_PG_DELAY, data);
6429
6430 data = RREG32(RLC_PG_DELAY_2);
6431 data &= ~0xff;
6432 data |= 0x3;
6433 WREG32(RLC_PG_DELAY_2, data);
6434
6435 data = RREG32(RLC_AUTO_PG_CTRL);
6436 data &= ~GRBM_REG_SGIT_MASK;
6437 data |= GRBM_REG_SGIT(0x700);
6438 WREG32(RLC_AUTO_PG_CTRL, data);
6439
6440}
6441
6442static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6443{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6444 cik_enable_gfx_cgpg(rdev, enable);
6445 cik_enable_gfx_static_mgpg(rdev, enable);
6446 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6447}
6448
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6449u32 cik_get_csb_size(struct radeon_device *rdev)
6450{
6451 u32 count = 0;
6452 const struct cs_section_def *sect = NULL;
6453 const struct cs_extent_def *ext = NULL;
6454
6455 if (rdev->rlc.cs_data == NULL)
6456 return 0;
6457
6458 /* begin clear state */
6459 count += 2;
6460 /* context control state */
6461 count += 3;
6462
6463 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6464 for (ext = sect->section; ext->extent != NULL; ++ext) {
6465 if (sect->id == SECT_CONTEXT)
6466 count += 2 + ext->reg_count;
6467 else
6468 return 0;
6469 }
6470 }
6471 /* pa_sc_raster_config/pa_sc_raster_config1 */
6472 count += 4;
6473 /* end clear state */
6474 count += 2;
6475 /* clear state */
6476 count += 2;
6477
6478 return count;
6479}
6480
6481void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6482{
6483 u32 count = 0, i;
6484 const struct cs_section_def *sect = NULL;
6485 const struct cs_extent_def *ext = NULL;
6486
6487 if (rdev->rlc.cs_data == NULL)
6488 return;
6489 if (buffer == NULL)
6490 return;
6491
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6492 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6493 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6494
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6495 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6496 buffer[count++] = cpu_to_le32(0x80000000);
6497 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6498
6499 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6500 for (ext = sect->section; ext->extent != NULL; ++ext) {
6501 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6502 buffer[count++] =
6503 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6504 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6505 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6506 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6507 } else {
6508 return;
6509 }
6510 }
6511 }
6512
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6513 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6514 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6515 switch (rdev->family) {
6516 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6517 buffer[count++] = cpu_to_le32(0x16000012);
6518 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6519 break;
6520 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6521 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6522 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6523 break;
6524 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6525 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6526 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6527 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame]6528 case CHIP_HAWAII:
6529 buffer[count++] = 0x3a00161a;
6530 buffer[count++] = 0x0000002e;
6531 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6532 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6533 buffer[count++] = cpu_to_le32(0x00000000);
6534 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6535 break;
6536 }
6537
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6538 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6539 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6540
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6541 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6542 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6543}
6544
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6545static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6546{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6547 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6548 cik_enable_sck_slowdown_on_pu(rdev, true);
6549 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6550 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6551 cik_init_gfx_cgpg(rdev);
6552 cik_enable_cp_pg(rdev, true);
6553 cik_enable_gds_pg(rdev, true);
6554 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6555 cik_init_ao_cu_mask(rdev);
6556 cik_update_gfx_pg(rdev, true);
6557 }
6558}
6559
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6560static void cik_fini_pg(struct radeon_device *rdev)
6561{
6562 if (rdev->pg_flags) {
6563 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6564 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6565 cik_enable_cp_pg(rdev, false);
6566 cik_enable_gds_pg(rdev, false);
6567 }
6568 }
6569}
6570
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6571/*
6572 * Interrupts
6573 * Starting with r6xx, interrupts are handled via a ring buffer.
6574 * Ring buffers are areas of GPU accessible memory that the GPU
6575 * writes interrupt vectors into and the host reads vectors out of.
6576 * There is a rptr (read pointer) that determines where the
6577 * host is currently reading, and a wptr (write pointer)
6578 * which determines where the GPU has written. When the
6579 * pointers are equal, the ring is idle. When the GPU
6580 * writes vectors to the ring buffer, it increments the
6581 * wptr. When there is an interrupt, the host then starts
6582 * fetching commands and processing them until the pointers are
6583 * equal again at which point it updates the rptr.
6584 */
6585
6586/**
6587 * cik_enable_interrupts - Enable the interrupt ring buffer
6588 *
6589 * @rdev: radeon_device pointer
6590 *
6591 * Enable the interrupt ring buffer (CIK).
6592 */
6593static void cik_enable_interrupts(struct radeon_device *rdev)
6594{
6595 u32 ih_cntl = RREG32(IH_CNTL);
6596 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6597
6598 ih_cntl |= ENABLE_INTR;
6599 ih_rb_cntl |= IH_RB_ENABLE;
6600 WREG32(IH_CNTL, ih_cntl);
6601 WREG32(IH_RB_CNTL, ih_rb_cntl);
6602 rdev->ih.enabled = true;
6603}
6604
6605/**
6606 * cik_disable_interrupts - Disable the interrupt ring buffer
6607 *
6608 * @rdev: radeon_device pointer
6609 *
6610 * Disable the interrupt ring buffer (CIK).
6611 */
6612static void cik_disable_interrupts(struct radeon_device *rdev)
6613{
6614 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6615 u32 ih_cntl = RREG32(IH_CNTL);
6616
6617 ih_rb_cntl &= ~IH_RB_ENABLE;
6618 ih_cntl &= ~ENABLE_INTR;
6619 WREG32(IH_RB_CNTL, ih_rb_cntl);
6620 WREG32(IH_CNTL, ih_cntl);
6621 /* set rptr, wptr to 0 */
6622 WREG32(IH_RB_RPTR, 0);
6623 WREG32(IH_RB_WPTR, 0);
6624 rdev->ih.enabled = false;
6625 rdev->ih.rptr = 0;
6626}
6627
6628/**
6629 * cik_disable_interrupt_state - Disable all interrupt sources
6630 *
6631 * @rdev: radeon_device pointer
6632 *
6633 * Clear all interrupt enable bits used by the driver (CIK).
6634 */
6635static void cik_disable_interrupt_state(struct radeon_device *rdev)
6636{
6637 u32 tmp;
6638
6639 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6640 tmp = RREG32(CP_INT_CNTL_RING0) &
6641 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6642 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6643 /* sdma */
6644 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6645 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6646 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6647 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6648 /* compute queues */
6649 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6650 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6651 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6652 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6653 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6654 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6655 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6656 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6657 /* grbm */
6658 WREG32(GRBM_INT_CNTL, 0);
6659 /* vline/vblank, etc. */
6660 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6661 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6662 if (rdev->num_crtc >= 4) {
6663 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6664 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6665 }
6666 if (rdev->num_crtc >= 6) {
6667 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6668 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6669 }
6670
6671 /* dac hotplug */
6672 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6673
6674 /* digital hotplug */
6675 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6676 WREG32(DC_HPD1_INT_CONTROL, tmp);
6677 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6678 WREG32(DC_HPD2_INT_CONTROL, tmp);
6679 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6680 WREG32(DC_HPD3_INT_CONTROL, tmp);
6681 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6682 WREG32(DC_HPD4_INT_CONTROL, tmp);
6683 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6684 WREG32(DC_HPD5_INT_CONTROL, tmp);
6685 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6686 WREG32(DC_HPD6_INT_CONTROL, tmp);
6687
6688}
6689
6690/**
6691 * cik_irq_init - init and enable the interrupt ring
6692 *
6693 * @rdev: radeon_device pointer
6694 *
6695 * Allocate a ring buffer for the interrupt controller,
6696 * enable the RLC, disable interrupts, enable the IH
6697 * ring buffer and enable it (CIK).
6698 * Called at device load and reume.
6699 * Returns 0 for success, errors for failure.
6700 */
6701static int cik_irq_init(struct radeon_device *rdev)
6702{
6703 int ret = 0;
6704 int rb_bufsz;
6705 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6706
6707 /* allocate ring */
6708 ret = r600_ih_ring_alloc(rdev);
6709 if (ret)
6710 return ret;
6711
6712 /* disable irqs */
6713 cik_disable_interrupts(rdev);
6714
6715 /* init rlc */
6716 ret = cik_rlc_resume(rdev);
6717 if (ret) {
6718 r600_ih_ring_fini(rdev);
6719 return ret;
6720 }
6721
6722 /* setup interrupt control */
6723 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6724 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6725 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6726 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6727 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6728 */
6729 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6730 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6731 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6732 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6733
6734 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6735 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6736
6737 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6738 IH_WPTR_OVERFLOW_CLEAR |
6739 (rb_bufsz << 1));
6740
6741 if (rdev->wb.enabled)
6742 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6743
6744 /* set the writeback address whether it's enabled or not */
6745 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6746 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6747
6748 WREG32(IH_RB_CNTL, ih_rb_cntl);
6749
6750 /* set rptr, wptr to 0 */
6751 WREG32(IH_RB_RPTR, 0);
6752 WREG32(IH_RB_WPTR, 0);
6753
6754 /* Default settings for IH_CNTL (disabled at first) */
6755 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6756 /* RPTR_REARM only works if msi's are enabled */
6757 if (rdev->msi_enabled)
6758 ih_cntl |= RPTR_REARM;
6759 WREG32(IH_CNTL, ih_cntl);
6760
6761 /* force the active interrupt state to all disabled */
6762 cik_disable_interrupt_state(rdev);
6763
6764 pci_set_master(rdev->pdev);
6765
6766 /* enable irqs */
6767 cik_enable_interrupts(rdev);
6768
6769 return ret;
6770}
6771
6772/**
6773 * cik_irq_set - enable/disable interrupt sources
6774 *
6775 * @rdev: radeon_device pointer
6776 *
6777 * Enable interrupt sources on the GPU (vblanks, hpd,
6778 * etc.) (CIK).
6779 * Returns 0 for success, errors for failure.
6780 */
6781int cik_irq_set(struct radeon_device *rdev)
6782{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6783 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6784 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6785 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6786 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6787 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6788 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6789 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6790 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6791
6792 if (!rdev->irq.installed) {
6793 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6794 return -EINVAL;
6795 }
6796 /* don't enable anything if the ih is disabled */
6797 if (!rdev->ih.enabled) {
6798 cik_disable_interrupts(rdev);
6799 /* force the active interrupt state to all disabled */
6800 cik_disable_interrupt_state(rdev);
6801 return 0;
6802 }
6803
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6804 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6805 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6806 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6807
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6808 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6809 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6810 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6811 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6812 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6813 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6814
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6815 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6816 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6817
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6818 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6819 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6820 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6821 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6822 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6823 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6824 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6825 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6826
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6827 if (rdev->flags & RADEON_IS_IGP)
6828 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6829 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6830 else
6831 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6832 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6833
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6834 /* enable CP interrupts on all rings */
6835 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6836 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6837 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6838 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6839 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6840 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6841 DRM_DEBUG("si_irq_set: sw int cp1\n");
6842 if (ring->me == 1) {
6843 switch (ring->pipe) {
6844 case 0:
6845 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6846 break;
6847 case 1:
6848 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6849 break;
6850 case 2:
6851 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6852 break;
6853 case 3:
6854 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6855 break;
6856 default:
6857 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6858 break;
6859 }
6860 } else if (ring->me == 2) {
6861 switch (ring->pipe) {
6862 case 0:
6863 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6864 break;
6865 case 1:
6866 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6867 break;
6868 case 2:
6869 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6870 break;
6871 case 3:
6872 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6873 break;
6874 default:
6875 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6876 break;
6877 }
6878 } else {
6879 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6880 }
6881 }
6882 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6883 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6884 DRM_DEBUG("si_irq_set: sw int cp2\n");
6885 if (ring->me == 1) {
6886 switch (ring->pipe) {
6887 case 0:
6888 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6889 break;
6890 case 1:
6891 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6892 break;
6893 case 2:
6894 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6895 break;
6896 case 3:
6897 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6898 break;
6899 default:
6900 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6901 break;
6902 }
6903 } else if (ring->me == 2) {
6904 switch (ring->pipe) {
6905 case 0:
6906 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6907 break;
6908 case 1:
6909 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6910 break;
6911 case 2:
6912 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6913 break;
6914 case 3:
6915 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6916 break;
6917 default:
6918 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6919 break;
6920 }
6921 } else {
6922 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6923 }
6924 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6925
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6926 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6927 DRM_DEBUG("cik_irq_set: sw int dma\n");
6928 dma_cntl |= TRAP_ENABLE;
6929 }
6930
6931 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6932 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6933 dma_cntl1 |= TRAP_ENABLE;
6934 }
6935
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6936 if (rdev->irq.crtc_vblank_int[0] ||
6937 atomic_read(&rdev->irq.pflip[0])) {
6938 DRM_DEBUG("cik_irq_set: vblank 0\n");
6939 crtc1 |= VBLANK_INTERRUPT_MASK;
6940 }
6941 if (rdev->irq.crtc_vblank_int[1] ||
6942 atomic_read(&rdev->irq.pflip[1])) {
6943 DRM_DEBUG("cik_irq_set: vblank 1\n");
6944 crtc2 |= VBLANK_INTERRUPT_MASK;
6945 }
6946 if (rdev->irq.crtc_vblank_int[2] ||
6947 atomic_read(&rdev->irq.pflip[2])) {
6948 DRM_DEBUG("cik_irq_set: vblank 2\n");
6949 crtc3 |= VBLANK_INTERRUPT_MASK;
6950 }
6951 if (rdev->irq.crtc_vblank_int[3] ||
6952 atomic_read(&rdev->irq.pflip[3])) {
6953 DRM_DEBUG("cik_irq_set: vblank 3\n");
6954 crtc4 |= VBLANK_INTERRUPT_MASK;
6955 }
6956 if (rdev->irq.crtc_vblank_int[4] ||
6957 atomic_read(&rdev->irq.pflip[4])) {
6958 DRM_DEBUG("cik_irq_set: vblank 4\n");
6959 crtc5 |= VBLANK_INTERRUPT_MASK;
6960 }
6961 if (rdev->irq.crtc_vblank_int[5] ||
6962 atomic_read(&rdev->irq.pflip[5])) {
6963 DRM_DEBUG("cik_irq_set: vblank 5\n");
6964 crtc6 |= VBLANK_INTERRUPT_MASK;
6965 }
6966 if (rdev->irq.hpd[0]) {
6967 DRM_DEBUG("cik_irq_set: hpd 1\n");
6968 hpd1 |= DC_HPDx_INT_EN;
6969 }
6970 if (rdev->irq.hpd[1]) {
6971 DRM_DEBUG("cik_irq_set: hpd 2\n");
6972 hpd2 |= DC_HPDx_INT_EN;
6973 }
6974 if (rdev->irq.hpd[2]) {
6975 DRM_DEBUG("cik_irq_set: hpd 3\n");
6976 hpd3 |= DC_HPDx_INT_EN;
6977 }
6978 if (rdev->irq.hpd[3]) {
6979 DRM_DEBUG("cik_irq_set: hpd 4\n");
6980 hpd4 |= DC_HPDx_INT_EN;
6981 }
6982 if (rdev->irq.hpd[4]) {
6983 DRM_DEBUG("cik_irq_set: hpd 5\n");
6984 hpd5 |= DC_HPDx_INT_EN;
6985 }
6986 if (rdev->irq.hpd[5]) {
6987 DRM_DEBUG("cik_irq_set: hpd 6\n");
6988 hpd6 |= DC_HPDx_INT_EN;
6989 }
6990
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6991 if (rdev->irq.dpm_thermal) {
6992 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6993 if (rdev->flags & RADEON_IS_IGP)
6994 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6995 else
6996 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6997 }
6998
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6999 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7000
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7001 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7002 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7003
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7004 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7005 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7006 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7007 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7008 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7009 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7010 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7011 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7012
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7013 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7014
7015 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7016 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7017 if (rdev->num_crtc >= 4) {
7018 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7019 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7020 }
7021 if (rdev->num_crtc >= 6) {
7022 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7023 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7024 }
7025
7026 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7027 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7028 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7029 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7030 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7031 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7032
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]7033 if (rdev->flags & RADEON_IS_IGP)
7034 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7035 else
7036 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7037
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7038 return 0;
7039}
7040
7041/**
7042 * cik_irq_ack - ack interrupt sources
7043 *
7044 * @rdev: radeon_device pointer
7045 *
7046 * Ack interrupt sources on the GPU (vblanks, hpd,
7047 * etc.) (CIK). Certain interrupts sources are sw
7048 * generated and do not require an explicit ack.
7049 */
7050static inline void cik_irq_ack(struct radeon_device *rdev)
7051{
7052 u32 tmp;
7053
7054 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7055 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7056 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7057 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7058 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7059 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7060 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7061
7062 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7063 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7064 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7065 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7066 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7067 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7068 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7069 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7070
7071 if (rdev->num_crtc >= 4) {
7072 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7073 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7074 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7075 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7076 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7077 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7078 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7079 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7080 }
7081
7082 if (rdev->num_crtc >= 6) {
7083 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7084 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7085 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7086 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7087 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7088 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7089 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7090 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7091 }
7092
7093 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7094 tmp = RREG32(DC_HPD1_INT_CONTROL);
7095 tmp |= DC_HPDx_INT_ACK;
7096 WREG32(DC_HPD1_INT_CONTROL, tmp);
7097 }
7098 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7099 tmp = RREG32(DC_HPD2_INT_CONTROL);
7100 tmp |= DC_HPDx_INT_ACK;
7101 WREG32(DC_HPD2_INT_CONTROL, tmp);
7102 }
7103 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7104 tmp = RREG32(DC_HPD3_INT_CONTROL);
7105 tmp |= DC_HPDx_INT_ACK;
7106 WREG32(DC_HPD3_INT_CONTROL, tmp);
7107 }
7108 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7109 tmp = RREG32(DC_HPD4_INT_CONTROL);
7110 tmp |= DC_HPDx_INT_ACK;
7111 WREG32(DC_HPD4_INT_CONTROL, tmp);
7112 }
7113 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7114 tmp = RREG32(DC_HPD5_INT_CONTROL);
7115 tmp |= DC_HPDx_INT_ACK;
7116 WREG32(DC_HPD5_INT_CONTROL, tmp);
7117 }
7118 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7119 tmp = RREG32(DC_HPD5_INT_CONTROL);
7120 tmp |= DC_HPDx_INT_ACK;
7121 WREG32(DC_HPD6_INT_CONTROL, tmp);
7122 }
7123}
7124
7125/**
7126 * cik_irq_disable - disable interrupts
7127 *
7128 * @rdev: radeon_device pointer
7129 *
7130 * Disable interrupts on the hw (CIK).
7131 */
7132static void cik_irq_disable(struct radeon_device *rdev)
7133{
7134 cik_disable_interrupts(rdev);
7135 /* Wait and acknowledge irq */
7136 mdelay(1);
7137 cik_irq_ack(rdev);
7138 cik_disable_interrupt_state(rdev);
7139}
7140
7141/**
7142 * cik_irq_disable - disable interrupts for suspend
7143 *
7144 * @rdev: radeon_device pointer
7145 *
7146 * Disable interrupts and stop the RLC (CIK).
7147 * Used for suspend.
7148 */
7149static void cik_irq_suspend(struct radeon_device *rdev)
7150{
7151 cik_irq_disable(rdev);
7152 cik_rlc_stop(rdev);
7153}
7154
7155/**
7156 * cik_irq_fini - tear down interrupt support
7157 *
7158 * @rdev: radeon_device pointer
7159 *
7160 * Disable interrupts on the hw and free the IH ring
7161 * buffer (CIK).
7162 * Used for driver unload.
7163 */
7164static void cik_irq_fini(struct radeon_device *rdev)
7165{
7166 cik_irq_suspend(rdev);
7167 r600_ih_ring_fini(rdev);
7168}
7169
7170/**
7171 * cik_get_ih_wptr - get the IH ring buffer wptr
7172 *
7173 * @rdev: radeon_device pointer
7174 *
7175 * Get the IH ring buffer wptr from either the register
7176 * or the writeback memory buffer (CIK). Also check for
7177 * ring buffer overflow and deal with it.
7178 * Used by cik_irq_process().
7179 * Returns the value of the wptr.
7180 */
7181static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7182{
7183 u32 wptr, tmp;
7184
7185 if (rdev->wb.enabled)
7186 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7187 else
7188 wptr = RREG32(IH_RB_WPTR);
7189
7190 if (wptr & RB_OVERFLOW) {
7191 /* When a ring buffer overflow happen start parsing interrupt
7192 * from the last not overwritten vector (wptr + 16). Hopefully
7193 * this should allow us to catchup.
7194 */
7195 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7196 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7197 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7198 tmp = RREG32(IH_RB_CNTL);
7199 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7200 WREG32(IH_RB_CNTL, tmp);
7201 }
7202 return (wptr & rdev->ih.ptr_mask);
7203}
7204
7205/* CIK IV Ring
7206 * Each IV ring entry is 128 bits:
7207 * [7:0] - interrupt source id
7208 * [31:8] - reserved
7209 * [59:32] - interrupt source data
7210 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7211 * [71:64] - RINGID
7212 * CP:
7213 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7214 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7215 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7216 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7217 * PIPE_ID - ME0 0=3D
7218 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7219 * SDMA:
7220 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7221 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7222 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7223 * [79:72] - VMID
7224 * [95:80] - PASID
7225 * [127:96] - reserved
7226 */
7227/**
7228 * cik_irq_process - interrupt handler
7229 *
7230 * @rdev: radeon_device pointer
7231 *
7232 * Interrupt hander (CIK). Walk the IH ring,
7233 * ack interrupts and schedule work to handle
7234 * interrupt events.
7235 * Returns irq process return code.
7236 */
7237int cik_irq_process(struct radeon_device *rdev)
7238{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7239 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7240 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7241 u32 wptr;
7242 u32 rptr;
7243 u32 src_id, src_data, ring_id;
7244 u8 me_id, pipe_id, queue_id;
7245 u32 ring_index;
7246 bool queue_hotplug = false;
7247 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7248 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7249 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7250
7251 if (!rdev->ih.enabled || rdev->shutdown)
7252 return IRQ_NONE;
7253
7254 wptr = cik_get_ih_wptr(rdev);
7255
7256restart_ih:
7257 /* is somebody else already processing irqs? */
7258 if (atomic_xchg(&rdev->ih.lock, 1))
7259 return IRQ_NONE;
7260
7261 rptr = rdev->ih.rptr;
7262 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7263
7264 /* Order reading of wptr vs. reading of IH ring data */
7265 rmb();
7266
7267 /* display interrupts */
7268 cik_irq_ack(rdev);
7269
7270 while (rptr != wptr) {
7271 /* wptr/rptr are in bytes! */
7272 ring_index = rptr / 4;
7273 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7274 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7275 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7276
7277 switch (src_id) {
7278 case 1: /* D1 vblank/vline */
7279 switch (src_data) {
7280 case 0: /* D1 vblank */
7281 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7282 if (rdev->irq.crtc_vblank_int[0]) {
7283 drm_handle_vblank(rdev->ddev, 0);
7284 rdev->pm.vblank_sync = true;
7285 wake_up(&rdev->irq.vblank_queue);
7286 }
7287 if (atomic_read(&rdev->irq.pflip[0]))
7288 radeon_crtc_handle_flip(rdev, 0);
7289 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7290 DRM_DEBUG("IH: D1 vblank\n");
7291 }
7292 break;
7293 case 1: /* D1 vline */
7294 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7295 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7296 DRM_DEBUG("IH: D1 vline\n");
7297 }
7298 break;
7299 default:
7300 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7301 break;
7302 }
7303 break;
7304 case 2: /* D2 vblank/vline */
7305 switch (src_data) {
7306 case 0: /* D2 vblank */
7307 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7308 if (rdev->irq.crtc_vblank_int[1]) {
7309 drm_handle_vblank(rdev->ddev, 1);
7310 rdev->pm.vblank_sync = true;
7311 wake_up(&rdev->irq.vblank_queue);
7312 }
7313 if (atomic_read(&rdev->irq.pflip[1]))
7314 radeon_crtc_handle_flip(rdev, 1);
7315 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7316 DRM_DEBUG("IH: D2 vblank\n");
7317 }
7318 break;
7319 case 1: /* D2 vline */
7320 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7321 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7322 DRM_DEBUG("IH: D2 vline\n");
7323 }
7324 break;
7325 default:
7326 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7327 break;
7328 }
7329 break;
7330 case 3: /* D3 vblank/vline */
7331 switch (src_data) {
7332 case 0: /* D3 vblank */
7333 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7334 if (rdev->irq.crtc_vblank_int[2]) {
7335 drm_handle_vblank(rdev->ddev, 2);
7336 rdev->pm.vblank_sync = true;
7337 wake_up(&rdev->irq.vblank_queue);
7338 }
7339 if (atomic_read(&rdev->irq.pflip[2]))
7340 radeon_crtc_handle_flip(rdev, 2);
7341 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7342 DRM_DEBUG("IH: D3 vblank\n");
7343 }
7344 break;
7345 case 1: /* D3 vline */
7346 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7347 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7348 DRM_DEBUG("IH: D3 vline\n");
7349 }
7350 break;
7351 default:
7352 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7353 break;
7354 }
7355 break;
7356 case 4: /* D4 vblank/vline */
7357 switch (src_data) {
7358 case 0: /* D4 vblank */
7359 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7360 if (rdev->irq.crtc_vblank_int[3]) {
7361 drm_handle_vblank(rdev->ddev, 3);
7362 rdev->pm.vblank_sync = true;
7363 wake_up(&rdev->irq.vblank_queue);
7364 }
7365 if (atomic_read(&rdev->irq.pflip[3]))
7366 radeon_crtc_handle_flip(rdev, 3);
7367 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7368 DRM_DEBUG("IH: D4 vblank\n");
7369 }
7370 break;
7371 case 1: /* D4 vline */
7372 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7373 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7374 DRM_DEBUG("IH: D4 vline\n");
7375 }
7376 break;
7377 default:
7378 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7379 break;
7380 }
7381 break;
7382 case 5: /* D5 vblank/vline */
7383 switch (src_data) {
7384 case 0: /* D5 vblank */
7385 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7386 if (rdev->irq.crtc_vblank_int[4]) {
7387 drm_handle_vblank(rdev->ddev, 4);
7388 rdev->pm.vblank_sync = true;
7389 wake_up(&rdev->irq.vblank_queue);
7390 }
7391 if (atomic_read(&rdev->irq.pflip[4]))
7392 radeon_crtc_handle_flip(rdev, 4);
7393 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7394 DRM_DEBUG("IH: D5 vblank\n");
7395 }
7396 break;
7397 case 1: /* D5 vline */
7398 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7399 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7400 DRM_DEBUG("IH: D5 vline\n");
7401 }
7402 break;
7403 default:
7404 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7405 break;
7406 }
7407 break;
7408 case 6: /* D6 vblank/vline */
7409 switch (src_data) {
7410 case 0: /* D6 vblank */
7411 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7412 if (rdev->irq.crtc_vblank_int[5]) {
7413 drm_handle_vblank(rdev->ddev, 5);
7414 rdev->pm.vblank_sync = true;
7415 wake_up(&rdev->irq.vblank_queue);
7416 }
7417 if (atomic_read(&rdev->irq.pflip[5]))
7418 radeon_crtc_handle_flip(rdev, 5);
7419 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7420 DRM_DEBUG("IH: D6 vblank\n");
7421 }
7422 break;
7423 case 1: /* D6 vline */
7424 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7425 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7426 DRM_DEBUG("IH: D6 vline\n");
7427 }
7428 break;
7429 default:
7430 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7431 break;
7432 }
7433 break;
7434 case 42: /* HPD hotplug */
7435 switch (src_data) {
7436 case 0:
7437 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7438 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7439 queue_hotplug = true;
7440 DRM_DEBUG("IH: HPD1\n");
7441 }
7442 break;
7443 case 1:
7444 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7445 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7446 queue_hotplug = true;
7447 DRM_DEBUG("IH: HPD2\n");
7448 }
7449 break;
7450 case 2:
7451 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7452 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7453 queue_hotplug = true;
7454 DRM_DEBUG("IH: HPD3\n");
7455 }
7456 break;
7457 case 3:
7458 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7459 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7460 queue_hotplug = true;
7461 DRM_DEBUG("IH: HPD4\n");
7462 }
7463 break;
7464 case 4:
7465 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7466 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7467 queue_hotplug = true;
7468 DRM_DEBUG("IH: HPD5\n");
7469 }
7470 break;
7471 case 5:
7472 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7473 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7474 queue_hotplug = true;
7475 DRM_DEBUG("IH: HPD6\n");
7476 }
7477 break;
7478 default:
7479 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7480 break;
7481 }
7482 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7483 case 124: /* UVD */
7484 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7485 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7486 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7487 case 146:
7488 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7489 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7490 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7491 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7492 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7493 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7494 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7495 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7496 status);
7497 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7498 /* reset addr and status */
7499 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7500 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7501 case 176: /* GFX RB CP_INT */
7502 case 177: /* GFX IB CP_INT */
7503 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7504 break;
7505 case 181: /* CP EOP event */
7506 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7507 /* XXX check the bitfield order! */
7508 me_id = (ring_id & 0x60) >> 5;
7509 pipe_id = (ring_id & 0x18) >> 3;
7510 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7511 switch (me_id) {
7512 case 0:
7513 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7514 break;
7515 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7516 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7517 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7518 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7519 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7520 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7521 break;
7522 }
7523 break;
7524 case 184: /* CP Privileged reg access */
7525 DRM_ERROR("Illegal register access in command stream\n");
7526 /* XXX check the bitfield order! */
7527 me_id = (ring_id & 0x60) >> 5;
7528 pipe_id = (ring_id & 0x18) >> 3;
7529 queue_id = (ring_id & 0x7) >> 0;
7530 switch (me_id) {
7531 case 0:
7532 /* This results in a full GPU reset, but all we need to do is soft
7533 * reset the CP for gfx
7534 */
7535 queue_reset = true;
7536 break;
7537 case 1:
7538 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7539 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7540 break;
7541 case 2:
7542 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7543 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7544 break;
7545 }
7546 break;
7547 case 185: /* CP Privileged inst */
7548 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7549 /* XXX check the bitfield order! */
7550 me_id = (ring_id & 0x60) >> 5;
7551 pipe_id = (ring_id & 0x18) >> 3;
7552 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7553 switch (me_id) {
7554 case 0:
7555 /* This results in a full GPU reset, but all we need to do is soft
7556 * reset the CP for gfx
7557 */
7558 queue_reset = true;
7559 break;
7560 case 1:
7561 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7562 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7563 break;
7564 case 2:
7565 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7566 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7567 break;
7568 }
7569 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7570 case 224: /* SDMA trap event */
7571 /* XXX check the bitfield order! */
7572 me_id = (ring_id & 0x3) >> 0;
7573 queue_id = (ring_id & 0xc) >> 2;
7574 DRM_DEBUG("IH: SDMA trap\n");
7575 switch (me_id) {
7576 case 0:
7577 switch (queue_id) {
7578 case 0:
7579 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7580 break;
7581 case 1:
7582 /* XXX compute */
7583 break;
7584 case 2:
7585 /* XXX compute */
7586 break;
7587 }
7588 break;
7589 case 1:
7590 switch (queue_id) {
7591 case 0:
7592 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7593 break;
7594 case 1:
7595 /* XXX compute */
7596 break;
7597 case 2:
7598 /* XXX compute */
7599 break;
7600 }
7601 break;
7602 }
7603 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7604 case 230: /* thermal low to high */
7605 DRM_DEBUG("IH: thermal low to high\n");
7606 rdev->pm.dpm.thermal.high_to_low = false;
7607 queue_thermal = true;
7608 break;
7609 case 231: /* thermal high to low */
7610 DRM_DEBUG("IH: thermal high to low\n");
7611 rdev->pm.dpm.thermal.high_to_low = true;
7612 queue_thermal = true;
7613 break;
7614 case 233: /* GUI IDLE */
7615 DRM_DEBUG("IH: GUI idle\n");
7616 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7617 case 241: /* SDMA Privileged inst */
7618 case 247: /* SDMA Privileged inst */
7619 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7620 /* XXX check the bitfield order! */
7621 me_id = (ring_id & 0x3) >> 0;
7622 queue_id = (ring_id & 0xc) >> 2;
7623 switch (me_id) {
7624 case 0:
7625 switch (queue_id) {
7626 case 0:
7627 queue_reset = true;
7628 break;
7629 case 1:
7630 /* XXX compute */
7631 queue_reset = true;
7632 break;
7633 case 2:
7634 /* XXX compute */
7635 queue_reset = true;
7636 break;
7637 }
7638 break;
7639 case 1:
7640 switch (queue_id) {
7641 case 0:
7642 queue_reset = true;
7643 break;
7644 case 1:
7645 /* XXX compute */
7646 queue_reset = true;
7647 break;
7648 case 2:
7649 /* XXX compute */
7650 queue_reset = true;
7651 break;
7652 }
7653 break;
7654 }
7655 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7656 default:
7657 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7658 break;
7659 }
7660
7661 /* wptr/rptr are in bytes! */
7662 rptr += 16;
7663 rptr &= rdev->ih.ptr_mask;
7664 }
7665 if (queue_hotplug)
7666 schedule_work(&rdev->hotplug_work);
7667 if (queue_reset)
7668 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7669 if (queue_thermal)
7670 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7671 rdev->ih.rptr = rptr;
7672 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7673 atomic_set(&rdev->ih.lock, 0);
7674
7675 /* make sure wptr hasn't changed while processing */
7676 wptr = cik_get_ih_wptr(rdev);
7677 if (wptr != rptr)
7678 goto restart_ih;
7679
7680 return IRQ_HANDLED;
7681}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7682
7683/*
7684 * startup/shutdown callbacks
7685 */
7686/**
7687 * cik_startup - program the asic to a functional state
7688 *
7689 * @rdev: radeon_device pointer
7690 *
7691 * Programs the asic to a functional state (CIK).
7692 * Called by cik_init() and cik_resume().
7693 * Returns 0 for success, error for failure.
7694 */
7695static int cik_startup(struct radeon_device *rdev)
7696{
7697 struct radeon_ring *ring;
7698 int r;
7699
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7700 /* enable pcie gen2/3 link */
7701 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7702 /* enable aspm */
7703 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7704
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7705 /* scratch needs to be initialized before MC */
7706 r = r600_vram_scratch_init(rdev);
7707 if (r)
7708 return r;
7709
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7710 cik_mc_program(rdev);
7711
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7712 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7713 r = ci_mc_load_microcode(rdev);
7714 if (r) {
7715 DRM_ERROR("Failed to load MC firmware!\n");
7716 return r;
7717 }
7718 }
7719
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7720 r = cik_pcie_gart_enable(rdev);
7721 if (r)
7722 return r;
7723 cik_gpu_init(rdev);
7724
7725 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7726 if (rdev->flags & RADEON_IS_IGP) {
7727 if (rdev->family == CHIP_KAVERI) {
7728 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7729 rdev->rlc.reg_list_size =
7730 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7731 } else {
7732 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7733 rdev->rlc.reg_list_size =
7734 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7735 }
7736 }
7737 rdev->rlc.cs_data = ci_cs_data;
7738 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7739 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7740 if (r) {
7741 DRM_ERROR("Failed to init rlc BOs!\n");
7742 return r;
7743 }
7744
7745 /* allocate wb buffer */
7746 r = radeon_wb_init(rdev);
7747 if (r)
7748 return r;
7749
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7750 /* allocate mec buffers */
7751 r = cik_mec_init(rdev);
7752 if (r) {
7753 DRM_ERROR("Failed to init MEC BOs!\n");
7754 return r;
7755 }
7756
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7757 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7758 if (r) {
7759 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7760 return r;
7761 }
7762
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7763 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7764 if (r) {
7765 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7766 return r;
7767 }
7768
7769 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7770 if (r) {
7771 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7772 return r;
7773 }
7774
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7775 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7776 if (r) {
7777 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7778 return r;
7779 }
7780
7781 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7782 if (r) {
7783 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7784 return r;
7785 }
7786
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7787 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7788 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7789 r = uvd_v4_2_resume(rdev);
7790 if (!r) {
7791 r = radeon_fence_driver_start_ring(rdev,
7792 R600_RING_TYPE_UVD_INDEX);
7793 if (r)
7794 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7795 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7796 }
7797 if (r)
7798 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7799
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7800 /* Enable IRQ */
7801 if (!rdev->irq.installed) {
7802 r = radeon_irq_kms_init(rdev);
7803 if (r)
7804 return r;
7805 }
7806
7807 r = cik_irq_init(rdev);
7808 if (r) {
7809 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7810 radeon_irq_kms_fini(rdev);
7811 return r;
7812 }
7813 cik_irq_set(rdev);
7814
7815 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7816 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7817 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7818 if (r)
7819 return r;
7820
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7821 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7822 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7823 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7824 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7825 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7826 if (r)
7827 return r;
7828 ring->me = 1; /* first MEC */
7829 ring->pipe = 0; /* first pipe */
7830 ring->queue = 0; /* first queue */
7831 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7832
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7833 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7834 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7835 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7836 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7837 if (r)
7838 return r;
7839 /* dGPU only have 1 MEC */
7840 ring->me = 1; /* first MEC */
7841 ring->pipe = 0; /* first pipe */
7842 ring->queue = 1; /* second queue */
7843 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7844
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7845 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7846 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7847 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7848 if (r)
7849 return r;
7850
7851 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7852 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7853 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7854 if (r)
7855 return r;
7856
7857 r = cik_cp_resume(rdev);
7858 if (r)
7859 return r;
7860
7861 r = cik_sdma_resume(rdev);
7862 if (r)
7863 return r;
7864
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7865 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7866 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7867 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7868 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7869 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7870 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7871 if (r)
7872 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7873 }
7874
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7875 r = radeon_ib_pool_init(rdev);
7876 if (r) {
7877 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7878 return r;
7879 }
7880
7881 r = radeon_vm_manager_init(rdev);
7882 if (r) {
7883 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7884 return r;
7885 }
7886
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7887 r = dce6_audio_init(rdev);
7888 if (r)
7889 return r;
7890
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7891 return 0;
7892}
7893
7894/**
7895 * cik_resume - resume the asic to a functional state
7896 *
7897 * @rdev: radeon_device pointer
7898 *
7899 * Programs the asic to a functional state (CIK).
7900 * Called at resume.
7901 * Returns 0 for success, error for failure.
7902 */
7903int cik_resume(struct radeon_device *rdev)
7904{
7905 int r;
7906
7907 /* post card */
7908 atom_asic_init(rdev->mode_info.atom_context);
7909
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7910 /* init golden registers */
7911 cik_init_golden_registers(rdev);
7912
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7913 radeon_pm_resume(rdev);
7914
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7915 rdev->accel_working = true;
7916 r = cik_startup(rdev);
7917 if (r) {
7918 DRM_ERROR("cik startup failed on resume\n");
7919 rdev->accel_working = false;
7920 return r;
7921 }
7922
7923 return r;
7924
7925}
7926
7927/**
7928 * cik_suspend - suspend the asic
7929 *
7930 * @rdev: radeon_device pointer
7931 *
7932 * Bring the chip into a state suitable for suspend (CIK).
7933 * Called at suspend.
7934 * Returns 0 for success.
7935 */
7936int cik_suspend(struct radeon_device *rdev)
7937{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7938 radeon_pm_suspend(rdev);
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7939 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7940 radeon_vm_manager_fini(rdev);
7941 cik_cp_enable(rdev, false);
7942 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7943 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7944 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7945 cik_fini_pg(rdev);
7946 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7947 cik_irq_suspend(rdev);
7948 radeon_wb_disable(rdev);
7949 cik_pcie_gart_disable(rdev);
7950 return 0;
7951}
7952
7953/* Plan is to move initialization in that function and use
7954 * helper function so that radeon_device_init pretty much
7955 * do nothing more than calling asic specific function. This
7956 * should also allow to remove a bunch of callback function
7957 * like vram_info.
7958 */
7959/**
7960 * cik_init - asic specific driver and hw init
7961 *
7962 * @rdev: radeon_device pointer
7963 *
7964 * Setup asic specific driver variables and program the hw
7965 * to a functional state (CIK).
7966 * Called at driver startup.
7967 * Returns 0 for success, errors for failure.
7968 */
7969int cik_init(struct radeon_device *rdev)
7970{
7971 struct radeon_ring *ring;
7972 int r;
7973
7974 /* Read BIOS */
7975 if (!radeon_get_bios(rdev)) {
7976 if (ASIC_IS_AVIVO(rdev))
7977 return -EINVAL;
7978 }
7979 /* Must be an ATOMBIOS */
7980 if (!rdev->is_atom_bios) {
7981 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7982 return -EINVAL;
7983 }
7984 r = radeon_atombios_init(rdev);
7985 if (r)
7986 return r;
7987
7988 /* Post card if necessary */
7989 if (!radeon_card_posted(rdev)) {
7990 if (!rdev->bios) {
7991 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7992 return -EINVAL;
7993 }
7994 DRM_INFO("GPU not posted. posting now...\n");
7995 atom_asic_init(rdev->mode_info.atom_context);
7996 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7997 /* init golden registers */
7998 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7999 /* Initialize scratch registers */
8000 cik_scratch_init(rdev);
8001 /* Initialize surface registers */
8002 radeon_surface_init(rdev);
8003 /* Initialize clocks */
8004 radeon_get_clock_info(rdev->ddev);
8005
8006 /* Fence driver */
8007 r = radeon_fence_driver_init(rdev);
8008 if (r)
8009 return r;
8010
8011 /* initialize memory controller */
8012 r = cik_mc_init(rdev);
8013 if (r)
8014 return r;
8015 /* Memory manager */
8016 r = radeon_bo_init(rdev);
8017 if (r)
8018 return r;
8019
Alex Deucher01ac8792013-12-18 19:11:27 -0500[diff] [blame]8020 if (rdev->flags & RADEON_IS_IGP) {
8021 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8022 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8023 r = cik_init_microcode(rdev);
8024 if (r) {
8025 DRM_ERROR("Failed to load firmware!\n");
8026 return r;
8027 }
8028 }
8029 } else {
8030 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8031 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8032 !rdev->mc_fw) {
8033 r = cik_init_microcode(rdev);
8034 if (r) {
8035 DRM_ERROR("Failed to load firmware!\n");
8036 return r;
8037 }
8038 }
8039 }
8040
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8041 /* Initialize power management */
8042 radeon_pm_init(rdev);
8043
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8044 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8045 ring->ring_obj = NULL;
8046 r600_ring_init(rdev, ring, 1024 * 1024);
8047
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8048 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8049 ring->ring_obj = NULL;
8050 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8051 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8052 if (r)
8053 return r;
8054
8055 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8056 ring->ring_obj = NULL;
8057 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8058 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8059 if (r)
8060 return r;
8061
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8062 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8063 ring->ring_obj = NULL;
8064 r600_ring_init(rdev, ring, 256 * 1024);
8065
8066 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8067 ring->ring_obj = NULL;
8068 r600_ring_init(rdev, ring, 256 * 1024);
8069
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8070 r = radeon_uvd_init(rdev);
8071 if (!r) {
8072 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8073 ring->ring_obj = NULL;
8074 r600_ring_init(rdev, ring, 4096);
8075 }
8076
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8077 rdev->ih.ring_obj = NULL;
8078 r600_ih_ring_init(rdev, 64 * 1024);
8079
8080 r = r600_pcie_gart_init(rdev);
8081 if (r)
8082 return r;
8083
8084 rdev->accel_working = true;
8085 r = cik_startup(rdev);
8086 if (r) {
8087 dev_err(rdev->dev, "disabling GPU acceleration\n");
8088 cik_cp_fini(rdev);
8089 cik_sdma_fini(rdev);
8090 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8091 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8092 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8093 radeon_wb_fini(rdev);
8094 radeon_ib_pool_fini(rdev);
8095 radeon_vm_manager_fini(rdev);
8096 radeon_irq_kms_fini(rdev);
8097 cik_pcie_gart_fini(rdev);
8098 rdev->accel_working = false;
8099 }
8100
8101 /* Don't start up if the MC ucode is missing.
8102 * The default clocks and voltages before the MC ucode
8103 * is loaded are not suffient for advanced operations.
8104 */
8105 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8106 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8107 return -EINVAL;
8108 }
8109
8110 return 0;
8111}
8112
8113/**
8114 * cik_fini - asic specific driver and hw fini
8115 *
8116 * @rdev: radeon_device pointer
8117 *
8118 * Tear down the asic specific driver variables and program the hw
8119 * to an idle state (CIK).
8120 * Called at driver unload.
8121 */
8122void cik_fini(struct radeon_device *rdev)
8123{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8124 radeon_pm_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8125 cik_cp_fini(rdev);
8126 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]8127 cik_fini_pg(rdev);
8128 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8129 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8130 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8131 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8132 radeon_wb_fini(rdev);
8133 radeon_vm_manager_fini(rdev);
8134 radeon_ib_pool_fini(rdev);
8135 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]8136 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8137 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8138 cik_pcie_gart_fini(rdev);
8139 r600_vram_scratch_fini(rdev);
8140 radeon_gem_fini(rdev);
8141 radeon_fence_driver_fini(rdev);
8142 radeon_bo_fini(rdev);
8143 radeon_atombios_fini(rdev);
8144 kfree(rdev->bios);
8145 rdev->bios = NULL;
8146}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8147
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8148void dce8_program_fmt(struct drm_encoder *encoder)
8149{
8150 struct drm_device *dev = encoder->dev;
8151 struct radeon_device *rdev = dev->dev_private;
8152 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8153 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8154 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8155 int bpc = 0;
8156 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8157 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8158
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8159 if (connector) {
8160 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8161 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8162 dither = radeon_connector->dither;
8163 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8164
8165 /* LVDS/eDP FMT is set up by atom */
8166 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8167 return;
8168
8169 /* not needed for analog */
8170 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8171 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8172 return;
8173
8174 if (bpc == 0)
8175 return;
8176
8177 switch (bpc) {
8178 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8179 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8180 /* XXX sort out optimal dither settings */
8181 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8182 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8183 else
8184 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8185 break;
8186 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8187 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8188 /* XXX sort out optimal dither settings */
8189 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8190 FMT_RGB_RANDOM_ENABLE |
8191 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8192 else
8193 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8194 break;
8195 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8196 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8197 /* XXX sort out optimal dither settings */
8198 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8199 FMT_RGB_RANDOM_ENABLE |
8200 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8201 else
8202 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8203 break;
8204 default:
8205 /* not needed */
8206 break;
8207 }
8208
8209 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8210}
8211
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8212/* display watermark setup */
8213/**
8214 * dce8_line_buffer_adjust - Set up the line buffer
8215 *
8216 * @rdev: radeon_device pointer
8217 * @radeon_crtc: the selected display controller
8218 * @mode: the current display mode on the selected display
8219 * controller
8220 *
8221 * Setup up the line buffer allocation for
8222 * the selected display controller (CIK).
8223 * Returns the line buffer size in pixels.
8224 */
8225static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8226 struct radeon_crtc *radeon_crtc,
8227 struct drm_display_mode *mode)
8228{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8229 u32 tmp, buffer_alloc, i;
8230 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8231 /*
8232 * Line Buffer Setup
8233 * There are 6 line buffers, one for each display controllers.
8234 * There are 3 partitions per LB. Select the number of partitions
8235 * to enable based on the display width. For display widths larger
8236 * than 4096, you need use to use 2 display controllers and combine
8237 * them using the stereo blender.
8238 */
8239 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8240 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8241 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8242 buffer_alloc = 2;
8243 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8244 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8245 buffer_alloc = 2;
8246 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8247 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8248 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8249 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8250 DRM_DEBUG_KMS("Mode too big for LB!\n");
8251 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8252 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8253 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8254 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8255 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8256 buffer_alloc = 0;
8257 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8258
8259 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8260 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8261
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8262 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8263 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8264 for (i = 0; i < rdev->usec_timeout; i++) {
8265 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8266 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8267 break;
8268 udelay(1);
8269 }
8270
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8271 if (radeon_crtc->base.enabled && mode) {
8272 switch (tmp) {
8273 case 0:
8274 default:
8275 return 4096 * 2;
8276 case 1:
8277 return 1920 * 2;
8278 case 2:
8279 return 2560 * 2;
8280 }
8281 }
8282
8283 /* controller not enabled, so no lb used */
8284 return 0;
8285}
8286
8287/**
8288 * cik_get_number_of_dram_channels - get the number of dram channels
8289 *
8290 * @rdev: radeon_device pointer
8291 *
8292 * Look up the number of video ram channels (CIK).
8293 * Used for display watermark bandwidth calculations
8294 * Returns the number of dram channels
8295 */
8296static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8297{
8298 u32 tmp = RREG32(MC_SHARED_CHMAP);
8299
8300 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8301 case 0:
8302 default:
8303 return 1;
8304 case 1:
8305 return 2;
8306 case 2:
8307 return 4;
8308 case 3:
8309 return 8;
8310 case 4:
8311 return 3;
8312 case 5:
8313 return 6;
8314 case 6:
8315 return 10;
8316 case 7:
8317 return 12;
8318 case 8:
8319 return 16;
8320 }
8321}
8322
8323struct dce8_wm_params {
8324 u32 dram_channels; /* number of dram channels */
8325 u32 yclk; /* bandwidth per dram data pin in kHz */
8326 u32 sclk; /* engine clock in kHz */
8327 u32 disp_clk; /* display clock in kHz */
8328 u32 src_width; /* viewport width */
8329 u32 active_time; /* active display time in ns */
8330 u32 blank_time; /* blank time in ns */
8331 bool interlaced; /* mode is interlaced */
8332 fixed20_12 vsc; /* vertical scale ratio */
8333 u32 num_heads; /* number of active crtcs */
8334 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8335 u32 lb_size; /* line buffer allocated to pipe */
8336 u32 vtaps; /* vertical scaler taps */
8337};
8338
8339/**
8340 * dce8_dram_bandwidth - get the dram bandwidth
8341 *
8342 * @wm: watermark calculation data
8343 *
8344 * Calculate the raw dram bandwidth (CIK).
8345 * Used for display watermark bandwidth calculations
8346 * Returns the dram bandwidth in MBytes/s
8347 */
8348static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8349{
8350 /* Calculate raw DRAM Bandwidth */
8351 fixed20_12 dram_efficiency; /* 0.7 */
8352 fixed20_12 yclk, dram_channels, bandwidth;
8353 fixed20_12 a;
8354
8355 a.full = dfixed_const(1000);
8356 yclk.full = dfixed_const(wm->yclk);
8357 yclk.full = dfixed_div(yclk, a);
8358 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8359 a.full = dfixed_const(10);
8360 dram_efficiency.full = dfixed_const(7);
8361 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8362 bandwidth.full = dfixed_mul(dram_channels, yclk);
8363 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8364
8365 return dfixed_trunc(bandwidth);
8366}
8367
8368/**
8369 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8370 *
8371 * @wm: watermark calculation data
8372 *
8373 * Calculate the dram bandwidth used for display (CIK).
8374 * Used for display watermark bandwidth calculations
8375 * Returns the dram bandwidth for display in MBytes/s
8376 */
8377static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8378{
8379 /* Calculate DRAM Bandwidth and the part allocated to display. */
8380 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8381 fixed20_12 yclk, dram_channels, bandwidth;
8382 fixed20_12 a;
8383
8384 a.full = dfixed_const(1000);
8385 yclk.full = dfixed_const(wm->yclk);
8386 yclk.full = dfixed_div(yclk, a);
8387 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8388 a.full = dfixed_const(10);
8389 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8390 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8391 bandwidth.full = dfixed_mul(dram_channels, yclk);
8392 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8393
8394 return dfixed_trunc(bandwidth);
8395}
8396
8397/**
8398 * dce8_data_return_bandwidth - get the data return bandwidth
8399 *
8400 * @wm: watermark calculation data
8401 *
8402 * Calculate the data return bandwidth used for display (CIK).
8403 * Used for display watermark bandwidth calculations
8404 * Returns the data return bandwidth in MBytes/s
8405 */
8406static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8407{
8408 /* Calculate the display Data return Bandwidth */
8409 fixed20_12 return_efficiency; /* 0.8 */
8410 fixed20_12 sclk, bandwidth;
8411 fixed20_12 a;
8412
8413 a.full = dfixed_const(1000);
8414 sclk.full = dfixed_const(wm->sclk);
8415 sclk.full = dfixed_div(sclk, a);
8416 a.full = dfixed_const(10);
8417 return_efficiency.full = dfixed_const(8);
8418 return_efficiency.full = dfixed_div(return_efficiency, a);
8419 a.full = dfixed_const(32);
8420 bandwidth.full = dfixed_mul(a, sclk);
8421 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8422
8423 return dfixed_trunc(bandwidth);
8424}
8425
8426/**
8427 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8428 *
8429 * @wm: watermark calculation data
8430 *
8431 * Calculate the dmif bandwidth used for display (CIK).
8432 * Used for display watermark bandwidth calculations
8433 * Returns the dmif bandwidth in MBytes/s
8434 */
8435static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8436{
8437 /* Calculate the DMIF Request Bandwidth */
8438 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8439 fixed20_12 disp_clk, bandwidth;
8440 fixed20_12 a, b;
8441
8442 a.full = dfixed_const(1000);
8443 disp_clk.full = dfixed_const(wm->disp_clk);
8444 disp_clk.full = dfixed_div(disp_clk, a);
8445 a.full = dfixed_const(32);
8446 b.full = dfixed_mul(a, disp_clk);
8447
8448 a.full = dfixed_const(10);
8449 disp_clk_request_efficiency.full = dfixed_const(8);
8450 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8451
8452 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8453
8454 return dfixed_trunc(bandwidth);
8455}
8456
8457/**
8458 * dce8_available_bandwidth - get the min available bandwidth
8459 *
8460 * @wm: watermark calculation data
8461 *
8462 * Calculate the min available bandwidth used for display (CIK).
8463 * Used for display watermark bandwidth calculations
8464 * Returns the min available bandwidth in MBytes/s
8465 */
8466static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8467{
8468 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8469 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8470 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8471 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8472
8473 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8474}
8475
8476/**
8477 * dce8_average_bandwidth - get the average available bandwidth
8478 *
8479 * @wm: watermark calculation data
8480 *
8481 * Calculate the average available bandwidth used for display (CIK).
8482 * Used for display watermark bandwidth calculations
8483 * Returns the average available bandwidth in MBytes/s
8484 */
8485static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8486{
8487 /* Calculate the display mode Average Bandwidth
8488 * DisplayMode should contain the source and destination dimensions,
8489 * timing, etc.
8490 */
8491 fixed20_12 bpp;
8492 fixed20_12 line_time;
8493 fixed20_12 src_width;
8494 fixed20_12 bandwidth;
8495 fixed20_12 a;
8496
8497 a.full = dfixed_const(1000);
8498 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8499 line_time.full = dfixed_div(line_time, a);
8500 bpp.full = dfixed_const(wm->bytes_per_pixel);
8501 src_width.full = dfixed_const(wm->src_width);
8502 bandwidth.full = dfixed_mul(src_width, bpp);
8503 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8504 bandwidth.full = dfixed_div(bandwidth, line_time);
8505
8506 return dfixed_trunc(bandwidth);
8507}
8508
8509/**
8510 * dce8_latency_watermark - get the latency watermark
8511 *
8512 * @wm: watermark calculation data
8513 *
8514 * Calculate the latency watermark (CIK).
8515 * Used for display watermark bandwidth calculations
8516 * Returns the latency watermark in ns
8517 */
8518static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8519{
8520 /* First calculate the latency in ns */
8521 u32 mc_latency = 2000; /* 2000 ns. */
8522 u32 available_bandwidth = dce8_available_bandwidth(wm);
8523 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8524 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8525 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8526 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8527 (wm->num_heads * cursor_line_pair_return_time);
8528 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8529 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8530 u32 tmp, dmif_size = 12288;
8531 fixed20_12 a, b, c;
8532
8533 if (wm->num_heads == 0)
8534 return 0;
8535
8536 a.full = dfixed_const(2);
8537 b.full = dfixed_const(1);
8538 if ((wm->vsc.full > a.full) ||
8539 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8540 (wm->vtaps >= 5) ||
8541 ((wm->vsc.full >= a.full) && wm->interlaced))
8542 max_src_lines_per_dst_line = 4;
8543 else
8544 max_src_lines_per_dst_line = 2;
8545
8546 a.full = dfixed_const(available_bandwidth);
8547 b.full = dfixed_const(wm->num_heads);
8548 a.full = dfixed_div(a, b);
8549
8550 b.full = dfixed_const(mc_latency + 512);
8551 c.full = dfixed_const(wm->disp_clk);
8552 b.full = dfixed_div(b, c);
8553
8554 c.full = dfixed_const(dmif_size);
8555 b.full = dfixed_div(c, b);
8556
8557 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8558
8559 b.full = dfixed_const(1000);
8560 c.full = dfixed_const(wm->disp_clk);
8561 b.full = dfixed_div(c, b);
8562 c.full = dfixed_const(wm->bytes_per_pixel);
8563 b.full = dfixed_mul(b, c);
8564
8565 lb_fill_bw = min(tmp, dfixed_trunc(b));
8566
8567 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8568 b.full = dfixed_const(1000);
8569 c.full = dfixed_const(lb_fill_bw);
8570 b.full = dfixed_div(c, b);
8571 a.full = dfixed_div(a, b);
8572 line_fill_time = dfixed_trunc(a);
8573
8574 if (line_fill_time < wm->active_time)
8575 return latency;
8576 else
8577 return latency + (line_fill_time - wm->active_time);
8578
8579}
8580
8581/**
8582 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8583 * average and available dram bandwidth
8584 *
8585 * @wm: watermark calculation data
8586 *
8587 * Check if the display average bandwidth fits in the display
8588 * dram bandwidth (CIK).
8589 * Used for display watermark bandwidth calculations
8590 * Returns true if the display fits, false if not.
8591 */
8592static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8593{
8594 if (dce8_average_bandwidth(wm) <=
8595 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8596 return true;
8597 else
8598 return false;
8599}
8600
8601/**
8602 * dce8_average_bandwidth_vs_available_bandwidth - check
8603 * average and available bandwidth
8604 *
8605 * @wm: watermark calculation data
8606 *
8607 * Check if the display average bandwidth fits in the display
8608 * available bandwidth (CIK).
8609 * Used for display watermark bandwidth calculations
8610 * Returns true if the display fits, false if not.
8611 */
8612static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8613{
8614 if (dce8_average_bandwidth(wm) <=
8615 (dce8_available_bandwidth(wm) / wm->num_heads))
8616 return true;
8617 else
8618 return false;
8619}
8620
8621/**
8622 * dce8_check_latency_hiding - check latency hiding
8623 *
8624 * @wm: watermark calculation data
8625 *
8626 * Check latency hiding (CIK).
8627 * Used for display watermark bandwidth calculations
8628 * Returns true if the display fits, false if not.
8629 */
8630static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8631{
8632 u32 lb_partitions = wm->lb_size / wm->src_width;
8633 u32 line_time = wm->active_time + wm->blank_time;
8634 u32 latency_tolerant_lines;
8635 u32 latency_hiding;
8636 fixed20_12 a;
8637
8638 a.full = dfixed_const(1);
8639 if (wm->vsc.full > a.full)
8640 latency_tolerant_lines = 1;
8641 else {
8642 if (lb_partitions <= (wm->vtaps + 1))
8643 latency_tolerant_lines = 1;
8644 else
8645 latency_tolerant_lines = 2;
8646 }
8647
8648 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8649
8650 if (dce8_latency_watermark(wm) <= latency_hiding)
8651 return true;
8652 else
8653 return false;
8654}
8655
8656/**
8657 * dce8_program_watermarks - program display watermarks
8658 *
8659 * @rdev: radeon_device pointer
8660 * @radeon_crtc: the selected display controller
8661 * @lb_size: line buffer size
8662 * @num_heads: number of display controllers in use
8663 *
8664 * Calculate and program the display watermarks for the
8665 * selected display controller (CIK).
8666 */
8667static void dce8_program_watermarks(struct radeon_device *rdev,
8668 struct radeon_crtc *radeon_crtc,
8669 u32 lb_size, u32 num_heads)
8670{
8671 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8672 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8673 u32 pixel_period;
8674 u32 line_time = 0;
8675 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8676 u32 tmp, wm_mask;
8677
8678 if (radeon_crtc->base.enabled && num_heads && mode) {
8679 pixel_period = 1000000 / (u32)mode->clock;
8680 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8681
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8682 /* watermark for high clocks */
8683 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8684 rdev->pm.dpm_enabled) {
8685 wm_high.yclk =
8686 radeon_dpm_get_mclk(rdev, false) * 10;
8687 wm_high.sclk =
8688 radeon_dpm_get_sclk(rdev, false) * 10;
8689 } else {
8690 wm_high.yclk = rdev->pm.current_mclk * 10;
8691 wm_high.sclk = rdev->pm.current_sclk * 10;
8692 }
8693
8694 wm_high.disp_clk = mode->clock;
8695 wm_high.src_width = mode->crtc_hdisplay;
8696 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8697 wm_high.blank_time = line_time - wm_high.active_time;
8698 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8699 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8700 wm_high.interlaced = true;
8701 wm_high.vsc = radeon_crtc->vsc;
8702 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8703 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8704 wm_high.vtaps = 2;
8705 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8706 wm_high.lb_size = lb_size;
8707 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8708 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8709
8710 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8711 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8712
8713 /* possibly force display priority to high */
8714 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8715 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8716 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8717 !dce8_check_latency_hiding(&wm_high) ||
8718 (rdev->disp_priority == 2)) {
8719 DRM_DEBUG_KMS("force priority to high\n");
8720 }
8721
8722 /* watermark for low clocks */
8723 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8724 rdev->pm.dpm_enabled) {
8725 wm_low.yclk =
8726 radeon_dpm_get_mclk(rdev, true) * 10;
8727 wm_low.sclk =
8728 radeon_dpm_get_sclk(rdev, true) * 10;
8729 } else {
8730 wm_low.yclk = rdev->pm.current_mclk * 10;
8731 wm_low.sclk = rdev->pm.current_sclk * 10;
8732 }
8733
8734 wm_low.disp_clk = mode->clock;
8735 wm_low.src_width = mode->crtc_hdisplay;
8736 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8737 wm_low.blank_time = line_time - wm_low.active_time;
8738 wm_low.interlaced = false;
8739 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8740 wm_low.interlaced = true;
8741 wm_low.vsc = radeon_crtc->vsc;
8742 wm_low.vtaps = 1;
8743 if (radeon_crtc->rmx_type != RMX_OFF)
8744 wm_low.vtaps = 2;
8745 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8746 wm_low.lb_size = lb_size;
8747 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8748 wm_low.num_heads = num_heads;
8749
8750 /* set for low clocks */
8751 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8752
8753 /* possibly force display priority to high */
8754 /* should really do this at mode validation time... */
8755 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8756 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8757 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8758 (rdev->disp_priority == 2)) {
8759 DRM_DEBUG_KMS("force priority to high\n");
8760 }
8761 }
8762
8763 /* select wm A */
8764 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8765 tmp = wm_mask;
8766 tmp &= ~LATENCY_WATERMARK_MASK(3);
8767 tmp |= LATENCY_WATERMARK_MASK(1);
8768 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8769 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8770 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8771 LATENCY_HIGH_WATERMARK(line_time)));
8772 /* select wm B */
8773 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8774 tmp &= ~LATENCY_WATERMARK_MASK(3);
8775 tmp |= LATENCY_WATERMARK_MASK(2);
8776 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8777 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8778 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8779 LATENCY_HIGH_WATERMARK(line_time)));
8780 /* restore original selection */
8781 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8782
8783 /* save values for DPM */
8784 radeon_crtc->line_time = line_time;
8785 radeon_crtc->wm_high = latency_watermark_a;
8786 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8787}
8788
8789/**
8790 * dce8_bandwidth_update - program display watermarks
8791 *
8792 * @rdev: radeon_device pointer
8793 *
8794 * Calculate and program the display watermarks and line
8795 * buffer allocation (CIK).
8796 */
8797void dce8_bandwidth_update(struct radeon_device *rdev)
8798{
8799 struct drm_display_mode *mode = NULL;
8800 u32 num_heads = 0, lb_size;
8801 int i;
8802
8803 radeon_update_display_priority(rdev);
8804
8805 for (i = 0; i < rdev->num_crtc; i++) {
8806 if (rdev->mode_info.crtcs[i]->base.enabled)
8807 num_heads++;
8808 }
8809 for (i = 0; i < rdev->num_crtc; i++) {
8810 mode = &rdev->mode_info.crtcs[i]->base.mode;
8811 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8812 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8813 }
8814}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8815
8816/**
8817 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8818 *
8819 * @rdev: radeon_device pointer
8820 *
8821 * Fetches a GPU clock counter snapshot (SI).
8822 * Returns the 64 bit clock counter snapshot.
8823 */
8824uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8825{
8826 uint64_t clock;
8827
8828 mutex_lock(&rdev->gpu_clock_mutex);
8829 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8830 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8831 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8832 mutex_unlock(&rdev->gpu_clock_mutex);
8833 return clock;
8834}
8835
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8836static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8837 u32 cntl_reg, u32 status_reg)
8838{
8839 int r, i;
8840 struct atom_clock_dividers dividers;
8841 uint32_t tmp;
8842
8843 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8844 clock, false, &dividers);
8845 if (r)
8846 return r;
8847
8848 tmp = RREG32_SMC(cntl_reg);
8849 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8850 tmp |= dividers.post_divider;
8851 WREG32_SMC(cntl_reg, tmp);
8852
8853 for (i = 0; i < 100; i++) {
8854 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8855 break;
8856 mdelay(10);
8857 }
8858 if (i == 100)
8859 return -ETIMEDOUT;
8860
8861 return 0;
8862}
8863
8864int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8865{
8866 int r = 0;
8867
8868 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8869 if (r)
8870 return r;
8871
8872 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8873 return r;
8874}
8875
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8876static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8877{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8878 struct pci_dev *root = rdev->pdev->bus->self;
8879 int bridge_pos, gpu_pos;
8880 u32 speed_cntl, mask, current_data_rate;
8881 int ret, i;
8882 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8883
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8884 if (radeon_pcie_gen2 == 0)
8885 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8886
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8887 if (rdev->flags & RADEON_IS_IGP)
8888 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8889
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8890 if (!(rdev->flags & RADEON_IS_PCIE))
8891 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8892
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8893 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8894 if (ret != 0)
8895 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8896
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8897 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8898 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8899
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8900 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8901 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8902 LC_CURRENT_DATA_RATE_SHIFT;
8903 if (mask & DRM_PCIE_SPEED_80) {
8904 if (current_data_rate == 2) {
8905 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8906 return;
8907 }
8908 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8909 } else if (mask & DRM_PCIE_SPEED_50) {
8910 if (current_data_rate == 1) {
8911 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8912 return;
8913 }
8914 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8915 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8916
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8917 bridge_pos = pci_pcie_cap(root);
8918 if (!bridge_pos)
8919 return;
8920
8921 gpu_pos = pci_pcie_cap(rdev->pdev);
8922 if (!gpu_pos)
8923 return;
8924
8925 if (mask & DRM_PCIE_SPEED_80) {
8926 /* re-try equalization if gen3 is not already enabled */
8927 if (current_data_rate != 2) {
8928 u16 bridge_cfg, gpu_cfg;
8929 u16 bridge_cfg2, gpu_cfg2;
8930 u32 max_lw, current_lw, tmp;
8931
8932 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8933 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8934
8935 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8936 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8937
8938 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8939 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8940
8941 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8942 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8943 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8944
8945 if (current_lw < max_lw) {
8946 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8947 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8948 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8949 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8950 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8951 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8952 }
8953 }
8954
8955 for (i = 0; i < 10; i++) {
8956 /* check status */
8957 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8958 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8959 break;
8960
8961 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8962 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8963
8964 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8965 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8966
8967 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8968 tmp |= LC_SET_QUIESCE;
8969 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8970
8971 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8972 tmp |= LC_REDO_EQ;
8973 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8974
8975 mdelay(100);
8976
8977 /* linkctl */
8978 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8979 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8980 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8981 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8982
8983 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8984 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8985 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8986 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8987
8988 /* linkctl2 */
8989 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8990 tmp16 &= ~((1 << 4) | (7 << 9));
8991 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8992 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8993
8994 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8995 tmp16 &= ~((1 << 4) | (7 << 9));
8996 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8997 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8998
8999 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9000 tmp &= ~LC_SET_QUIESCE;
9001 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9002 }
9003 }
9004 }
9005
9006 /* set the link speed */
9007 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9008 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9009 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9010
9011 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9012 tmp16 &= ~0xf;
9013 if (mask & DRM_PCIE_SPEED_80)
9014 tmp16 |= 3; /* gen3 */
9015 else if (mask & DRM_PCIE_SPEED_50)
9016 tmp16 |= 2; /* gen2 */
9017 else
9018 tmp16 |= 1; /* gen1 */
9019 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9020
9021 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9022 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9023 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9024
9025 for (i = 0; i < rdev->usec_timeout; i++) {
9026 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9027 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9028 break;
9029 udelay(1);
9030 }
9031}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]9032
9033static void cik_program_aspm(struct radeon_device *rdev)
9034{
9035 u32 data, orig;
9036 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9037 bool disable_clkreq = false;
9038
9039 if (radeon_aspm == 0)
9040 return;
9041
9042 /* XXX double check IGPs */
9043 if (rdev->flags & RADEON_IS_IGP)
9044 return;
9045
9046 if (!(rdev->flags & RADEON_IS_PCIE))
9047 return;
9048
9049 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9050 data &= ~LC_XMIT_N_FTS_MASK;
9051 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9052 if (orig != data)
9053 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9054
9055 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9056 data |= LC_GO_TO_RECOVERY;
9057 if (orig != data)
9058 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9059
9060 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9061 data |= P_IGNORE_EDB_ERR;
9062 if (orig != data)
9063 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9064
9065 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9066 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9067 data |= LC_PMI_TO_L1_DIS;
9068 if (!disable_l0s)
9069 data |= LC_L0S_INACTIVITY(7);
9070
9071 if (!disable_l1) {
9072 data |= LC_L1_INACTIVITY(7);
9073 data &= ~LC_PMI_TO_L1_DIS;
9074 if (orig != data)
9075 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9076
9077 if (!disable_plloff_in_l1) {
9078 bool clk_req_support;
9079
9080 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9081 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9082 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9083 if (orig != data)
9084 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9085
9086 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9087 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9088 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9089 if (orig != data)
9090 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9091
9092 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9093 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9094 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9095 if (orig != data)
9096 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9097
9098 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9099 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9100 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9101 if (orig != data)
9102 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9103
9104 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9105 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9106 data |= LC_DYN_LANES_PWR_STATE(3);
9107 if (orig != data)
9108 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9109
9110 if (!disable_clkreq) {
9111 struct pci_dev *root = rdev->pdev->bus->self;
9112 u32 lnkcap;
9113
9114 clk_req_support = false;
9115 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9116 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9117 clk_req_support = true;
9118 } else {
9119 clk_req_support = false;
9120 }
9121
9122 if (clk_req_support) {
9123 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9124 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9125 if (orig != data)
9126 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9127
9128 orig = data = RREG32_SMC(THM_CLK_CNTL);
9129 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9130 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9131 if (orig != data)
9132 WREG32_SMC(THM_CLK_CNTL, data);
9133
9134 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9135 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9136 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9137 if (orig != data)
9138 WREG32_SMC(MISC_CLK_CTRL, data);
9139
9140 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9141 data &= ~BCLK_AS_XCLK;
9142 if (orig != data)
9143 WREG32_SMC(CG_CLKPIN_CNTL, data);
9144
9145 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9146 data &= ~FORCE_BIF_REFCLK_EN;
9147 if (orig != data)
9148 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9149
9150 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9151 data &= ~MPLL_CLKOUT_SEL_MASK;
9152 data |= MPLL_CLKOUT_SEL(4);
9153 if (orig != data)
9154 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9155 }
9156 }
9157 } else {
9158 if (orig != data)
9159 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9160 }
9161
9162 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9163 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9164 if (orig != data)
9165 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9166
9167 if (!disable_l0s) {
9168 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9169 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9170 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9171 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9172 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9173 data &= ~LC_L0S_INACTIVITY_MASK;
9174 if (orig != data)
9175 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9176 }
9177 }
9178 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]9179}