Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 0279ed19bd962434d334f5eeb16d14fdd9459a00 / . / drivers / gpu / drm / radeon / cik.c
blob: 9e50dd5d0e4212755240328729fb2164ae454a1d [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]78static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]80static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]87
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]124
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]130 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]131 u32 r;
132
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]151}
152
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1532 default:
1533 break;
1534 }
1535}
1536
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1563 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1564 *
1565 * Returns the value in the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1566 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1567 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1569{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1572 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1582 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1586 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1587 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1589{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1592 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1594 }
1595}
1596
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]1700int ci_mc_load_microcode(struct radeon_device *rdev)
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1802 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1904 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1925 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1932 }
1933
1934out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1952 }
1953 return err;
1954}
1955
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1994 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1995
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2432 }
2433 } else if (num_pipe_configs == 4) {
2434 if (num_rbs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2437 case 0:
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2442 break;
2443 case 1:
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2448 break;
2449 case 2:
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2454 break;
2455 case 3:
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2460 break;
2461 case 4:
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2466 break;
2467 case 5:
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2470 break;
2471 case 6:
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2476 break;
2477 case 7:
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2482 break;
2483 case 8:
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2486 break;
2487 case 9:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2490 break;
2491 case 10:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2496 break;
2497 case 11:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2502 break;
2503 case 12:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2508 break;
2509 case 13:
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2512 break;
2513 case 14:
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518 break;
2519 case 16:
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2524 break;
2525 case 17:
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 break;
2531 case 27:
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2534 break;
2535 case 28:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 29:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 30:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 default:
2554 gb_tile_moden = 0;
2555 break;
2556 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2559 }
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2563 case 0:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2568 break;
2569 case 1:
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2574 break;
2575 case 2:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 break;
2581 case 3:
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2586 break;
2587 case 4:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2592 break;
2593 case 5:
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2596 break;
2597 case 6:
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2602 break;
2603 case 7:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2608 break;
2609 case 8:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2612 break;
2613 case 9:
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2616 break;
2617 case 10:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 11:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 12:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 13:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2638 break;
2639 case 14:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 16:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2650 break;
2651 case 17:
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2656 break;
2657 case 27:
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2660 break;
2661 case 28:
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2666 break;
2667 case 29:
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 break;
2673 case 30:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 default:
2680 gb_tile_moden = 0;
2681 break;
2682 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2685 }
2686 }
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2689 case 0:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2694 break;
2695 case 1:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 2:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 3:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 4:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 5:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 6:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 8:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 9:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 10:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 11:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2754 break;
2755 case 12:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 break;
2761 case 13:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2766 break;
2767 case 14:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2772 break;
2773 default:
2774 gb_tile_moden = 0;
2775 break;
2776 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779 }
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2783 case 0:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2788 break;
2789 case 1:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2794 break;
2795 case 2:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 3:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2806 break;
2807 case 4:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2812 break;
2813 case 5:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2816 break;
2817 case 6:
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2822 break;
2823 case 7:
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2828 break;
2829 case 8:
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2831 break;
2832 case 9:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2835 break;
2836 case 10:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 11:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 12:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 13:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2857 break;
2858 case 14:
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 break;
2864 case 16:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 17:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 27:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2879 break;
2880 case 28:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 29:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 30:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 break;
2898 default:
2899 gb_tile_moden = 0;
2900 break;
2901 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2904 }
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2907 case 0:
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2912 break;
2913 case 1:
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2918 break;
2919 case 2:
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2924 break;
2925 case 3:
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2930 break;
2931 case 4:
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2936 break;
2937 case 5:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 6:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2948 break;
2949 case 8:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 9:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 10:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 11:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 12:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2978 break;
2979 case 13:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 14:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2990 break;
2991 default:
2992 gb_tile_moden = 0;
2993 break;
2994 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2997 }
2998 } else
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3000}
3001
3002/**
3003 * cik_select_se_sh - select which SE, SH to address
3004 *
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3008 *
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3012 */
3013static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3015{
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3017
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3024 else
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3027}
3028
3029/**
3030 * cik_create_bitmask - create a bitmask
3031 *
3032 * @bit_width: length of the mask
3033 *
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3036 */
3037static u32 cik_create_bitmask(u32 bit_width)
3038{
3039 u32 i, mask = 0;
3040
3041 for (i = 0; i < bit_width; i++) {
3042 mask <<= 1;
3043 mask |= 1;
3044 }
3045 return mask;
3046}
3047
3048/**
3049 * cik_select_se_sh - select which SE, SH to address
3050 *
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3055 *
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3058 */
3059static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3060 u32 max_rb_num, u32 se_num,
3061 u32 sh_per_se)
3062{
3063 u32 data, mask;
3064
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3066 if (data & 1)
3067 data &= BACKEND_DISABLE_MASK;
3068 else
3069 data = 0;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3071
3072 data >>= BACKEND_DISABLE_SHIFT;
3073
3074 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3075
3076 return data & mask;
3077}
3078
3079/**
3080 * cik_setup_rb - setup the RBs on the asic
3081 *
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3086 *
3087 * Configures per-SE/SH RB registers (CIK).
3088 */
3089static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
3091 u32 max_rb_num)
3092{
3093 int i, j;
3094 u32 data, mask;
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3097
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
3101 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3104 else
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3106 }
3107 }
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3109
3110 mask = 1;
3111 for (i = 0; i < max_rb_num; i++) {
3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3114 mask <<= 1;
3115 }
3116
3117 for (i = 0; i < se_num; i++) {
3118 cik_select_se_sh(rdev, i, 0xffffffff);
3119 data = 0;
3120 for (j = 0; j < sh_per_se; j++) {
3121 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3122 case 0:
3123 if (j == 0)
3124 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3125 else
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3127 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3128 case 1:
3129 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3130 break;
3131 case 2:
3132 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3133 break;
3134 case 3:
3135 default:
3136 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3137 break;
3138 }
3139 enabled_rbs >>= 2;
3140 }
3141 WREG32(PA_SC_RASTER_CONFIG, data);
3142 }
3143 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3144}
3145
3146/**
3147 * cik_gpu_init - setup the 3D engine
3148 *
3149 * @rdev: radeon_device pointer
3150 *
3151 * Configures the 3D engine and tiling configuration
3152 * registers so that the 3D engine is usable.
3153 */
3154static void cik_gpu_init(struct radeon_device *rdev)
3155{
3156 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3157 u32 mc_shared_chmap, mc_arb_ramcfg;
3158 u32 hdp_host_path_cntl;
3159 u32 tmp;
3160 int i, j;
3161
3162 switch (rdev->family) {
3163 case CHIP_BONAIRE:
3164 rdev->config.cik.max_shader_engines = 2;
3165 rdev->config.cik.max_tile_pipes = 4;
3166 rdev->config.cik.max_cu_per_sh = 7;
3167 rdev->config.cik.max_sh_per_se = 1;
3168 rdev->config.cik.max_backends_per_se = 2;
3169 rdev->config.cik.max_texture_channel_caches = 4;
3170 rdev->config.cik.max_gprs = 256;
3171 rdev->config.cik.max_gs_threads = 32;
3172 rdev->config.cik.max_hw_contexts = 8;
3173
3174 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3175 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3176 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3177 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3178 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3179 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3180 case CHIP_HAWAII:
3181 rdev->config.cik.max_shader_engines = 4;
3182 rdev->config.cik.max_tile_pipes = 16;
3183 rdev->config.cik.max_cu_per_sh = 11;
3184 rdev->config.cik.max_sh_per_se = 1;
3185 rdev->config.cik.max_backends_per_se = 4;
3186 rdev->config.cik.max_texture_channel_caches = 16;
3187 rdev->config.cik.max_gprs = 256;
3188 rdev->config.cik.max_gs_threads = 32;
3189 rdev->config.cik.max_hw_contexts = 8;
3190
3191 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3192 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3193 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3194 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3195 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3196 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3197 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3198 rdev->config.cik.max_shader_engines = 1;
3199 rdev->config.cik.max_tile_pipes = 4;
3200 if ((rdev->pdev->device == 0x1304) ||
3201 (rdev->pdev->device == 0x1305) ||
3202 (rdev->pdev->device == 0x130C) ||
3203 (rdev->pdev->device == 0x130F) ||
3204 (rdev->pdev->device == 0x1310) ||
3205 (rdev->pdev->device == 0x1311) ||
3206 (rdev->pdev->device == 0x131C)) {
3207 rdev->config.cik.max_cu_per_sh = 8;
3208 rdev->config.cik.max_backends_per_se = 2;
3209 } else if ((rdev->pdev->device == 0x1309) ||
3210 (rdev->pdev->device == 0x130A) ||
3211 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3212 (rdev->pdev->device == 0x1313) ||
3213 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3214 rdev->config.cik.max_cu_per_sh = 6;
3215 rdev->config.cik.max_backends_per_se = 2;
3216 } else if ((rdev->pdev->device == 0x1306) ||
3217 (rdev->pdev->device == 0x1307) ||
3218 (rdev->pdev->device == 0x130B) ||
3219 (rdev->pdev->device == 0x130E) ||
3220 (rdev->pdev->device == 0x1315) ||
3221 (rdev->pdev->device == 0x131B)) {
3222 rdev->config.cik.max_cu_per_sh = 4;
3223 rdev->config.cik.max_backends_per_se = 1;
3224 } else {
3225 rdev->config.cik.max_cu_per_sh = 3;
3226 rdev->config.cik.max_backends_per_se = 1;
3227 }
3228 rdev->config.cik.max_sh_per_se = 1;
3229 rdev->config.cik.max_texture_channel_caches = 4;
3230 rdev->config.cik.max_gprs = 256;
3231 rdev->config.cik.max_gs_threads = 16;
3232 rdev->config.cik.max_hw_contexts = 8;
3233
3234 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3235 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3236 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3237 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3238 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3239 break;
3240 case CHIP_KABINI:
3241 default:
3242 rdev->config.cik.max_shader_engines = 1;
3243 rdev->config.cik.max_tile_pipes = 2;
3244 rdev->config.cik.max_cu_per_sh = 2;
3245 rdev->config.cik.max_sh_per_se = 1;
3246 rdev->config.cik.max_backends_per_se = 1;
3247 rdev->config.cik.max_texture_channel_caches = 2;
3248 rdev->config.cik.max_gprs = 256;
3249 rdev->config.cik.max_gs_threads = 16;
3250 rdev->config.cik.max_hw_contexts = 8;
3251
3252 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3253 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3254 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3255 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3256 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3257 break;
3258 }
3259
3260 /* Initialize HDP */
3261 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3262 WREG32((0x2c14 + j), 0x00000000);
3263 WREG32((0x2c18 + j), 0x00000000);
3264 WREG32((0x2c1c + j), 0x00000000);
3265 WREG32((0x2c20 + j), 0x00000000);
3266 WREG32((0x2c24 + j), 0x00000000);
3267 }
3268
3269 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3270
3271 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3272
3273 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3274 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3275
3276 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3277 rdev->config.cik.mem_max_burst_length_bytes = 256;
3278 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3279 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3280 if (rdev->config.cik.mem_row_size_in_kb > 4)
3281 rdev->config.cik.mem_row_size_in_kb = 4;
3282 /* XXX use MC settings? */
3283 rdev->config.cik.shader_engine_tile_size = 32;
3284 rdev->config.cik.num_gpus = 1;
3285 rdev->config.cik.multi_gpu_tile_size = 64;
3286
3287 /* fix up row size */
3288 gb_addr_config &= ~ROW_SIZE_MASK;
3289 switch (rdev->config.cik.mem_row_size_in_kb) {
3290 case 1:
3291 default:
3292 gb_addr_config |= ROW_SIZE(0);
3293 break;
3294 case 2:
3295 gb_addr_config |= ROW_SIZE(1);
3296 break;
3297 case 4:
3298 gb_addr_config |= ROW_SIZE(2);
3299 break;
3300 }
3301
3302 /* setup tiling info dword. gb_addr_config is not adequate since it does
3303 * not have bank info, so create a custom tiling dword.
3304 * bits 3:0 num_pipes
3305 * bits 7:4 num_banks
3306 * bits 11:8 group_size
3307 * bits 15:12 row_size
3308 */
3309 rdev->config.cik.tile_config = 0;
3310 switch (rdev->config.cik.num_tile_pipes) {
3311 case 1:
3312 rdev->config.cik.tile_config |= (0 << 0);
3313 break;
3314 case 2:
3315 rdev->config.cik.tile_config |= (1 << 0);
3316 break;
3317 case 4:
3318 rdev->config.cik.tile_config |= (2 << 0);
3319 break;
3320 case 8:
3321 default:
3322 /* XXX what about 12? */
3323 rdev->config.cik.tile_config |= (3 << 0);
3324 break;
3325 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3326 rdev->config.cik.tile_config |=
3327 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3328 rdev->config.cik.tile_config |=
3329 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3332
3333 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3334 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3335 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3336 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3337 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3338 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3339 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3340 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3341
3342 cik_tiling_mode_table_init(rdev);
3343
3344 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3345 rdev->config.cik.max_sh_per_se,
3346 rdev->config.cik.max_backends_per_se);
3347
3348 /* set HW defaults for 3D engine */
3349 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3350
3351 WREG32(SX_DEBUG_1, 0x20);
3352
3353 WREG32(TA_CNTL_AUX, 0x00010000);
3354
3355 tmp = RREG32(SPI_CONFIG_CNTL);
3356 tmp |= 0x03000000;
3357 WREG32(SPI_CONFIG_CNTL, tmp);
3358
3359 WREG32(SQ_CONFIG, 1);
3360
3361 WREG32(DB_DEBUG, 0);
3362
3363 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3364 tmp |= 0x00000400;
3365 WREG32(DB_DEBUG2, tmp);
3366
3367 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3368 tmp |= 0x00020200;
3369 WREG32(DB_DEBUG3, tmp);
3370
3371 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3372 tmp |= 0x00018208;
3373 WREG32(CB_HW_CONTROL, tmp);
3374
3375 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3376
3377 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3378 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3379 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3380 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3381
3382 WREG32(VGT_NUM_INSTANCES, 1);
3383
3384 WREG32(CP_PERFMON_CNTL, 0);
3385
3386 WREG32(SQ_CONFIG, 0);
3387
3388 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3389 FORCE_EOV_MAX_REZ_CNT(255)));
3390
3391 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3392 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3393
3394 WREG32(VGT_GS_VERTEX_REUSE, 16);
3395 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3396
3397 tmp = RREG32(HDP_MISC_CNTL);
3398 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3399 WREG32(HDP_MISC_CNTL, tmp);
3400
3401 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3402 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3403
3404 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3405 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3406
3407 udelay(50);
3408}
3409
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3410/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3411 * GPU scratch registers helpers function.
3412 */
3413/**
3414 * cik_scratch_init - setup driver info for CP scratch regs
3415 *
3416 * @rdev: radeon_device pointer
3417 *
3418 * Set up the number and offset of the CP scratch registers.
3419 * NOTE: use of CP scratch registers is a legacy inferface and
3420 * is not used by default on newer asics (r6xx+). On newer asics,
3421 * memory buffers are used for fences rather than scratch regs.
3422 */
3423static void cik_scratch_init(struct radeon_device *rdev)
3424{
3425 int i;
3426
3427 rdev->scratch.num_reg = 7;
3428 rdev->scratch.reg_base = SCRATCH_REG0;
3429 for (i = 0; i < rdev->scratch.num_reg; i++) {
3430 rdev->scratch.free[i] = true;
3431 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3432 }
3433}
3434
3435/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3436 * cik_ring_test - basic gfx ring test
3437 *
3438 * @rdev: radeon_device pointer
3439 * @ring: radeon_ring structure holding ring information
3440 *
3441 * Allocate a scratch register and write to it using the gfx ring (CIK).
3442 * Provides a basic gfx ring test to verify that the ring is working.
3443 * Used by cik_cp_gfx_resume();
3444 * Returns 0 on success, error on failure.
3445 */
3446int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3447{
3448 uint32_t scratch;
3449 uint32_t tmp = 0;
3450 unsigned i;
3451 int r;
3452
3453 r = radeon_scratch_get(rdev, &scratch);
3454 if (r) {
3455 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3456 return r;
3457 }
3458 WREG32(scratch, 0xCAFEDEAD);
3459 r = radeon_ring_lock(rdev, ring, 3);
3460 if (r) {
3461 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3462 radeon_scratch_free(rdev, scratch);
3463 return r;
3464 }
3465 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3466 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3467 radeon_ring_write(ring, 0xDEADBEEF);
3468 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3469
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3470 for (i = 0; i < rdev->usec_timeout; i++) {
3471 tmp = RREG32(scratch);
3472 if (tmp == 0xDEADBEEF)
3473 break;
3474 DRM_UDELAY(1);
3475 }
3476 if (i < rdev->usec_timeout) {
3477 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3478 } else {
3479 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3480 ring->idx, scratch, tmp);
3481 r = -EINVAL;
3482 }
3483 radeon_scratch_free(rdev, scratch);
3484 return r;
3485}
3486
3487/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3488 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3489 *
3490 * @rdev: radeon_device pointer
3491 * @fence: radeon fence object
3492 *
3493 * Emits a fence sequnce number on the gfx ring and flushes
3494 * GPU caches.
3495 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3496void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3497 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3498{
3499 struct radeon_ring *ring = &rdev->ring[fence->ring];
3500 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3501
3502 /* EVENT_WRITE_EOP - flush caches, send int */
3503 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3504 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3505 EOP_TC_ACTION_EN |
3506 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3507 EVENT_INDEX(5)));
3508 radeon_ring_write(ring, addr & 0xfffffffc);
3509 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3510 radeon_ring_write(ring, fence->seq);
3511 radeon_ring_write(ring, 0);
3512 /* HDP flush */
3513 /* We should be using the new WAIT_REG_MEM special op packet here
3514 * but it causes the CP to hang
3515 */
3516 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3517 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3518 WRITE_DATA_DST_SEL(0)));
3519 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3520 radeon_ring_write(ring, 0);
3521 radeon_ring_write(ring, 0);
3522}
3523
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3524/**
3525 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3526 *
3527 * @rdev: radeon_device pointer
3528 * @fence: radeon fence object
3529 *
3530 * Emits a fence sequnce number on the compute ring and flushes
3531 * GPU caches.
3532 */
3533void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3534 struct radeon_fence *fence)
3535{
3536 struct radeon_ring *ring = &rdev->ring[fence->ring];
3537 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3538
3539 /* RELEASE_MEM - flush caches, send int */
3540 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3541 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3542 EOP_TC_ACTION_EN |
3543 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3544 EVENT_INDEX(5)));
3545 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3546 radeon_ring_write(ring, addr & 0xfffffffc);
3547 radeon_ring_write(ring, upper_32_bits(addr));
3548 radeon_ring_write(ring, fence->seq);
3549 radeon_ring_write(ring, 0);
3550 /* HDP flush */
3551 /* We should be using the new WAIT_REG_MEM special op packet here
3552 * but it causes the CP to hang
3553 */
3554 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3555 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3556 WRITE_DATA_DST_SEL(0)));
3557 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3558 radeon_ring_write(ring, 0);
3559 radeon_ring_write(ring, 0);
3560}
3561
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3562bool cik_semaphore_ring_emit(struct radeon_device *rdev,
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3563 struct radeon_ring *ring,
3564 struct radeon_semaphore *semaphore,
3565 bool emit_wait)
3566{
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3567/* TODO: figure out why semaphore cause lockups */
3568#if 0
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3569 uint64_t addr = semaphore->gpu_addr;
3570 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3571
3572 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3573 radeon_ring_write(ring, addr & 0xffffffff);
3574 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3575
3576 return true;
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3577#else
3578 return false;
3579#endif
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3580}
3581
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3582/**
3583 * cik_copy_cpdma - copy pages using the CP DMA engine
3584 *
3585 * @rdev: radeon_device pointer
3586 * @src_offset: src GPU address
3587 * @dst_offset: dst GPU address
3588 * @num_gpu_pages: number of GPU pages to xfer
3589 * @fence: radeon fence object
3590 *
3591 * Copy GPU paging using the CP DMA engine (CIK+).
3592 * Used by the radeon ttm implementation to move pages if
3593 * registered as the asic copy callback.
3594 */
3595int cik_copy_cpdma(struct radeon_device *rdev,
3596 uint64_t src_offset, uint64_t dst_offset,
3597 unsigned num_gpu_pages,
3598 struct radeon_fence **fence)
3599{
3600 struct radeon_semaphore *sem = NULL;
3601 int ring_index = rdev->asic->copy.blit_ring_index;
3602 struct radeon_ring *ring = &rdev->ring[ring_index];
3603 u32 size_in_bytes, cur_size_in_bytes, control;
3604 int i, num_loops;
3605 int r = 0;
3606
3607 r = radeon_semaphore_create(rdev, &sem);
3608 if (r) {
3609 DRM_ERROR("radeon: moving bo (%d).\n", r);
3610 return r;
3611 }
3612
3613 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3614 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3615 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3616 if (r) {
3617 DRM_ERROR("radeon: moving bo (%d).\n", r);
3618 radeon_semaphore_free(rdev, &sem, NULL);
3619 return r;
3620 }
3621
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3622 radeon_semaphore_sync_to(sem, *fence);
3623 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3624
3625 for (i = 0; i < num_loops; i++) {
3626 cur_size_in_bytes = size_in_bytes;
3627 if (cur_size_in_bytes > 0x1fffff)
3628 cur_size_in_bytes = 0x1fffff;
3629 size_in_bytes -= cur_size_in_bytes;
3630 control = 0;
3631 if (size_in_bytes == 0)
3632 control |= PACKET3_DMA_DATA_CP_SYNC;
3633 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3634 radeon_ring_write(ring, control);
3635 radeon_ring_write(ring, lower_32_bits(src_offset));
3636 radeon_ring_write(ring, upper_32_bits(src_offset));
3637 radeon_ring_write(ring, lower_32_bits(dst_offset));
3638 radeon_ring_write(ring, upper_32_bits(dst_offset));
3639 radeon_ring_write(ring, cur_size_in_bytes);
3640 src_offset += cur_size_in_bytes;
3641 dst_offset += cur_size_in_bytes;
3642 }
3643
3644 r = radeon_fence_emit(rdev, fence, ring->idx);
3645 if (r) {
3646 radeon_ring_unlock_undo(rdev, ring);
3647 return r;
3648 }
3649
3650 radeon_ring_unlock_commit(rdev, ring);
3651 radeon_semaphore_free(rdev, &sem, *fence);
3652
3653 return r;
3654}
3655
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3656/*
3657 * IB stuff
3658 */
3659/**
3660 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3661 *
3662 * @rdev: radeon_device pointer
3663 * @ib: radeon indirect buffer object
3664 *
3665 * Emits an DE (drawing engine) or CE (constant engine) IB
3666 * on the gfx ring. IBs are usually generated by userspace
3667 * acceleration drivers and submitted to the kernel for
3668 * sheduling on the ring. This function schedules the IB
3669 * on the gfx ring for execution by the GPU.
3670 */
3671void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3672{
3673 struct radeon_ring *ring = &rdev->ring[ib->ring];
3674 u32 header, control = INDIRECT_BUFFER_VALID;
3675
3676 if (ib->is_const_ib) {
3677 /* set switch buffer packet before const IB */
3678 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3679 radeon_ring_write(ring, 0);
3680
3681 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3682 } else {
3683 u32 next_rptr;
3684 if (ring->rptr_save_reg) {
3685 next_rptr = ring->wptr + 3 + 4;
3686 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3687 radeon_ring_write(ring, ((ring->rptr_save_reg -
3688 PACKET3_SET_UCONFIG_REG_START) >> 2));
3689 radeon_ring_write(ring, next_rptr);
3690 } else if (rdev->wb.enabled) {
3691 next_rptr = ring->wptr + 5 + 4;
3692 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3693 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3694 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3695 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3696 radeon_ring_write(ring, next_rptr);
3697 }
3698
3699 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3700 }
3701
3702 control |= ib->length_dw |
3703 (ib->vm ? (ib->vm->id << 24) : 0);
3704
3705 radeon_ring_write(ring, header);
3706 radeon_ring_write(ring,
3707#ifdef __BIG_ENDIAN
3708 (2 << 0) |
3709#endif
3710 (ib->gpu_addr & 0xFFFFFFFC));
3711 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3712 radeon_ring_write(ring, control);
3713}
3714
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3715/**
3716 * cik_ib_test - basic gfx ring IB test
3717 *
3718 * @rdev: radeon_device pointer
3719 * @ring: radeon_ring structure holding ring information
3720 *
3721 * Allocate an IB and execute it on the gfx ring (CIK).
3722 * Provides a basic gfx ring test to verify that IBs are working.
3723 * Returns 0 on success, error on failure.
3724 */
3725int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3726{
3727 struct radeon_ib ib;
3728 uint32_t scratch;
3729 uint32_t tmp = 0;
3730 unsigned i;
3731 int r;
3732
3733 r = radeon_scratch_get(rdev, &scratch);
3734 if (r) {
3735 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3736 return r;
3737 }
3738 WREG32(scratch, 0xCAFEDEAD);
3739 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3740 if (r) {
3741 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3742 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3743 return r;
3744 }
3745 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3746 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3747 ib.ptr[2] = 0xDEADBEEF;
3748 ib.length_dw = 3;
3749 r = radeon_ib_schedule(rdev, &ib, NULL);
3750 if (r) {
3751 radeon_scratch_free(rdev, scratch);
3752 radeon_ib_free(rdev, &ib);
3753 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3754 return r;
3755 }
3756 r = radeon_fence_wait(ib.fence, false);
3757 if (r) {
3758 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3759 radeon_scratch_free(rdev, scratch);
3760 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3761 return r;
3762 }
3763 for (i = 0; i < rdev->usec_timeout; i++) {
3764 tmp = RREG32(scratch);
3765 if (tmp == 0xDEADBEEF)
3766 break;
3767 DRM_UDELAY(1);
3768 }
3769 if (i < rdev->usec_timeout) {
3770 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3771 } else {
3772 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3773 scratch, tmp);
3774 r = -EINVAL;
3775 }
3776 radeon_scratch_free(rdev, scratch);
3777 radeon_ib_free(rdev, &ib);
3778 return r;
3779}
3780
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3781/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3782 * CP.
3783 * On CIK, gfx and compute now have independant command processors.
3784 *
3785 * GFX
3786 * Gfx consists of a single ring and can process both gfx jobs and
3787 * compute jobs. The gfx CP consists of three microengines (ME):
3788 * PFP - Pre-Fetch Parser
3789 * ME - Micro Engine
3790 * CE - Constant Engine
3791 * The PFP and ME make up what is considered the Drawing Engine (DE).
3792 * The CE is an asynchronous engine used for updating buffer desciptors
3793 * used by the DE so that they can be loaded into cache in parallel
3794 * while the DE is processing state update packets.
3795 *
3796 * Compute
3797 * The compute CP consists of two microengines (ME):
3798 * MEC1 - Compute MicroEngine 1
3799 * MEC2 - Compute MicroEngine 2
3800 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3801 * The queues are exposed to userspace and are programmed directly
3802 * by the compute runtime.
3803 */
3804/**
3805 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3806 *
3807 * @rdev: radeon_device pointer
3808 * @enable: enable or disable the MEs
3809 *
3810 * Halts or unhalts the gfx MEs.
3811 */
3812static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3813{
3814 if (enable)
3815 WREG32(CP_ME_CNTL, 0);
3816 else {
3817 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3818 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3819 }
3820 udelay(50);
3821}
3822
3823/**
3824 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3825 *
3826 * @rdev: radeon_device pointer
3827 *
3828 * Loads the gfx PFP, ME, and CE ucode.
3829 * Returns 0 for success, -EINVAL if the ucode is not available.
3830 */
3831static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3832{
3833 const __be32 *fw_data;
3834 int i;
3835
3836 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3837 return -EINVAL;
3838
3839 cik_cp_gfx_enable(rdev, false);
3840
3841 /* PFP */
3842 fw_data = (const __be32 *)rdev->pfp_fw->data;
3843 WREG32(CP_PFP_UCODE_ADDR, 0);
3844 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3845 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3846 WREG32(CP_PFP_UCODE_ADDR, 0);
3847
3848 /* CE */
3849 fw_data = (const __be32 *)rdev->ce_fw->data;
3850 WREG32(CP_CE_UCODE_ADDR, 0);
3851 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3852 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3853 WREG32(CP_CE_UCODE_ADDR, 0);
3854
3855 /* ME */
3856 fw_data = (const __be32 *)rdev->me_fw->data;
3857 WREG32(CP_ME_RAM_WADDR, 0);
3858 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3859 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3860 WREG32(CP_ME_RAM_WADDR, 0);
3861
3862 WREG32(CP_PFP_UCODE_ADDR, 0);
3863 WREG32(CP_CE_UCODE_ADDR, 0);
3864 WREG32(CP_ME_RAM_WADDR, 0);
3865 WREG32(CP_ME_RAM_RADDR, 0);
3866 return 0;
3867}
3868
3869/**
3870 * cik_cp_gfx_start - start the gfx ring
3871 *
3872 * @rdev: radeon_device pointer
3873 *
3874 * Enables the ring and loads the clear state context and other
3875 * packets required to init the ring.
3876 * Returns 0 for success, error for failure.
3877 */
3878static int cik_cp_gfx_start(struct radeon_device *rdev)
3879{
3880 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3881 int r, i;
3882
3883 /* init the CP */
3884 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3885 WREG32(CP_ENDIAN_SWAP, 0);
3886 WREG32(CP_DEVICE_ID, 1);
3887
3888 cik_cp_gfx_enable(rdev, true);
3889
3890 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3891 if (r) {
3892 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3893 return r;
3894 }
3895
3896 /* init the CE partitions. CE only used for gfx on CIK */
3897 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3898 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3899 radeon_ring_write(ring, 0xc000);
3900 radeon_ring_write(ring, 0xc000);
3901
3902 /* setup clear context state */
3903 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3904 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3905
3906 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3907 radeon_ring_write(ring, 0x80000000);
3908 radeon_ring_write(ring, 0x80000000);
3909
3910 for (i = 0; i < cik_default_size; i++)
3911 radeon_ring_write(ring, cik_default_state[i]);
3912
3913 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3914 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3915
3916 /* set clear context state */
3917 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3918 radeon_ring_write(ring, 0);
3919
3920 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3921 radeon_ring_write(ring, 0x00000316);
3922 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3923 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3924
3925 radeon_ring_unlock_commit(rdev, ring);
3926
3927 return 0;
3928}
3929
3930/**
3931 * cik_cp_gfx_fini - stop the gfx ring
3932 *
3933 * @rdev: radeon_device pointer
3934 *
3935 * Stop the gfx ring and tear down the driver ring
3936 * info.
3937 */
3938static void cik_cp_gfx_fini(struct radeon_device *rdev)
3939{
3940 cik_cp_gfx_enable(rdev, false);
3941 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3942}
3943
3944/**
3945 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3946 *
3947 * @rdev: radeon_device pointer
3948 *
3949 * Program the location and size of the gfx ring buffer
3950 * and test it to make sure it's working.
3951 * Returns 0 for success, error for failure.
3952 */
3953static int cik_cp_gfx_resume(struct radeon_device *rdev)
3954{
3955 struct radeon_ring *ring;
3956 u32 tmp;
3957 u32 rb_bufsz;
3958 u64 rb_addr;
3959 int r;
3960
3961 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3962 if (rdev->family != CHIP_HAWAII)
3963 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3964
3965 /* Set the write pointer delay */
3966 WREG32(CP_RB_WPTR_DELAY, 0);
3967
3968 /* set the RB to use vmid 0 */
3969 WREG32(CP_RB_VMID, 0);
3970
3971 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3972
3973 /* ring 0 - compute and gfx */
3974 /* Set ring buffer size */
3975 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3976 rb_bufsz = order_base_2(ring->ring_size / 8);
3977 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3978#ifdef __BIG_ENDIAN
3979 tmp |= BUF_SWAP_32BIT;
3980#endif
3981 WREG32(CP_RB0_CNTL, tmp);
3982
3983 /* Initialize the ring buffer's read and write pointers */
3984 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3985 ring->wptr = 0;
3986 WREG32(CP_RB0_WPTR, ring->wptr);
3987
3988 /* set the wb address wether it's enabled or not */
3989 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3990 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3991
3992 /* scratch register shadowing is no longer supported */
3993 WREG32(SCRATCH_UMSK, 0);
3994
3995 if (!rdev->wb.enabled)
3996 tmp |= RB_NO_UPDATE;
3997
3998 mdelay(1);
3999 WREG32(CP_RB0_CNTL, tmp);
4000
4001 rb_addr = ring->gpu_addr >> 8;
4002 WREG32(CP_RB0_BASE, rb_addr);
4003 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4004
4005 ring->rptr = RREG32(CP_RB0_RPTR);
4006
4007 /* start the ring */
4008 cik_cp_gfx_start(rdev);
4009 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4010 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4011 if (r) {
4012 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4013 return r;
4014 }
4015 return 0;
4016}
4017
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4018u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4019 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4020{
4021 u32 rptr;
4022
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4023 if (rdev->wb.enabled)
4024 rptr = rdev->wb.wb[ring->rptr_offs/4];
4025 else
4026 rptr = RREG32(CP_RB0_RPTR);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4027
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4028 return rptr;
4029}
4030
4031u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4032 struct radeon_ring *ring)
4033{
4034 u32 wptr;
4035
4036 wptr = RREG32(CP_RB0_WPTR);
4037
4038 return wptr;
4039}
4040
4041void cik_gfx_set_wptr(struct radeon_device *rdev,
4042 struct radeon_ring *ring)
4043{
4044 WREG32(CP_RB0_WPTR, ring->wptr);
4045 (void)RREG32(CP_RB0_WPTR);
4046}
4047
4048u32 cik_compute_get_rptr(struct radeon_device *rdev,
4049 struct radeon_ring *ring)
4050{
4051 u32 rptr;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4052
4053 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4054 rptr = rdev->wb.wb[ring->rptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4055 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4056 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4057 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4058 rptr = RREG32(CP_HQD_PQ_RPTR);
4059 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4060 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4061 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4062
4063 return rptr;
4064}
4065
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4066u32 cik_compute_get_wptr(struct radeon_device *rdev,
4067 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4068{
4069 u32 wptr;
4070
4071 if (rdev->wb.enabled) {
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4072 /* XXX check if swapping is necessary on BE */
4073 wptr = rdev->wb.wb[ring->wptr_offs/4];
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4074 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4075 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4076 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4077 wptr = RREG32(CP_HQD_PQ_WPTR);
4078 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4079 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4080 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4081
4082 return wptr;
4083}
4084
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4085void cik_compute_set_wptr(struct radeon_device *rdev,
4086 struct radeon_ring *ring)
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4087{
Alex Deucherea31bf62013-12-09 19:44:30 -0500[diff] [blame]4088 /* XXX check if swapping is necessary on BE */
4089 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4090 WDOORBELL32(ring->doorbell_index, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4091}
4092
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4093/**
4094 * cik_cp_compute_enable - enable/disable the compute CP MEs
4095 *
4096 * @rdev: radeon_device pointer
4097 * @enable: enable or disable the MEs
4098 *
4099 * Halts or unhalts the compute MEs.
4100 */
4101static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4102{
4103 if (enable)
4104 WREG32(CP_MEC_CNTL, 0);
4105 else
4106 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4107 udelay(50);
4108}
4109
4110/**
4111 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4112 *
4113 * @rdev: radeon_device pointer
4114 *
4115 * Loads the compute MEC1&2 ucode.
4116 * Returns 0 for success, -EINVAL if the ucode is not available.
4117 */
4118static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4119{
4120 const __be32 *fw_data;
4121 int i;
4122
4123 if (!rdev->mec_fw)
4124 return -EINVAL;
4125
4126 cik_cp_compute_enable(rdev, false);
4127
4128 /* MEC1 */
4129 fw_data = (const __be32 *)rdev->mec_fw->data;
4130 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4131 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4132 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4133 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4134
4135 if (rdev->family == CHIP_KAVERI) {
4136 /* MEC2 */
4137 fw_data = (const __be32 *)rdev->mec_fw->data;
4138 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4139 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4140 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4141 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4142 }
4143
4144 return 0;
4145}
4146
4147/**
4148 * cik_cp_compute_start - start the compute queues
4149 *
4150 * @rdev: radeon_device pointer
4151 *
4152 * Enable the compute queues.
4153 * Returns 0 for success, error for failure.
4154 */
4155static int cik_cp_compute_start(struct radeon_device *rdev)
4156{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4157 cik_cp_compute_enable(rdev, true);
4158
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4159 return 0;
4160}
4161
4162/**
4163 * cik_cp_compute_fini - stop the compute queues
4164 *
4165 * @rdev: radeon_device pointer
4166 *
4167 * Stop the compute queues and tear down the driver queue
4168 * info.
4169 */
4170static void cik_cp_compute_fini(struct radeon_device *rdev)
4171{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4172 int i, idx, r;
4173
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4174 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4175
4176 for (i = 0; i < 2; i++) {
4177 if (i == 0)
4178 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4179 else
4180 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4181
4182 if (rdev->ring[idx].mqd_obj) {
4183 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4184 if (unlikely(r != 0))
4185 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4186
4187 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4188 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4189
4190 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4191 rdev->ring[idx].mqd_obj = NULL;
4192 }
4193 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4194}
4195
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4196static void cik_mec_fini(struct radeon_device *rdev)
4197{
4198 int r;
4199
4200 if (rdev->mec.hpd_eop_obj) {
4201 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4202 if (unlikely(r != 0))
4203 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4204 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4205 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4206
4207 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4208 rdev->mec.hpd_eop_obj = NULL;
4209 }
4210}
4211
4212#define MEC_HPD_SIZE 2048
4213
4214static int cik_mec_init(struct radeon_device *rdev)
4215{
4216 int r;
4217 u32 *hpd;
4218
4219 /*
4220 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4221 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4222 */
4223 if (rdev->family == CHIP_KAVERI)
4224 rdev->mec.num_mec = 2;
4225 else
4226 rdev->mec.num_mec = 1;
4227 rdev->mec.num_pipe = 4;
4228 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4229
4230 if (rdev->mec.hpd_eop_obj == NULL) {
4231 r = radeon_bo_create(rdev,
4232 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4233 PAGE_SIZE, true,
4234 RADEON_GEM_DOMAIN_GTT, NULL,
4235 &rdev->mec.hpd_eop_obj);
4236 if (r) {
4237 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4238 return r;
4239 }
4240 }
4241
4242 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4243 if (unlikely(r != 0)) {
4244 cik_mec_fini(rdev);
4245 return r;
4246 }
4247 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4248 &rdev->mec.hpd_eop_gpu_addr);
4249 if (r) {
4250 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4251 cik_mec_fini(rdev);
4252 return r;
4253 }
4254 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4255 if (r) {
4256 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4257 cik_mec_fini(rdev);
4258 return r;
4259 }
4260
4261 /* clear memory. Not sure if this is required or not */
4262 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4263
4264 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4265 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4266
4267 return 0;
4268}
4269
4270struct hqd_registers
4271{
4272 u32 cp_mqd_base_addr;
4273 u32 cp_mqd_base_addr_hi;
4274 u32 cp_hqd_active;
4275 u32 cp_hqd_vmid;
4276 u32 cp_hqd_persistent_state;
4277 u32 cp_hqd_pipe_priority;
4278 u32 cp_hqd_queue_priority;
4279 u32 cp_hqd_quantum;
4280 u32 cp_hqd_pq_base;
4281 u32 cp_hqd_pq_base_hi;
4282 u32 cp_hqd_pq_rptr;
4283 u32 cp_hqd_pq_rptr_report_addr;
4284 u32 cp_hqd_pq_rptr_report_addr_hi;
4285 u32 cp_hqd_pq_wptr_poll_addr;
4286 u32 cp_hqd_pq_wptr_poll_addr_hi;
4287 u32 cp_hqd_pq_doorbell_control;
4288 u32 cp_hqd_pq_wptr;
4289 u32 cp_hqd_pq_control;
4290 u32 cp_hqd_ib_base_addr;
4291 u32 cp_hqd_ib_base_addr_hi;
4292 u32 cp_hqd_ib_rptr;
4293 u32 cp_hqd_ib_control;
4294 u32 cp_hqd_iq_timer;
4295 u32 cp_hqd_iq_rptr;
4296 u32 cp_hqd_dequeue_request;
4297 u32 cp_hqd_dma_offload;
4298 u32 cp_hqd_sema_cmd;
4299 u32 cp_hqd_msg_type;
4300 u32 cp_hqd_atomic0_preop_lo;
4301 u32 cp_hqd_atomic0_preop_hi;
4302 u32 cp_hqd_atomic1_preop_lo;
4303 u32 cp_hqd_atomic1_preop_hi;
4304 u32 cp_hqd_hq_scheduler0;
4305 u32 cp_hqd_hq_scheduler1;
4306 u32 cp_mqd_control;
4307};
4308
4309struct bonaire_mqd
4310{
4311 u32 header;
4312 u32 dispatch_initiator;
4313 u32 dimensions[3];
4314 u32 start_idx[3];
4315 u32 num_threads[3];
4316 u32 pipeline_stat_enable;
4317 u32 perf_counter_enable;
4318 u32 pgm[2];
4319 u32 tba[2];
4320 u32 tma[2];
4321 u32 pgm_rsrc[2];
4322 u32 vmid;
4323 u32 resource_limits;
4324 u32 static_thread_mgmt01[2];
4325 u32 tmp_ring_size;
4326 u32 static_thread_mgmt23[2];
4327 u32 restart[3];
4328 u32 thread_trace_enable;
4329 u32 reserved1;
4330 u32 user_data[16];
4331 u32 vgtcs_invoke_count[2];
4332 struct hqd_registers queue_state;
4333 u32 dequeue_cntr;
4334 u32 interrupt_queue[64];
4335};
4336
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4337/**
4338 * cik_cp_compute_resume - setup the compute queue registers
4339 *
4340 * @rdev: radeon_device pointer
4341 *
4342 * Program the compute queues and test them to make sure they
4343 * are working.
4344 * Returns 0 for success, error for failure.
4345 */
4346static int cik_cp_compute_resume(struct radeon_device *rdev)
4347{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4348 int r, i, idx;
4349 u32 tmp;
4350 bool use_doorbell = true;
4351 u64 hqd_gpu_addr;
4352 u64 mqd_gpu_addr;
4353 u64 eop_gpu_addr;
4354 u64 wb_gpu_addr;
4355 u32 *buf;
4356 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4357
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4358 r = cik_cp_compute_start(rdev);
4359 if (r)
4360 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4361
4362 /* fix up chicken bits */
4363 tmp = RREG32(CP_CPF_DEBUG);
4364 tmp |= (1 << 23);
4365 WREG32(CP_CPF_DEBUG, tmp);
4366
4367 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4368 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4369 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4370 int me = (i < 4) ? 1 : 2;
4371 int pipe = (i < 4) ? i : (i - 4);
4372
4373 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4374
4375 cik_srbm_select(rdev, me, pipe, 0, 0);
4376
4377 /* write the EOP addr */
4378 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4379 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4380
4381 /* set the VMID assigned */
4382 WREG32(CP_HPD_EOP_VMID, 0);
4383
4384 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4385 tmp = RREG32(CP_HPD_EOP_CONTROL);
4386 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4387 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4388 WREG32(CP_HPD_EOP_CONTROL, tmp);
4389 }
4390 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4391 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4392
4393 /* init the queues. Just two for now. */
4394 for (i = 0; i < 2; i++) {
4395 if (i == 0)
4396 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4397 else
4398 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4399
4400 if (rdev->ring[idx].mqd_obj == NULL) {
4401 r = radeon_bo_create(rdev,
4402 sizeof(struct bonaire_mqd),
4403 PAGE_SIZE, true,
4404 RADEON_GEM_DOMAIN_GTT, NULL,
4405 &rdev->ring[idx].mqd_obj);
4406 if (r) {
4407 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4408 return r;
4409 }
4410 }
4411
4412 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4413 if (unlikely(r != 0)) {
4414 cik_cp_compute_fini(rdev);
4415 return r;
4416 }
4417 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4418 &mqd_gpu_addr);
4419 if (r) {
4420 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4421 cik_cp_compute_fini(rdev);
4422 return r;
4423 }
4424 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4425 if (r) {
4426 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4427 cik_cp_compute_fini(rdev);
4428 return r;
4429 }
4430
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4431 /* init the mqd struct */
4432 memset(buf, 0, sizeof(struct bonaire_mqd));
4433
4434 mqd = (struct bonaire_mqd *)buf;
4435 mqd->header = 0xC0310800;
4436 mqd->static_thread_mgmt01[0] = 0xffffffff;
4437 mqd->static_thread_mgmt01[1] = 0xffffffff;
4438 mqd->static_thread_mgmt23[0] = 0xffffffff;
4439 mqd->static_thread_mgmt23[1] = 0xffffffff;
4440
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4441 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4442 cik_srbm_select(rdev, rdev->ring[idx].me,
4443 rdev->ring[idx].pipe,
4444 rdev->ring[idx].queue, 0);
4445
4446 /* disable wptr polling */
4447 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4448 tmp &= ~WPTR_POLL_EN;
4449 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4450
4451 /* enable doorbell? */
4452 mqd->queue_state.cp_hqd_pq_doorbell_control =
4453 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4454 if (use_doorbell)
4455 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4456 else
4457 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4458 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4459 mqd->queue_state.cp_hqd_pq_doorbell_control);
4460
4461 /* disable the queue if it's active */
4462 mqd->queue_state.cp_hqd_dequeue_request = 0;
4463 mqd->queue_state.cp_hqd_pq_rptr = 0;
4464 mqd->queue_state.cp_hqd_pq_wptr= 0;
4465 if (RREG32(CP_HQD_ACTIVE) & 1) {
4466 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4467 for (i = 0; i < rdev->usec_timeout; i++) {
4468 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4469 break;
4470 udelay(1);
4471 }
4472 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4473 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4474 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4475 }
4476
4477 /* set the pointer to the MQD */
4478 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4479 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4480 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4481 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4482 /* set MQD vmid to 0 */
4483 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4484 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4485 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4486
4487 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4488 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4489 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4490 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4491 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4492 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4493
4494 /* set up the HQD, this is similar to CP_RB0_CNTL */
4495 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4496 mqd->queue_state.cp_hqd_pq_control &=
4497 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4498
4499 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4500 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4501 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4502 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4503#ifdef __BIG_ENDIAN
4504 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4505#endif
4506 mqd->queue_state.cp_hqd_pq_control &=
4507 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4508 mqd->queue_state.cp_hqd_pq_control |=
4509 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4510 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4511
4512 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4513 if (i == 0)
4514 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4515 else
4516 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4517 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4518 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4519 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4520 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4521 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4522
4523 /* set the wb address wether it's enabled or not */
4524 if (i == 0)
4525 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4526 else
4527 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4528 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4529 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4530 upper_32_bits(wb_gpu_addr) & 0xffff;
4531 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4532 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4533 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4534 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4535
4536 /* enable the doorbell if requested */
4537 if (use_doorbell) {
4538 mqd->queue_state.cp_hqd_pq_doorbell_control =
4539 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4540 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4541 mqd->queue_state.cp_hqd_pq_doorbell_control |=
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4542 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4543 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4544 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4545 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4546
4547 } else {
4548 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4549 }
4550 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4551 mqd->queue_state.cp_hqd_pq_doorbell_control);
4552
4553 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4554 rdev->ring[idx].wptr = 0;
4555 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4556 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4557 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4558 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4559
4560 /* set the vmid for the queue */
4561 mqd->queue_state.cp_hqd_vmid = 0;
4562 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4563
4564 /* activate the queue */
4565 mqd->queue_state.cp_hqd_active = 1;
4566 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4567
4568 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4569 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4570
4571 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4572 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4573
4574 rdev->ring[idx].ready = true;
4575 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4576 if (r)
4577 rdev->ring[idx].ready = false;
4578 }
4579
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4580 return 0;
4581}
4582
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4583static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4584{
4585 cik_cp_gfx_enable(rdev, enable);
4586 cik_cp_compute_enable(rdev, enable);
4587}
4588
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4589static int cik_cp_load_microcode(struct radeon_device *rdev)
4590{
4591 int r;
4592
4593 r = cik_cp_gfx_load_microcode(rdev);
4594 if (r)
4595 return r;
4596 r = cik_cp_compute_load_microcode(rdev);
4597 if (r)
4598 return r;
4599
4600 return 0;
4601}
4602
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4603static void cik_cp_fini(struct radeon_device *rdev)
4604{
4605 cik_cp_gfx_fini(rdev);
4606 cik_cp_compute_fini(rdev);
4607}
4608
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4609static int cik_cp_resume(struct radeon_device *rdev)
4610{
4611 int r;
4612
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4613 cik_enable_gui_idle_interrupt(rdev, false);
4614
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4615 r = cik_cp_load_microcode(rdev);
4616 if (r)
4617 return r;
4618
4619 r = cik_cp_gfx_resume(rdev);
4620 if (r)
4621 return r;
4622 r = cik_cp_compute_resume(rdev);
4623 if (r)
4624 return r;
4625
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4626 cik_enable_gui_idle_interrupt(rdev, true);
4627
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4628 return 0;
4629}
4630
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4631static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4632{
4633 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4634 RREG32(GRBM_STATUS));
4635 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4636 RREG32(GRBM_STATUS2));
4637 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4638 RREG32(GRBM_STATUS_SE0));
4639 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4640 RREG32(GRBM_STATUS_SE1));
4641 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4642 RREG32(GRBM_STATUS_SE2));
4643 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4644 RREG32(GRBM_STATUS_SE3));
4645 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4646 RREG32(SRBM_STATUS));
4647 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4648 RREG32(SRBM_STATUS2));
4649 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4650 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4651 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4652 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4653 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4654 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4655 RREG32(CP_STALLED_STAT1));
4656 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4657 RREG32(CP_STALLED_STAT2));
4658 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4659 RREG32(CP_STALLED_STAT3));
4660 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4661 RREG32(CP_CPF_BUSY_STAT));
4662 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4663 RREG32(CP_CPF_STALLED_STAT1));
4664 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4665 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4666 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4667 RREG32(CP_CPC_STALLED_STAT1));
4668 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4669}
4670
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4671/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4672 * cik_gpu_check_soft_reset - check which blocks are busy
4673 *
4674 * @rdev: radeon_device pointer
4675 *
4676 * Check which blocks are busy and return the relevant reset
4677 * mask to be used by cik_gpu_soft_reset().
4678 * Returns a mask of the blocks to be reset.
4679 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4680u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4681{
4682 u32 reset_mask = 0;
4683 u32 tmp;
4684
4685 /* GRBM_STATUS */
4686 tmp = RREG32(GRBM_STATUS);
4687 if (tmp & (PA_BUSY | SC_BUSY |
4688 BCI_BUSY | SX_BUSY |
4689 TA_BUSY | VGT_BUSY |
4690 DB_BUSY | CB_BUSY |
4691 GDS_BUSY | SPI_BUSY |
4692 IA_BUSY | IA_BUSY_NO_DMA))
4693 reset_mask |= RADEON_RESET_GFX;
4694
4695 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4696 reset_mask |= RADEON_RESET_CP;
4697
4698 /* GRBM_STATUS2 */
4699 tmp = RREG32(GRBM_STATUS2);
4700 if (tmp & RLC_BUSY)
4701 reset_mask |= RADEON_RESET_RLC;
4702
4703 /* SDMA0_STATUS_REG */
4704 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4705 if (!(tmp & SDMA_IDLE))
4706 reset_mask |= RADEON_RESET_DMA;
4707
4708 /* SDMA1_STATUS_REG */
4709 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4710 if (!(tmp & SDMA_IDLE))
4711 reset_mask |= RADEON_RESET_DMA1;
4712
4713 /* SRBM_STATUS2 */
4714 tmp = RREG32(SRBM_STATUS2);
4715 if (tmp & SDMA_BUSY)
4716 reset_mask |= RADEON_RESET_DMA;
4717
4718 if (tmp & SDMA1_BUSY)
4719 reset_mask |= RADEON_RESET_DMA1;
4720
4721 /* SRBM_STATUS */
4722 tmp = RREG32(SRBM_STATUS);
4723
4724 if (tmp & IH_BUSY)
4725 reset_mask |= RADEON_RESET_IH;
4726
4727 if (tmp & SEM_BUSY)
4728 reset_mask |= RADEON_RESET_SEM;
4729
4730 if (tmp & GRBM_RQ_PENDING)
4731 reset_mask |= RADEON_RESET_GRBM;
4732
4733 if (tmp & VMC_BUSY)
4734 reset_mask |= RADEON_RESET_VMC;
4735
4736 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4737 MCC_BUSY | MCD_BUSY))
4738 reset_mask |= RADEON_RESET_MC;
4739
4740 if (evergreen_is_display_hung(rdev))
4741 reset_mask |= RADEON_RESET_DISPLAY;
4742
4743 /* Skip MC reset as it's mostly likely not hung, just busy */
4744 if (reset_mask & RADEON_RESET_MC) {
4745 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4746 reset_mask &= ~RADEON_RESET_MC;
4747 }
4748
4749 return reset_mask;
4750}
4751
4752/**
4753 * cik_gpu_soft_reset - soft reset GPU
4754 *
4755 * @rdev: radeon_device pointer
4756 * @reset_mask: mask of which blocks to reset
4757 *
4758 * Soft reset the blocks specified in @reset_mask.
4759 */
4760static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4761{
4762 struct evergreen_mc_save save;
4763 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4764 u32 tmp;
4765
4766 if (reset_mask == 0)
4767 return;
4768
4769 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4770
4771 cik_print_gpu_status_regs(rdev);
4772 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4773 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4774 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4775 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4776
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4777 /* disable CG/PG */
4778 cik_fini_pg(rdev);
4779 cik_fini_cg(rdev);
4780
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4781 /* stop the rlc */
4782 cik_rlc_stop(rdev);
4783
4784 /* Disable GFX parsing/prefetching */
4785 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4786
4787 /* Disable MEC parsing/prefetching */
4788 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4789
4790 if (reset_mask & RADEON_RESET_DMA) {
4791 /* sdma0 */
4792 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4793 tmp |= SDMA_HALT;
4794 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4795 }
4796 if (reset_mask & RADEON_RESET_DMA1) {
4797 /* sdma1 */
4798 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4799 tmp |= SDMA_HALT;
4800 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4801 }
4802
4803 evergreen_mc_stop(rdev, &save);
4804 if (evergreen_mc_wait_for_idle(rdev)) {
4805 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4806 }
4807
4808 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4809 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4810
4811 if (reset_mask & RADEON_RESET_CP) {
4812 grbm_soft_reset |= SOFT_RESET_CP;
4813
4814 srbm_soft_reset |= SOFT_RESET_GRBM;
4815 }
4816
4817 if (reset_mask & RADEON_RESET_DMA)
4818 srbm_soft_reset |= SOFT_RESET_SDMA;
4819
4820 if (reset_mask & RADEON_RESET_DMA1)
4821 srbm_soft_reset |= SOFT_RESET_SDMA1;
4822
4823 if (reset_mask & RADEON_RESET_DISPLAY)
4824 srbm_soft_reset |= SOFT_RESET_DC;
4825
4826 if (reset_mask & RADEON_RESET_RLC)
4827 grbm_soft_reset |= SOFT_RESET_RLC;
4828
4829 if (reset_mask & RADEON_RESET_SEM)
4830 srbm_soft_reset |= SOFT_RESET_SEM;
4831
4832 if (reset_mask & RADEON_RESET_IH)
4833 srbm_soft_reset |= SOFT_RESET_IH;
4834
4835 if (reset_mask & RADEON_RESET_GRBM)
4836 srbm_soft_reset |= SOFT_RESET_GRBM;
4837
4838 if (reset_mask & RADEON_RESET_VMC)
4839 srbm_soft_reset |= SOFT_RESET_VMC;
4840
4841 if (!(rdev->flags & RADEON_IS_IGP)) {
4842 if (reset_mask & RADEON_RESET_MC)
4843 srbm_soft_reset |= SOFT_RESET_MC;
4844 }
4845
4846 if (grbm_soft_reset) {
4847 tmp = RREG32(GRBM_SOFT_RESET);
4848 tmp |= grbm_soft_reset;
4849 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4850 WREG32(GRBM_SOFT_RESET, tmp);
4851 tmp = RREG32(GRBM_SOFT_RESET);
4852
4853 udelay(50);
4854
4855 tmp &= ~grbm_soft_reset;
4856 WREG32(GRBM_SOFT_RESET, tmp);
4857 tmp = RREG32(GRBM_SOFT_RESET);
4858 }
4859
4860 if (srbm_soft_reset) {
4861 tmp = RREG32(SRBM_SOFT_RESET);
4862 tmp |= srbm_soft_reset;
4863 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4864 WREG32(SRBM_SOFT_RESET, tmp);
4865 tmp = RREG32(SRBM_SOFT_RESET);
4866
4867 udelay(50);
4868
4869 tmp &= ~srbm_soft_reset;
4870 WREG32(SRBM_SOFT_RESET, tmp);
4871 tmp = RREG32(SRBM_SOFT_RESET);
4872 }
4873
4874 /* Wait a little for things to settle down */
4875 udelay(50);
4876
4877 evergreen_mc_resume(rdev, &save);
4878 udelay(50);
4879
4880 cik_print_gpu_status_regs(rdev);
4881}
4882
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame^]4883struct kv_reset_save_regs {
4884 u32 gmcon_reng_execute;
4885 u32 gmcon_misc;
4886 u32 gmcon_misc3;
4887};
4888
4889static void kv_save_regs_for_reset(struct radeon_device *rdev,
4890 struct kv_reset_save_regs *save)
4891{
4892 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4893 save->gmcon_misc = RREG32(GMCON_MISC);
4894 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4895
4896 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4897 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4898 STCTRL_STUTTER_EN));
4899}
4900
4901static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4902 struct kv_reset_save_regs *save)
4903{
4904 int i;
4905
4906 WREG32(GMCON_PGFSM_WRITE, 0);
4907 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4908
4909 for (i = 0; i < 5; i++)
4910 WREG32(GMCON_PGFSM_WRITE, 0);
4911
4912 WREG32(GMCON_PGFSM_WRITE, 0);
4913 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4914
4915 for (i = 0; i < 5; i++)
4916 WREG32(GMCON_PGFSM_WRITE, 0);
4917
4918 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4919 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4920
4921 for (i = 0; i < 5; i++)
4922 WREG32(GMCON_PGFSM_WRITE, 0);
4923
4924 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4925 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4926
4927 for (i = 0; i < 5; i++)
4928 WREG32(GMCON_PGFSM_WRITE, 0);
4929
4930 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4931 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4932
4933 for (i = 0; i < 5; i++)
4934 WREG32(GMCON_PGFSM_WRITE, 0);
4935
4936 WREG32(GMCON_PGFSM_WRITE, 0);
4937 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
4938
4939 for (i = 0; i < 5; i++)
4940 WREG32(GMCON_PGFSM_WRITE, 0);
4941
4942 WREG32(GMCON_PGFSM_WRITE, 0x420000);
4943 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
4944
4945 for (i = 0; i < 5; i++)
4946 WREG32(GMCON_PGFSM_WRITE, 0);
4947
4948 WREG32(GMCON_PGFSM_WRITE, 0x120202);
4949 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
4950
4951 for (i = 0; i < 5; i++)
4952 WREG32(GMCON_PGFSM_WRITE, 0);
4953
4954 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
4955 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
4956
4957 for (i = 0; i < 5; i++)
4958 WREG32(GMCON_PGFSM_WRITE, 0);
4959
4960 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
4961 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
4962
4963 for (i = 0; i < 5; i++)
4964 WREG32(GMCON_PGFSM_WRITE, 0);
4965
4966 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
4967 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
4968
4969 WREG32(GMCON_MISC3, save->gmcon_misc3);
4970 WREG32(GMCON_MISC, save->gmcon_misc);
4971 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
4972}
4973
4974static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
4975{
4976 struct evergreen_mc_save save;
4977 struct kv_reset_save_regs kv_save = { 0 };
4978 u32 tmp, i;
4979
4980 dev_info(rdev->dev, "GPU pci config reset\n");
4981
4982 /* disable dpm? */
4983
4984 /* disable cg/pg */
4985 cik_fini_pg(rdev);
4986 cik_fini_cg(rdev);
4987
4988 /* Disable GFX parsing/prefetching */
4989 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4990
4991 /* Disable MEC parsing/prefetching */
4992 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4993
4994 /* sdma0 */
4995 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4996 tmp |= SDMA_HALT;
4997 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4998 /* sdma1 */
4999 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5000 tmp |= SDMA_HALT;
5001 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5002 /* XXX other engines? */
5003
5004 /* halt the rlc, disable cp internal ints */
5005 cik_rlc_stop(rdev);
5006
5007 udelay(50);
5008
5009 /* disable mem access */
5010 evergreen_mc_stop(rdev, &save);
5011 if (evergreen_mc_wait_for_idle(rdev)) {
5012 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5013 }
5014
5015 if (rdev->flags & RADEON_IS_IGP)
5016 kv_save_regs_for_reset(rdev, &kv_save);
5017
5018 /* disable BM */
5019 pci_clear_master(rdev->pdev);
5020 /* reset */
5021 radeon_pci_config_reset(rdev);
5022
5023 udelay(100);
5024
5025 /* wait for asic to come out of reset */
5026 for (i = 0; i < rdev->usec_timeout; i++) {
5027 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5028 break;
5029 udelay(1);
5030 }
5031
5032 /* does asic init need to be run first??? */
5033 if (rdev->flags & RADEON_IS_IGP)
5034 kv_restore_regs_for_reset(rdev, &kv_save);
5035}
5036
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5037/**
5038 * cik_asic_reset - soft reset GPU
5039 *
5040 * @rdev: radeon_device pointer
5041 *
5042 * Look up which blocks are hung and attempt
5043 * to reset them.
5044 * Returns 0 for success.
5045 */
5046int cik_asic_reset(struct radeon_device *rdev)
5047{
5048 u32 reset_mask;
5049
5050 reset_mask = cik_gpu_check_soft_reset(rdev);
5051
5052 if (reset_mask)
5053 r600_set_bios_scratch_engine_hung(rdev, true);
5054
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame^]5055 /* try soft reset */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5056 cik_gpu_soft_reset(rdev, reset_mask);
5057
5058 reset_mask = cik_gpu_check_soft_reset(rdev);
5059
Alex Deucher0279ed12013-10-02 15:18:14 -0400[diff] [blame^]5060 /* try pci config reset */
5061 if (reset_mask && radeon_hard_reset)
5062 cik_gpu_pci_config_reset(rdev);
5063
5064 reset_mask = cik_gpu_check_soft_reset(rdev);
5065
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5066 if (!reset_mask)
5067 r600_set_bios_scratch_engine_hung(rdev, false);
5068
5069 return 0;
5070}
5071
5072/**
5073 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5074 *
5075 * @rdev: radeon_device pointer
5076 * @ring: radeon_ring structure holding ring information
5077 *
5078 * Check if the 3D engine is locked up (CIK).
5079 * Returns true if the engine is locked, false if not.
5080 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5081bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5082{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5083 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5084
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]5085 if (!(reset_mask & (RADEON_RESET_GFX |
5086 RADEON_RESET_COMPUTE |
5087 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]5088 radeon_ring_lockup_update(ring);
5089 return false;
5090 }
5091 /* force CP activities */
5092 radeon_ring_force_activity(rdev, ring);
5093 return radeon_ring_test_lockup(rdev, ring);
5094}
5095
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5096/* MC */
5097/**
5098 * cik_mc_program - program the GPU memory controller
5099 *
5100 * @rdev: radeon_device pointer
5101 *
5102 * Set the location of vram, gart, and AGP in the GPU's
5103 * physical address space (CIK).
5104 */
5105static void cik_mc_program(struct radeon_device *rdev)
5106{
5107 struct evergreen_mc_save save;
5108 u32 tmp;
5109 int i, j;
5110
5111 /* Initialize HDP */
5112 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5113 WREG32((0x2c14 + j), 0x00000000);
5114 WREG32((0x2c18 + j), 0x00000000);
5115 WREG32((0x2c1c + j), 0x00000000);
5116 WREG32((0x2c20 + j), 0x00000000);
5117 WREG32((0x2c24 + j), 0x00000000);
5118 }
5119 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5120
5121 evergreen_mc_stop(rdev, &save);
5122 if (radeon_mc_wait_for_idle(rdev)) {
5123 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5124 }
5125 /* Lockout access through VGA aperture*/
5126 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5127 /* Update configuration */
5128 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5129 rdev->mc.vram_start >> 12);
5130 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5131 rdev->mc.vram_end >> 12);
5132 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5133 rdev->vram_scratch.gpu_addr >> 12);
5134 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5135 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5136 WREG32(MC_VM_FB_LOCATION, tmp);
5137 /* XXX double check these! */
5138 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5139 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5140 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5141 WREG32(MC_VM_AGP_BASE, 0);
5142 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5143 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5144 if (radeon_mc_wait_for_idle(rdev)) {
5145 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5146 }
5147 evergreen_mc_resume(rdev, &save);
5148 /* we need to own VRAM, so turn off the VGA renderer here
5149 * to stop it overwriting our objects */
5150 rv515_vga_render_disable(rdev);
5151}
5152
5153/**
5154 * cik_mc_init - initialize the memory controller driver params
5155 *
5156 * @rdev: radeon_device pointer
5157 *
5158 * Look up the amount of vram, vram width, and decide how to place
5159 * vram and gart within the GPU's physical address space (CIK).
5160 * Returns 0 for success.
5161 */
5162static int cik_mc_init(struct radeon_device *rdev)
5163{
5164 u32 tmp;
5165 int chansize, numchan;
5166
5167 /* Get VRAM informations */
5168 rdev->mc.vram_is_ddr = true;
5169 tmp = RREG32(MC_ARB_RAMCFG);
5170 if (tmp & CHANSIZE_MASK) {
5171 chansize = 64;
5172 } else {
5173 chansize = 32;
5174 }
5175 tmp = RREG32(MC_SHARED_CHMAP);
5176 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5177 case 0:
5178 default:
5179 numchan = 1;
5180 break;
5181 case 1:
5182 numchan = 2;
5183 break;
5184 case 2:
5185 numchan = 4;
5186 break;
5187 case 3:
5188 numchan = 8;
5189 break;
5190 case 4:
5191 numchan = 3;
5192 break;
5193 case 5:
5194 numchan = 6;
5195 break;
5196 case 6:
5197 numchan = 10;
5198 break;
5199 case 7:
5200 numchan = 12;
5201 break;
5202 case 8:
5203 numchan = 16;
5204 break;
5205 }
5206 rdev->mc.vram_width = numchan * chansize;
5207 /* Could aper size report 0 ? */
5208 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5209 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5210 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5211 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5212 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5213 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5214 si_vram_gtt_location(rdev, &rdev->mc);
5215 radeon_update_bandwidth_info(rdev);
5216
5217 return 0;
5218}
5219
5220/*
5221 * GART
5222 * VMID 0 is the physical GPU addresses as used by the kernel.
5223 * VMIDs 1-15 are used for userspace clients and are handled
5224 * by the radeon vm/hsa code.
5225 */
5226/**
5227 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5228 *
5229 * @rdev: radeon_device pointer
5230 *
5231 * Flush the TLB for the VMID 0 page table (CIK).
5232 */
5233void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5234{
5235 /* flush hdp cache */
5236 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5237
5238 /* bits 0-15 are the VM contexts0-15 */
5239 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5240}
5241
5242/**
5243 * cik_pcie_gart_enable - gart enable
5244 *
5245 * @rdev: radeon_device pointer
5246 *
5247 * This sets up the TLBs, programs the page tables for VMID0,
5248 * sets up the hw for VMIDs 1-15 which are allocated on
5249 * demand, and sets up the global locations for the LDS, GDS,
5250 * and GPUVM for FSA64 clients (CIK).
5251 * Returns 0 for success, errors for failure.
5252 */
5253static int cik_pcie_gart_enable(struct radeon_device *rdev)
5254{
5255 int r, i;
5256
5257 if (rdev->gart.robj == NULL) {
5258 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5259 return -EINVAL;
5260 }
5261 r = radeon_gart_table_vram_pin(rdev);
5262 if (r)
5263 return r;
5264 radeon_gart_restore(rdev);
5265 /* Setup TLB control */
5266 WREG32(MC_VM_MX_L1_TLB_CNTL,
5267 (0xA << 7) |
5268 ENABLE_L1_TLB |
5269 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5270 ENABLE_ADVANCED_DRIVER_MODEL |
5271 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5272 /* Setup L2 cache */
5273 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5274 ENABLE_L2_FRAGMENT_PROCESSING |
5275 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5276 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5277 EFFECTIVE_L2_QUEUE_SIZE(7) |
5278 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5279 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5280 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5281 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5282 /* setup context0 */
5283 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5284 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5285 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5286 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5287 (u32)(rdev->dummy_page.addr >> 12));
5288 WREG32(VM_CONTEXT0_CNTL2, 0);
5289 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5290 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5291
5292 WREG32(0x15D4, 0);
5293 WREG32(0x15D8, 0);
5294 WREG32(0x15DC, 0);
5295
5296 /* empty context1-15 */
5297 /* FIXME start with 4G, once using 2 level pt switch to full
5298 * vm size space
5299 */
5300 /* set vm size, must be a multiple of 4 */
5301 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5302 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5303 for (i = 1; i < 16; i++) {
5304 if (i < 8)
5305 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5306 rdev->gart.table_addr >> 12);
5307 else
5308 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5309 rdev->gart.table_addr >> 12);
5310 }
5311
5312 /* enable context1-15 */
5313 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5314 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5315 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5316 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5317 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5318 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5319 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5320 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5321 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5322 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5323 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5324 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5325 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5326 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5327 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5328 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5329
5330 /* TC cache setup ??? */
5331 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5332 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5333 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5334
5335 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5336 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5337 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5338 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5339 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5340
5341 WREG32(TC_CFG_L1_VOLATILE, 0);
5342 WREG32(TC_CFG_L2_VOLATILE, 0);
5343
5344 if (rdev->family == CHIP_KAVERI) {
5345 u32 tmp = RREG32(CHUB_CONTROL);
5346 tmp &= ~BYPASS_VM;
5347 WREG32(CHUB_CONTROL, tmp);
5348 }
5349
5350 /* XXX SH_MEM regs */
5351 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5352 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5353 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5354 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5355 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5356 WREG32(SH_MEM_CONFIG, 0);
5357 WREG32(SH_MEM_APE1_BASE, 1);
5358 WREG32(SH_MEM_APE1_LIMIT, 0);
5359 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5360 /* SDMA GFX */
5361 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5362 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5363 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5364 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5365 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5366 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5367 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5368 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5369
5370 cik_pcie_gart_tlb_flush(rdev);
5371 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5372 (unsigned)(rdev->mc.gtt_size >> 20),
5373 (unsigned long long)rdev->gart.table_addr);
5374 rdev->gart.ready = true;
5375 return 0;
5376}
5377
5378/**
5379 * cik_pcie_gart_disable - gart disable
5380 *
5381 * @rdev: radeon_device pointer
5382 *
5383 * This disables all VM page table (CIK).
5384 */
5385static void cik_pcie_gart_disable(struct radeon_device *rdev)
5386{
5387 /* Disable all tables */
5388 WREG32(VM_CONTEXT0_CNTL, 0);
5389 WREG32(VM_CONTEXT1_CNTL, 0);
5390 /* Setup TLB control */
5391 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5392 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5393 /* Setup L2 cache */
5394 WREG32(VM_L2_CNTL,
5395 ENABLE_L2_FRAGMENT_PROCESSING |
5396 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5397 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5398 EFFECTIVE_L2_QUEUE_SIZE(7) |
5399 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5400 WREG32(VM_L2_CNTL2, 0);
5401 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5402 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5403 radeon_gart_table_vram_unpin(rdev);
5404}
5405
5406/**
5407 * cik_pcie_gart_fini - vm fini callback
5408 *
5409 * @rdev: radeon_device pointer
5410 *
5411 * Tears down the driver GART/VM setup (CIK).
5412 */
5413static void cik_pcie_gart_fini(struct radeon_device *rdev)
5414{
5415 cik_pcie_gart_disable(rdev);
5416 radeon_gart_table_vram_free(rdev);
5417 radeon_gart_fini(rdev);
5418}
5419
5420/* vm parser */
5421/**
5422 * cik_ib_parse - vm ib_parse callback
5423 *
5424 * @rdev: radeon_device pointer
5425 * @ib: indirect buffer pointer
5426 *
5427 * CIK uses hw IB checking so this is a nop (CIK).
5428 */
5429int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5430{
5431 return 0;
5432}
5433
5434/*
5435 * vm
5436 * VMID 0 is the physical GPU addresses as used by the kernel.
5437 * VMIDs 1-15 are used for userspace clients and are handled
5438 * by the radeon vm/hsa code.
5439 */
5440/**
5441 * cik_vm_init - cik vm init callback
5442 *
5443 * @rdev: radeon_device pointer
5444 *
5445 * Inits cik specific vm parameters (number of VMs, base of vram for
5446 * VMIDs 1-15) (CIK).
5447 * Returns 0 for success.
5448 */
5449int cik_vm_init(struct radeon_device *rdev)
5450{
5451 /* number of VMs */
5452 rdev->vm_manager.nvm = 16;
5453 /* base offset of vram pages */
5454 if (rdev->flags & RADEON_IS_IGP) {
5455 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5456 tmp <<= 22;
5457 rdev->vm_manager.vram_base_offset = tmp;
5458 } else
5459 rdev->vm_manager.vram_base_offset = 0;
5460
5461 return 0;
5462}
5463
5464/**
5465 * cik_vm_fini - cik vm fini callback
5466 *
5467 * @rdev: radeon_device pointer
5468 *
5469 * Tear down any asic specific VM setup (CIK).
5470 */
5471void cik_vm_fini(struct radeon_device *rdev)
5472{
5473}
5474
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5475/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5476 * cik_vm_decode_fault - print human readable fault info
5477 *
5478 * @rdev: radeon_device pointer
5479 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5480 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5481 *
5482 * Print human readable fault information (CIK).
5483 */
5484static void cik_vm_decode_fault(struct radeon_device *rdev,
5485 u32 status, u32 addr, u32 mc_client)
5486{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5487 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5488 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5489 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5490 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5491 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5492
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5493 if (rdev->family == CHIP_HAWAII)
5494 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5495 else
5496 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5497
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5498 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5499 protections, vmid, addr,
5500 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5501 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5502}
5503
5504/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5505 * cik_vm_flush - cik vm flush using the CP
5506 *
5507 * @rdev: radeon_device pointer
5508 *
5509 * Update the page table base and flush the VM TLB
5510 * using the CP (CIK).
5511 */
5512void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5513{
5514 struct radeon_ring *ring = &rdev->ring[ridx];
5515
5516 if (vm == NULL)
5517 return;
5518
5519 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5520 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5521 WRITE_DATA_DST_SEL(0)));
5522 if (vm->id < 8) {
5523 radeon_ring_write(ring,
5524 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5525 } else {
5526 radeon_ring_write(ring,
5527 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5528 }
5529 radeon_ring_write(ring, 0);
5530 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5531
5532 /* update SH_MEM_* regs */
5533 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5534 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5535 WRITE_DATA_DST_SEL(0)));
5536 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5537 radeon_ring_write(ring, 0);
5538 radeon_ring_write(ring, VMID(vm->id));
5539
5540 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5541 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5542 WRITE_DATA_DST_SEL(0)));
5543 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5544 radeon_ring_write(ring, 0);
5545
5546 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5547 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5548 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5549 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5550
5551 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5552 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5553 WRITE_DATA_DST_SEL(0)));
5554 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5555 radeon_ring_write(ring, 0);
5556 radeon_ring_write(ring, VMID(0));
5557
5558 /* HDP flush */
5559 /* We should be using the WAIT_REG_MEM packet here like in
5560 * cik_fence_ring_emit(), but it causes the CP to hang in this
5561 * context...
5562 */
5563 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5564 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5565 WRITE_DATA_DST_SEL(0)));
5566 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5567 radeon_ring_write(ring, 0);
5568 radeon_ring_write(ring, 0);
5569
5570 /* bits 0-15 are the VM contexts0-15 */
5571 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5572 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5573 WRITE_DATA_DST_SEL(0)));
5574 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5575 radeon_ring_write(ring, 0);
5576 radeon_ring_write(ring, 1 << vm->id);
5577
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5578 /* compute doesn't have PFP */
5579 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5580 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5581 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5582 radeon_ring_write(ring, 0x0);
5583 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5584}
5585
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5586/*
5587 * RLC
5588 * The RLC is a multi-purpose microengine that handles a
5589 * variety of functions, the most important of which is
5590 * the interrupt controller.
5591 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5592static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5593 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5594{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5595 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5596
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5597 if (enable)
5598 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5599 else
5600 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5601 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5602}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5603
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5604static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5605{
5606 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5607
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5608 tmp = RREG32(RLC_LB_CNTL);
5609 if (enable)
5610 tmp |= LOAD_BALANCE_ENABLE;
5611 else
5612 tmp &= ~LOAD_BALANCE_ENABLE;
5613 WREG32(RLC_LB_CNTL, tmp);
5614}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5615
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5616static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5617{
5618 u32 i, j, k;
5619 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5620
5621 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5622 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5623 cik_select_se_sh(rdev, i, j);
5624 for (k = 0; k < rdev->usec_timeout; k++) {
5625 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5626 break;
5627 udelay(1);
5628 }
5629 }
5630 }
5631 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5632
5633 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5634 for (k = 0; k < rdev->usec_timeout; k++) {
5635 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5636 break;
5637 udelay(1);
5638 }
5639}
5640
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5641static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5642{
5643 u32 tmp;
5644
5645 tmp = RREG32(RLC_CNTL);
5646 if (tmp != rlc)
5647 WREG32(RLC_CNTL, rlc);
5648}
5649
5650static u32 cik_halt_rlc(struct radeon_device *rdev)
5651{
5652 u32 data, orig;
5653
5654 orig = data = RREG32(RLC_CNTL);
5655
5656 if (data & RLC_ENABLE) {
5657 u32 i;
5658
5659 data &= ~RLC_ENABLE;
5660 WREG32(RLC_CNTL, data);
5661
5662 for (i = 0; i < rdev->usec_timeout; i++) {
5663 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5664 break;
5665 udelay(1);
5666 }
5667
5668 cik_wait_for_rlc_serdes(rdev);
5669 }
5670
5671 return orig;
5672}
5673
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5674void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5675{
5676 u32 tmp, i, mask;
5677
5678 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5679 WREG32(RLC_GPR_REG2, tmp);
5680
5681 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5682 for (i = 0; i < rdev->usec_timeout; i++) {
5683 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5684 break;
5685 udelay(1);
5686 }
5687
5688 for (i = 0; i < rdev->usec_timeout; i++) {
5689 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5690 break;
5691 udelay(1);
5692 }
5693}
5694
5695void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5696{
5697 u32 tmp;
5698
5699 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5700 WREG32(RLC_GPR_REG2, tmp);
5701}
5702
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5703/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5704 * cik_rlc_stop - stop the RLC ME
5705 *
5706 * @rdev: radeon_device pointer
5707 *
5708 * Halt the RLC ME (MicroEngine) (CIK).
5709 */
5710static void cik_rlc_stop(struct radeon_device *rdev)
5711{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5712 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5713
5714 cik_enable_gui_idle_interrupt(rdev, false);
5715
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5716 cik_wait_for_rlc_serdes(rdev);
5717}
5718
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5719/**
5720 * cik_rlc_start - start the RLC ME
5721 *
5722 * @rdev: radeon_device pointer
5723 *
5724 * Unhalt the RLC ME (MicroEngine) (CIK).
5725 */
5726static void cik_rlc_start(struct radeon_device *rdev)
5727{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5728 WREG32(RLC_CNTL, RLC_ENABLE);
5729
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5730 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5731
5732 udelay(50);
5733}
5734
5735/**
5736 * cik_rlc_resume - setup the RLC hw
5737 *
5738 * @rdev: radeon_device pointer
5739 *
5740 * Initialize the RLC registers, load the ucode,
5741 * and start the RLC (CIK).
5742 * Returns 0 for success, -EINVAL if the ucode is not available.
5743 */
5744static int cik_rlc_resume(struct radeon_device *rdev)
5745{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5746 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5747 const __be32 *fw_data;
5748
5749 if (!rdev->rlc_fw)
5750 return -EINVAL;
5751
5752 switch (rdev->family) {
5753 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5754 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5755 default:
5756 size = BONAIRE_RLC_UCODE_SIZE;
5757 break;
5758 case CHIP_KAVERI:
5759 size = KV_RLC_UCODE_SIZE;
5760 break;
5761 case CHIP_KABINI:
5762 size = KB_RLC_UCODE_SIZE;
5763 break;
5764 }
5765
5766 cik_rlc_stop(rdev);
5767
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5768 /* disable CG */
5769 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5770 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5771
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5772 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5773
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5774 cik_init_pg(rdev);
5775
5776 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5777
5778 WREG32(RLC_LB_CNTR_INIT, 0);
5779 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5780
5781 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5782 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5783 WREG32(RLC_LB_PARAMS, 0x00600408);
5784 WREG32(RLC_LB_CNTL, 0x80000004);
5785
5786 WREG32(RLC_MC_CNTL, 0);
5787 WREG32(RLC_UCODE_CNTL, 0);
5788
5789 fw_data = (const __be32 *)rdev->rlc_fw->data;
5790 WREG32(RLC_GPM_UCODE_ADDR, 0);
5791 for (i = 0; i < size; i++)
5792 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5793 WREG32(RLC_GPM_UCODE_ADDR, 0);
5794
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5795 /* XXX - find out what chips support lbpw */
5796 cik_enable_lbpw(rdev, false);
5797
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5798 if (rdev->family == CHIP_BONAIRE)
5799 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5800
5801 cik_rlc_start(rdev);
5802
5803 return 0;
5804}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5805
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5806static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5807{
5808 u32 data, orig, tmp, tmp2;
5809
5810 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5811
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5812 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5813 cik_enable_gui_idle_interrupt(rdev, true);
5814
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5815 tmp = cik_halt_rlc(rdev);
5816
5817 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5818 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5819 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5820 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5821 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5822
5823 cik_update_rlc(rdev, tmp);
5824
5825 data |= CGCG_EN | CGLS_EN;
5826 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5827 cik_enable_gui_idle_interrupt(rdev, false);
5828
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5829 RREG32(CB_CGTT_SCLK_CTRL);
5830 RREG32(CB_CGTT_SCLK_CTRL);
5831 RREG32(CB_CGTT_SCLK_CTRL);
5832 RREG32(CB_CGTT_SCLK_CTRL);
5833
5834 data &= ~(CGCG_EN | CGLS_EN);
5835 }
5836
5837 if (orig != data)
5838 WREG32(RLC_CGCG_CGLS_CTRL, data);
5839
5840}
5841
5842static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5843{
5844 u32 data, orig, tmp = 0;
5845
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5846 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5847 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5848 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5849 orig = data = RREG32(CP_MEM_SLP_CNTL);
5850 data |= CP_MEM_LS_EN;
5851 if (orig != data)
5852 WREG32(CP_MEM_SLP_CNTL, data);
5853 }
5854 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5855
5856 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5857 data &= 0xfffffffd;
5858 if (orig != data)
5859 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5860
5861 tmp = cik_halt_rlc(rdev);
5862
5863 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5864 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5865 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5866 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5867 WREG32(RLC_SERDES_WR_CTRL, data);
5868
5869 cik_update_rlc(rdev, tmp);
5870
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5871 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5872 orig = data = RREG32(CGTS_SM_CTRL_REG);
5873 data &= ~SM_MODE_MASK;
5874 data |= SM_MODE(0x2);
5875 data |= SM_MODE_ENABLE;
5876 data &= ~CGTS_OVERRIDE;
5877 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5878 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5879 data &= ~CGTS_LS_OVERRIDE;
5880 data &= ~ON_MONITOR_ADD_MASK;
5881 data |= ON_MONITOR_ADD_EN;
5882 data |= ON_MONITOR_ADD(0x96);
5883 if (orig != data)
5884 WREG32(CGTS_SM_CTRL_REG, data);
5885 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5886 } else {
5887 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5888 data |= 0x00000002;
5889 if (orig != data)
5890 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5891
5892 data = RREG32(RLC_MEM_SLP_CNTL);
5893 if (data & RLC_MEM_LS_EN) {
5894 data &= ~RLC_MEM_LS_EN;
5895 WREG32(RLC_MEM_SLP_CNTL, data);
5896 }
5897
5898 data = RREG32(CP_MEM_SLP_CNTL);
5899 if (data & CP_MEM_LS_EN) {
5900 data &= ~CP_MEM_LS_EN;
5901 WREG32(CP_MEM_SLP_CNTL, data);
5902 }
5903
5904 orig = data = RREG32(CGTS_SM_CTRL_REG);
5905 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5906 if (orig != data)
5907 WREG32(CGTS_SM_CTRL_REG, data);
5908
5909 tmp = cik_halt_rlc(rdev);
5910
5911 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5912 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5913 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5914 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5915 WREG32(RLC_SERDES_WR_CTRL, data);
5916
5917 cik_update_rlc(rdev, tmp);
5918 }
5919}
5920
5921static const u32 mc_cg_registers[] =
5922{
5923 MC_HUB_MISC_HUB_CG,
5924 MC_HUB_MISC_SIP_CG,
5925 MC_HUB_MISC_VM_CG,
5926 MC_XPB_CLK_GAT,
5927 ATC_MISC_CG,
5928 MC_CITF_MISC_WR_CG,
5929 MC_CITF_MISC_RD_CG,
5930 MC_CITF_MISC_VM_CG,
5931 VM_L2_CG,
5932};
5933
5934static void cik_enable_mc_ls(struct radeon_device *rdev,
5935 bool enable)
5936{
5937 int i;
5938 u32 orig, data;
5939
5940 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5941 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5942 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5943 data |= MC_LS_ENABLE;
5944 else
5945 data &= ~MC_LS_ENABLE;
5946 if (data != orig)
5947 WREG32(mc_cg_registers[i], data);
5948 }
5949}
5950
5951static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5952 bool enable)
5953{
5954 int i;
5955 u32 orig, data;
5956
5957 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5958 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5959 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5960 data |= MC_CG_ENABLE;
5961 else
5962 data &= ~MC_CG_ENABLE;
5963 if (data != orig)
5964 WREG32(mc_cg_registers[i], data);
5965 }
5966}
5967
5968static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5969 bool enable)
5970{
5971 u32 orig, data;
5972
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5973 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5974 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5975 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5976 } else {
5977 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5978 data |= 0xff000000;
5979 if (data != orig)
5980 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5981
5982 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5983 data |= 0xff000000;
5984 if (data != orig)
5985 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5986 }
5987}
5988
5989static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5990 bool enable)
5991{
5992 u32 orig, data;
5993
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5994 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5995 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5996 data |= 0x100;
5997 if (orig != data)
5998 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5999
6000 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6001 data |= 0x100;
6002 if (orig != data)
6003 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6004 } else {
6005 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6006 data &= ~0x100;
6007 if (orig != data)
6008 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6009
6010 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6011 data &= ~0x100;
6012 if (orig != data)
6013 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6014 }
6015}
6016
6017static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6018 bool enable)
6019{
6020 u32 orig, data;
6021
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6022 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6023 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6024 data = 0xfff;
6025 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6026
6027 orig = data = RREG32(UVD_CGC_CTRL);
6028 data |= DCM;
6029 if (orig != data)
6030 WREG32(UVD_CGC_CTRL, data);
6031 } else {
6032 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6033 data &= ~0xfff;
6034 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6035
6036 orig = data = RREG32(UVD_CGC_CTRL);
6037 data &= ~DCM;
6038 if (orig != data)
6039 WREG32(UVD_CGC_CTRL, data);
6040 }
6041}
6042
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6043static void cik_enable_bif_mgls(struct radeon_device *rdev,
6044 bool enable)
6045{
6046 u32 orig, data;
6047
6048 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6049
6050 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6051 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6052 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6053 else
6054 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6055 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6056
6057 if (orig != data)
6058 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6059}
6060
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6061static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6062 bool enable)
6063{
6064 u32 orig, data;
6065
6066 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6067
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6068 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6069 data &= ~CLOCK_GATING_DIS;
6070 else
6071 data |= CLOCK_GATING_DIS;
6072
6073 if (orig != data)
6074 WREG32(HDP_HOST_PATH_CNTL, data);
6075}
6076
6077static void cik_enable_hdp_ls(struct radeon_device *rdev,
6078 bool enable)
6079{
6080 u32 orig, data;
6081
6082 orig = data = RREG32(HDP_MEM_POWER_LS);
6083
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6084 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6085 data |= HDP_LS_ENABLE;
6086 else
6087 data &= ~HDP_LS_ENABLE;
6088
6089 if (orig != data)
6090 WREG32(HDP_MEM_POWER_LS, data);
6091}
6092
6093void cik_update_cg(struct radeon_device *rdev,
6094 u32 block, bool enable)
6095{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6096
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6097 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6098 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6099 /* order matters! */
6100 if (enable) {
6101 cik_enable_mgcg(rdev, true);
6102 cik_enable_cgcg(rdev, true);
6103 } else {
6104 cik_enable_cgcg(rdev, false);
6105 cik_enable_mgcg(rdev, false);
6106 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6107 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6108 }
6109
6110 if (block & RADEON_CG_BLOCK_MC) {
6111 if (!(rdev->flags & RADEON_IS_IGP)) {
6112 cik_enable_mc_mgcg(rdev, enable);
6113 cik_enable_mc_ls(rdev, enable);
6114 }
6115 }
6116
6117 if (block & RADEON_CG_BLOCK_SDMA) {
6118 cik_enable_sdma_mgcg(rdev, enable);
6119 cik_enable_sdma_mgls(rdev, enable);
6120 }
6121
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6122 if (block & RADEON_CG_BLOCK_BIF) {
6123 cik_enable_bif_mgls(rdev, enable);
6124 }
6125
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6126 if (block & RADEON_CG_BLOCK_UVD) {
6127 if (rdev->has_uvd)
6128 cik_enable_uvd_mgcg(rdev, enable);
6129 }
6130
6131 if (block & RADEON_CG_BLOCK_HDP) {
6132 cik_enable_hdp_mgcg(rdev, enable);
6133 cik_enable_hdp_ls(rdev, enable);
6134 }
6135}
6136
6137static void cik_init_cg(struct radeon_device *rdev)
6138{
6139
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]6140 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6141
6142 if (rdev->has_uvd)
6143 si_init_uvd_internal_cg(rdev);
6144
6145 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6146 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6147 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6148 RADEON_CG_BLOCK_UVD |
6149 RADEON_CG_BLOCK_HDP), true);
6150}
6151
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6152static void cik_fini_cg(struct radeon_device *rdev)
6153{
6154 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6155 RADEON_CG_BLOCK_SDMA |
6156 RADEON_CG_BLOCK_BIF |
6157 RADEON_CG_BLOCK_UVD |
6158 RADEON_CG_BLOCK_HDP), false);
6159
6160 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6161}
6162
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6163static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6164 bool enable)
6165{
6166 u32 data, orig;
6167
6168 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6169 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6170 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6171 else
6172 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6173 if (orig != data)
6174 WREG32(RLC_PG_CNTL, data);
6175}
6176
6177static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6178 bool enable)
6179{
6180 u32 data, orig;
6181
6182 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6183 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6184 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6185 else
6186 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6187 if (orig != data)
6188 WREG32(RLC_PG_CNTL, data);
6189}
6190
6191static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6192{
6193 u32 data, orig;
6194
6195 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6196 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6197 data &= ~DISABLE_CP_PG;
6198 else
6199 data |= DISABLE_CP_PG;
6200 if (orig != data)
6201 WREG32(RLC_PG_CNTL, data);
6202}
6203
6204static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6205{
6206 u32 data, orig;
6207
6208 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6209 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6210 data &= ~DISABLE_GDS_PG;
6211 else
6212 data |= DISABLE_GDS_PG;
6213 if (orig != data)
6214 WREG32(RLC_PG_CNTL, data);
6215}
6216
6217#define CP_ME_TABLE_SIZE 96
6218#define CP_ME_TABLE_OFFSET 2048
6219#define CP_MEC_TABLE_OFFSET 4096
6220
6221void cik_init_cp_pg_table(struct radeon_device *rdev)
6222{
6223 const __be32 *fw_data;
6224 volatile u32 *dst_ptr;
6225 int me, i, max_me = 4;
6226 u32 bo_offset = 0;
6227 u32 table_offset;
6228
6229 if (rdev->family == CHIP_KAVERI)
6230 max_me = 5;
6231
6232 if (rdev->rlc.cp_table_ptr == NULL)
6233 return;
6234
6235 /* write the cp table buffer */
6236 dst_ptr = rdev->rlc.cp_table_ptr;
6237 for (me = 0; me < max_me; me++) {
6238 if (me == 0) {
6239 fw_data = (const __be32 *)rdev->ce_fw->data;
6240 table_offset = CP_ME_TABLE_OFFSET;
6241 } else if (me == 1) {
6242 fw_data = (const __be32 *)rdev->pfp_fw->data;
6243 table_offset = CP_ME_TABLE_OFFSET;
6244 } else if (me == 2) {
6245 fw_data = (const __be32 *)rdev->me_fw->data;
6246 table_offset = CP_ME_TABLE_OFFSET;
6247 } else {
6248 fw_data = (const __be32 *)rdev->mec_fw->data;
6249 table_offset = CP_MEC_TABLE_OFFSET;
6250 }
6251
6252 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6253 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6254 }
6255 bo_offset += CP_ME_TABLE_SIZE;
6256 }
6257}
6258
6259static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6260 bool enable)
6261{
6262 u32 data, orig;
6263
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6264 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6265 orig = data = RREG32(RLC_PG_CNTL);
6266 data |= GFX_PG_ENABLE;
6267 if (orig != data)
6268 WREG32(RLC_PG_CNTL, data);
6269
6270 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6271 data |= AUTO_PG_EN;
6272 if (orig != data)
6273 WREG32(RLC_AUTO_PG_CTRL, data);
6274 } else {
6275 orig = data = RREG32(RLC_PG_CNTL);
6276 data &= ~GFX_PG_ENABLE;
6277 if (orig != data)
6278 WREG32(RLC_PG_CNTL, data);
6279
6280 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6281 data &= ~AUTO_PG_EN;
6282 if (orig != data)
6283 WREG32(RLC_AUTO_PG_CTRL, data);
6284
6285 data = RREG32(DB_RENDER_CONTROL);
6286 }
6287}
6288
6289static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6290{
6291 u32 mask = 0, tmp, tmp1;
6292 int i;
6293
6294 cik_select_se_sh(rdev, se, sh);
6295 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6296 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6297 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6298
6299 tmp &= 0xffff0000;
6300
6301 tmp |= tmp1;
6302 tmp >>= 16;
6303
6304 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6305 mask <<= 1;
6306 mask |= 1;
6307 }
6308
6309 return (~tmp) & mask;
6310}
6311
6312static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6313{
6314 u32 i, j, k, active_cu_number = 0;
6315 u32 mask, counter, cu_bitmap;
6316 u32 tmp = 0;
6317
6318 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6319 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6320 mask = 1;
6321 cu_bitmap = 0;
6322 counter = 0;
6323 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6324 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6325 if (counter < 2)
6326 cu_bitmap |= mask;
6327 counter ++;
6328 }
6329 mask <<= 1;
6330 }
6331
6332 active_cu_number += counter;
6333 tmp |= (cu_bitmap << (i * 16 + j * 8));
6334 }
6335 }
6336
6337 WREG32(RLC_PG_AO_CU_MASK, tmp);
6338
6339 tmp = RREG32(RLC_MAX_PG_CU);
6340 tmp &= ~MAX_PU_CU_MASK;
6341 tmp |= MAX_PU_CU(active_cu_number);
6342 WREG32(RLC_MAX_PG_CU, tmp);
6343}
6344
6345static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6346 bool enable)
6347{
6348 u32 data, orig;
6349
6350 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6351 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6352 data |= STATIC_PER_CU_PG_ENABLE;
6353 else
6354 data &= ~STATIC_PER_CU_PG_ENABLE;
6355 if (orig != data)
6356 WREG32(RLC_PG_CNTL, data);
6357}
6358
6359static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6360 bool enable)
6361{
6362 u32 data, orig;
6363
6364 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6365 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6366 data |= DYN_PER_CU_PG_ENABLE;
6367 else
6368 data &= ~DYN_PER_CU_PG_ENABLE;
6369 if (orig != data)
6370 WREG32(RLC_PG_CNTL, data);
6371}
6372
6373#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6374#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6375
6376static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6377{
6378 u32 data, orig;
6379 u32 i;
6380
6381 if (rdev->rlc.cs_data) {
6382 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6383 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6384 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6385 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6386 } else {
6387 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6388 for (i = 0; i < 3; i++)
6389 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6390 }
6391 if (rdev->rlc.reg_list) {
6392 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6393 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6394 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6395 }
6396
6397 orig = data = RREG32(RLC_PG_CNTL);
6398 data |= GFX_PG_SRC;
6399 if (orig != data)
6400 WREG32(RLC_PG_CNTL, data);
6401
6402 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6403 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6404
6405 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6406 data &= ~IDLE_POLL_COUNT_MASK;
6407 data |= IDLE_POLL_COUNT(0x60);
6408 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6409
6410 data = 0x10101010;
6411 WREG32(RLC_PG_DELAY, data);
6412
6413 data = RREG32(RLC_PG_DELAY_2);
6414 data &= ~0xff;
6415 data |= 0x3;
6416 WREG32(RLC_PG_DELAY_2, data);
6417
6418 data = RREG32(RLC_AUTO_PG_CTRL);
6419 data &= ~GRBM_REG_SGIT_MASK;
6420 data |= GRBM_REG_SGIT(0x700);
6421 WREG32(RLC_AUTO_PG_CTRL, data);
6422
6423}
6424
6425static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6426{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6427 cik_enable_gfx_cgpg(rdev, enable);
6428 cik_enable_gfx_static_mgpg(rdev, enable);
6429 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6430}
6431
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6432u32 cik_get_csb_size(struct radeon_device *rdev)
6433{
6434 u32 count = 0;
6435 const struct cs_section_def *sect = NULL;
6436 const struct cs_extent_def *ext = NULL;
6437
6438 if (rdev->rlc.cs_data == NULL)
6439 return 0;
6440
6441 /* begin clear state */
6442 count += 2;
6443 /* context control state */
6444 count += 3;
6445
6446 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6447 for (ext = sect->section; ext->extent != NULL; ++ext) {
6448 if (sect->id == SECT_CONTEXT)
6449 count += 2 + ext->reg_count;
6450 else
6451 return 0;
6452 }
6453 }
6454 /* pa_sc_raster_config/pa_sc_raster_config1 */
6455 count += 4;
6456 /* end clear state */
6457 count += 2;
6458 /* clear state */
6459 count += 2;
6460
6461 return count;
6462}
6463
6464void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6465{
6466 u32 count = 0, i;
6467 const struct cs_section_def *sect = NULL;
6468 const struct cs_extent_def *ext = NULL;
6469
6470 if (rdev->rlc.cs_data == NULL)
6471 return;
6472 if (buffer == NULL)
6473 return;
6474
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6475 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6476 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6477
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6478 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6479 buffer[count++] = cpu_to_le32(0x80000000);
6480 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6481
6482 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6483 for (ext = sect->section; ext->extent != NULL; ++ext) {
6484 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6485 buffer[count++] =
6486 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6487 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6488 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6489 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6490 } else {
6491 return;
6492 }
6493 }
6494 }
6495
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6496 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6497 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6498 switch (rdev->family) {
6499 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6500 buffer[count++] = cpu_to_le32(0x16000012);
6501 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6502 break;
6503 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6504 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6505 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6506 break;
6507 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6508 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6509 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6510 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame]6511 case CHIP_HAWAII:
6512 buffer[count++] = 0x3a00161a;
6513 buffer[count++] = 0x0000002e;
6514 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6515 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6516 buffer[count++] = cpu_to_le32(0x00000000);
6517 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6518 break;
6519 }
6520
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6521 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6522 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6523
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6524 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6525 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6526}
6527
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6528static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6529{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6530 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6531 cik_enable_sck_slowdown_on_pu(rdev, true);
6532 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6533 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6534 cik_init_gfx_cgpg(rdev);
6535 cik_enable_cp_pg(rdev, true);
6536 cik_enable_gds_pg(rdev, true);
6537 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6538 cik_init_ao_cu_mask(rdev);
6539 cik_update_gfx_pg(rdev, true);
6540 }
6541}
6542
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6543static void cik_fini_pg(struct radeon_device *rdev)
6544{
6545 if (rdev->pg_flags) {
6546 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6547 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6548 cik_enable_cp_pg(rdev, false);
6549 cik_enable_gds_pg(rdev, false);
6550 }
6551 }
6552}
6553
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6554/*
6555 * Interrupts
6556 * Starting with r6xx, interrupts are handled via a ring buffer.
6557 * Ring buffers are areas of GPU accessible memory that the GPU
6558 * writes interrupt vectors into and the host reads vectors out of.
6559 * There is a rptr (read pointer) that determines where the
6560 * host is currently reading, and a wptr (write pointer)
6561 * which determines where the GPU has written. When the
6562 * pointers are equal, the ring is idle. When the GPU
6563 * writes vectors to the ring buffer, it increments the
6564 * wptr. When there is an interrupt, the host then starts
6565 * fetching commands and processing them until the pointers are
6566 * equal again at which point it updates the rptr.
6567 */
6568
6569/**
6570 * cik_enable_interrupts - Enable the interrupt ring buffer
6571 *
6572 * @rdev: radeon_device pointer
6573 *
6574 * Enable the interrupt ring buffer (CIK).
6575 */
6576static void cik_enable_interrupts(struct radeon_device *rdev)
6577{
6578 u32 ih_cntl = RREG32(IH_CNTL);
6579 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6580
6581 ih_cntl |= ENABLE_INTR;
6582 ih_rb_cntl |= IH_RB_ENABLE;
6583 WREG32(IH_CNTL, ih_cntl);
6584 WREG32(IH_RB_CNTL, ih_rb_cntl);
6585 rdev->ih.enabled = true;
6586}
6587
6588/**
6589 * cik_disable_interrupts - Disable the interrupt ring buffer
6590 *
6591 * @rdev: radeon_device pointer
6592 *
6593 * Disable the interrupt ring buffer (CIK).
6594 */
6595static void cik_disable_interrupts(struct radeon_device *rdev)
6596{
6597 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6598 u32 ih_cntl = RREG32(IH_CNTL);
6599
6600 ih_rb_cntl &= ~IH_RB_ENABLE;
6601 ih_cntl &= ~ENABLE_INTR;
6602 WREG32(IH_RB_CNTL, ih_rb_cntl);
6603 WREG32(IH_CNTL, ih_cntl);
6604 /* set rptr, wptr to 0 */
6605 WREG32(IH_RB_RPTR, 0);
6606 WREG32(IH_RB_WPTR, 0);
6607 rdev->ih.enabled = false;
6608 rdev->ih.rptr = 0;
6609}
6610
6611/**
6612 * cik_disable_interrupt_state - Disable all interrupt sources
6613 *
6614 * @rdev: radeon_device pointer
6615 *
6616 * Clear all interrupt enable bits used by the driver (CIK).
6617 */
6618static void cik_disable_interrupt_state(struct radeon_device *rdev)
6619{
6620 u32 tmp;
6621
6622 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6623 tmp = RREG32(CP_INT_CNTL_RING0) &
6624 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6625 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6626 /* sdma */
6627 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6628 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6629 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6630 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6631 /* compute queues */
6632 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6633 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6634 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6635 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6636 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6637 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6638 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6639 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6640 /* grbm */
6641 WREG32(GRBM_INT_CNTL, 0);
6642 /* vline/vblank, etc. */
6643 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6644 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6645 if (rdev->num_crtc >= 4) {
6646 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6647 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6648 }
6649 if (rdev->num_crtc >= 6) {
6650 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6651 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6652 }
6653
6654 /* dac hotplug */
6655 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6656
6657 /* digital hotplug */
6658 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6659 WREG32(DC_HPD1_INT_CONTROL, tmp);
6660 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6661 WREG32(DC_HPD2_INT_CONTROL, tmp);
6662 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6663 WREG32(DC_HPD3_INT_CONTROL, tmp);
6664 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6665 WREG32(DC_HPD4_INT_CONTROL, tmp);
6666 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6667 WREG32(DC_HPD5_INT_CONTROL, tmp);
6668 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6669 WREG32(DC_HPD6_INT_CONTROL, tmp);
6670
6671}
6672
6673/**
6674 * cik_irq_init - init and enable the interrupt ring
6675 *
6676 * @rdev: radeon_device pointer
6677 *
6678 * Allocate a ring buffer for the interrupt controller,
6679 * enable the RLC, disable interrupts, enable the IH
6680 * ring buffer and enable it (CIK).
6681 * Called at device load and reume.
6682 * Returns 0 for success, errors for failure.
6683 */
6684static int cik_irq_init(struct radeon_device *rdev)
6685{
6686 int ret = 0;
6687 int rb_bufsz;
6688 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6689
6690 /* allocate ring */
6691 ret = r600_ih_ring_alloc(rdev);
6692 if (ret)
6693 return ret;
6694
6695 /* disable irqs */
6696 cik_disable_interrupts(rdev);
6697
6698 /* init rlc */
6699 ret = cik_rlc_resume(rdev);
6700 if (ret) {
6701 r600_ih_ring_fini(rdev);
6702 return ret;
6703 }
6704
6705 /* setup interrupt control */
6706 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6707 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6708 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6709 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6710 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6711 */
6712 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6713 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6714 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6715 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6716
6717 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6718 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6719
6720 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6721 IH_WPTR_OVERFLOW_CLEAR |
6722 (rb_bufsz << 1));
6723
6724 if (rdev->wb.enabled)
6725 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6726
6727 /* set the writeback address whether it's enabled or not */
6728 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6729 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6730
6731 WREG32(IH_RB_CNTL, ih_rb_cntl);
6732
6733 /* set rptr, wptr to 0 */
6734 WREG32(IH_RB_RPTR, 0);
6735 WREG32(IH_RB_WPTR, 0);
6736
6737 /* Default settings for IH_CNTL (disabled at first) */
6738 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6739 /* RPTR_REARM only works if msi's are enabled */
6740 if (rdev->msi_enabled)
6741 ih_cntl |= RPTR_REARM;
6742 WREG32(IH_CNTL, ih_cntl);
6743
6744 /* force the active interrupt state to all disabled */
6745 cik_disable_interrupt_state(rdev);
6746
6747 pci_set_master(rdev->pdev);
6748
6749 /* enable irqs */
6750 cik_enable_interrupts(rdev);
6751
6752 return ret;
6753}
6754
6755/**
6756 * cik_irq_set - enable/disable interrupt sources
6757 *
6758 * @rdev: radeon_device pointer
6759 *
6760 * Enable interrupt sources on the GPU (vblanks, hpd,
6761 * etc.) (CIK).
6762 * Returns 0 for success, errors for failure.
6763 */
6764int cik_irq_set(struct radeon_device *rdev)
6765{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6766 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6767 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6768 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6769 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6770 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6771 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6772 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6773 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6774
6775 if (!rdev->irq.installed) {
6776 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6777 return -EINVAL;
6778 }
6779 /* don't enable anything if the ih is disabled */
6780 if (!rdev->ih.enabled) {
6781 cik_disable_interrupts(rdev);
6782 /* force the active interrupt state to all disabled */
6783 cik_disable_interrupt_state(rdev);
6784 return 0;
6785 }
6786
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6787 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6788 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6789 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6790
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6791 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6792 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6793 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6794 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6795 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6796 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6797
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6798 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6799 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6800
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6801 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6802 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6803 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6804 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6805 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6806 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6807 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6808 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6809
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6810 if (rdev->flags & RADEON_IS_IGP)
6811 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6812 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6813 else
6814 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6815 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6816
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6817 /* enable CP interrupts on all rings */
6818 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6819 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6820 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6821 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6822 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6823 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6824 DRM_DEBUG("si_irq_set: sw int cp1\n");
6825 if (ring->me == 1) {
6826 switch (ring->pipe) {
6827 case 0:
6828 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6829 break;
6830 case 1:
6831 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6832 break;
6833 case 2:
6834 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6835 break;
6836 case 3:
6837 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6838 break;
6839 default:
6840 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6841 break;
6842 }
6843 } else if (ring->me == 2) {
6844 switch (ring->pipe) {
6845 case 0:
6846 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6847 break;
6848 case 1:
6849 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6850 break;
6851 case 2:
6852 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6853 break;
6854 case 3:
6855 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6856 break;
6857 default:
6858 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6859 break;
6860 }
6861 } else {
6862 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6863 }
6864 }
6865 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6866 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6867 DRM_DEBUG("si_irq_set: sw int cp2\n");
6868 if (ring->me == 1) {
6869 switch (ring->pipe) {
6870 case 0:
6871 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6872 break;
6873 case 1:
6874 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6875 break;
6876 case 2:
6877 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6878 break;
6879 case 3:
6880 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6881 break;
6882 default:
6883 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6884 break;
6885 }
6886 } else if (ring->me == 2) {
6887 switch (ring->pipe) {
6888 case 0:
6889 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6890 break;
6891 case 1:
6892 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6893 break;
6894 case 2:
6895 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6896 break;
6897 case 3:
6898 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6899 break;
6900 default:
6901 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6902 break;
6903 }
6904 } else {
6905 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6906 }
6907 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6908
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6909 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6910 DRM_DEBUG("cik_irq_set: sw int dma\n");
6911 dma_cntl |= TRAP_ENABLE;
6912 }
6913
6914 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6915 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6916 dma_cntl1 |= TRAP_ENABLE;
6917 }
6918
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6919 if (rdev->irq.crtc_vblank_int[0] ||
6920 atomic_read(&rdev->irq.pflip[0])) {
6921 DRM_DEBUG("cik_irq_set: vblank 0\n");
6922 crtc1 |= VBLANK_INTERRUPT_MASK;
6923 }
6924 if (rdev->irq.crtc_vblank_int[1] ||
6925 atomic_read(&rdev->irq.pflip[1])) {
6926 DRM_DEBUG("cik_irq_set: vblank 1\n");
6927 crtc2 |= VBLANK_INTERRUPT_MASK;
6928 }
6929 if (rdev->irq.crtc_vblank_int[2] ||
6930 atomic_read(&rdev->irq.pflip[2])) {
6931 DRM_DEBUG("cik_irq_set: vblank 2\n");
6932 crtc3 |= VBLANK_INTERRUPT_MASK;
6933 }
6934 if (rdev->irq.crtc_vblank_int[3] ||
6935 atomic_read(&rdev->irq.pflip[3])) {
6936 DRM_DEBUG("cik_irq_set: vblank 3\n");
6937 crtc4 |= VBLANK_INTERRUPT_MASK;
6938 }
6939 if (rdev->irq.crtc_vblank_int[4] ||
6940 atomic_read(&rdev->irq.pflip[4])) {
6941 DRM_DEBUG("cik_irq_set: vblank 4\n");
6942 crtc5 |= VBLANK_INTERRUPT_MASK;
6943 }
6944 if (rdev->irq.crtc_vblank_int[5] ||
6945 atomic_read(&rdev->irq.pflip[5])) {
6946 DRM_DEBUG("cik_irq_set: vblank 5\n");
6947 crtc6 |= VBLANK_INTERRUPT_MASK;
6948 }
6949 if (rdev->irq.hpd[0]) {
6950 DRM_DEBUG("cik_irq_set: hpd 1\n");
6951 hpd1 |= DC_HPDx_INT_EN;
6952 }
6953 if (rdev->irq.hpd[1]) {
6954 DRM_DEBUG("cik_irq_set: hpd 2\n");
6955 hpd2 |= DC_HPDx_INT_EN;
6956 }
6957 if (rdev->irq.hpd[2]) {
6958 DRM_DEBUG("cik_irq_set: hpd 3\n");
6959 hpd3 |= DC_HPDx_INT_EN;
6960 }
6961 if (rdev->irq.hpd[3]) {
6962 DRM_DEBUG("cik_irq_set: hpd 4\n");
6963 hpd4 |= DC_HPDx_INT_EN;
6964 }
6965 if (rdev->irq.hpd[4]) {
6966 DRM_DEBUG("cik_irq_set: hpd 5\n");
6967 hpd5 |= DC_HPDx_INT_EN;
6968 }
6969 if (rdev->irq.hpd[5]) {
6970 DRM_DEBUG("cik_irq_set: hpd 6\n");
6971 hpd6 |= DC_HPDx_INT_EN;
6972 }
6973
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6974 if (rdev->irq.dpm_thermal) {
6975 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6976 if (rdev->flags & RADEON_IS_IGP)
6977 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6978 else
6979 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6980 }
6981
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6982 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6983
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6984 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6985 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6986
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6987 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6988 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6989 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6990 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6991 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6992 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6993 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6994 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6995
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6996 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6997
6998 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6999 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7000 if (rdev->num_crtc >= 4) {
7001 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7002 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7003 }
7004 if (rdev->num_crtc >= 6) {
7005 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7006 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7007 }
7008
7009 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7010 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7011 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7012 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7013 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7014 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7015
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]7016 if (rdev->flags & RADEON_IS_IGP)
7017 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7018 else
7019 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7020
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7021 return 0;
7022}
7023
7024/**
7025 * cik_irq_ack - ack interrupt sources
7026 *
7027 * @rdev: radeon_device pointer
7028 *
7029 * Ack interrupt sources on the GPU (vblanks, hpd,
7030 * etc.) (CIK). Certain interrupts sources are sw
7031 * generated and do not require an explicit ack.
7032 */
7033static inline void cik_irq_ack(struct radeon_device *rdev)
7034{
7035 u32 tmp;
7036
7037 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7038 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7039 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7040 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7041 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7042 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7043 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7044
7045 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7046 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7047 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7048 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7049 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7050 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7051 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7052 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7053
7054 if (rdev->num_crtc >= 4) {
7055 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7056 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7057 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7058 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7059 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7060 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7061 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7062 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7063 }
7064
7065 if (rdev->num_crtc >= 6) {
7066 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7067 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7068 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7069 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7070 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7071 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7072 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7073 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7074 }
7075
7076 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7077 tmp = RREG32(DC_HPD1_INT_CONTROL);
7078 tmp |= DC_HPDx_INT_ACK;
7079 WREG32(DC_HPD1_INT_CONTROL, tmp);
7080 }
7081 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7082 tmp = RREG32(DC_HPD2_INT_CONTROL);
7083 tmp |= DC_HPDx_INT_ACK;
7084 WREG32(DC_HPD2_INT_CONTROL, tmp);
7085 }
7086 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7087 tmp = RREG32(DC_HPD3_INT_CONTROL);
7088 tmp |= DC_HPDx_INT_ACK;
7089 WREG32(DC_HPD3_INT_CONTROL, tmp);
7090 }
7091 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7092 tmp = RREG32(DC_HPD4_INT_CONTROL);
7093 tmp |= DC_HPDx_INT_ACK;
7094 WREG32(DC_HPD4_INT_CONTROL, tmp);
7095 }
7096 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7097 tmp = RREG32(DC_HPD5_INT_CONTROL);
7098 tmp |= DC_HPDx_INT_ACK;
7099 WREG32(DC_HPD5_INT_CONTROL, tmp);
7100 }
7101 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7102 tmp = RREG32(DC_HPD5_INT_CONTROL);
7103 tmp |= DC_HPDx_INT_ACK;
7104 WREG32(DC_HPD6_INT_CONTROL, tmp);
7105 }
7106}
7107
7108/**
7109 * cik_irq_disable - disable interrupts
7110 *
7111 * @rdev: radeon_device pointer
7112 *
7113 * Disable interrupts on the hw (CIK).
7114 */
7115static void cik_irq_disable(struct radeon_device *rdev)
7116{
7117 cik_disable_interrupts(rdev);
7118 /* Wait and acknowledge irq */
7119 mdelay(1);
7120 cik_irq_ack(rdev);
7121 cik_disable_interrupt_state(rdev);
7122}
7123
7124/**
7125 * cik_irq_disable - disable interrupts for suspend
7126 *
7127 * @rdev: radeon_device pointer
7128 *
7129 * Disable interrupts and stop the RLC (CIK).
7130 * Used for suspend.
7131 */
7132static void cik_irq_suspend(struct radeon_device *rdev)
7133{
7134 cik_irq_disable(rdev);
7135 cik_rlc_stop(rdev);
7136}
7137
7138/**
7139 * cik_irq_fini - tear down interrupt support
7140 *
7141 * @rdev: radeon_device pointer
7142 *
7143 * Disable interrupts on the hw and free the IH ring
7144 * buffer (CIK).
7145 * Used for driver unload.
7146 */
7147static void cik_irq_fini(struct radeon_device *rdev)
7148{
7149 cik_irq_suspend(rdev);
7150 r600_ih_ring_fini(rdev);
7151}
7152
7153/**
7154 * cik_get_ih_wptr - get the IH ring buffer wptr
7155 *
7156 * @rdev: radeon_device pointer
7157 *
7158 * Get the IH ring buffer wptr from either the register
7159 * or the writeback memory buffer (CIK). Also check for
7160 * ring buffer overflow and deal with it.
7161 * Used by cik_irq_process().
7162 * Returns the value of the wptr.
7163 */
7164static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7165{
7166 u32 wptr, tmp;
7167
7168 if (rdev->wb.enabled)
7169 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7170 else
7171 wptr = RREG32(IH_RB_WPTR);
7172
7173 if (wptr & RB_OVERFLOW) {
7174 /* When a ring buffer overflow happen start parsing interrupt
7175 * from the last not overwritten vector (wptr + 16). Hopefully
7176 * this should allow us to catchup.
7177 */
7178 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7179 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7180 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7181 tmp = RREG32(IH_RB_CNTL);
7182 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7183 WREG32(IH_RB_CNTL, tmp);
7184 }
7185 return (wptr & rdev->ih.ptr_mask);
7186}
7187
7188/* CIK IV Ring
7189 * Each IV ring entry is 128 bits:
7190 * [7:0] - interrupt source id
7191 * [31:8] - reserved
7192 * [59:32] - interrupt source data
7193 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7194 * [71:64] - RINGID
7195 * CP:
7196 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7197 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7198 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7199 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7200 * PIPE_ID - ME0 0=3D
7201 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7202 * SDMA:
7203 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7204 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7205 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7206 * [79:72] - VMID
7207 * [95:80] - PASID
7208 * [127:96] - reserved
7209 */
7210/**
7211 * cik_irq_process - interrupt handler
7212 *
7213 * @rdev: radeon_device pointer
7214 *
7215 * Interrupt hander (CIK). Walk the IH ring,
7216 * ack interrupts and schedule work to handle
7217 * interrupt events.
7218 * Returns irq process return code.
7219 */
7220int cik_irq_process(struct radeon_device *rdev)
7221{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7222 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7223 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7224 u32 wptr;
7225 u32 rptr;
7226 u32 src_id, src_data, ring_id;
7227 u8 me_id, pipe_id, queue_id;
7228 u32 ring_index;
7229 bool queue_hotplug = false;
7230 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7231 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7232 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7233
7234 if (!rdev->ih.enabled || rdev->shutdown)
7235 return IRQ_NONE;
7236
7237 wptr = cik_get_ih_wptr(rdev);
7238
7239restart_ih:
7240 /* is somebody else already processing irqs? */
7241 if (atomic_xchg(&rdev->ih.lock, 1))
7242 return IRQ_NONE;
7243
7244 rptr = rdev->ih.rptr;
7245 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7246
7247 /* Order reading of wptr vs. reading of IH ring data */
7248 rmb();
7249
7250 /* display interrupts */
7251 cik_irq_ack(rdev);
7252
7253 while (rptr != wptr) {
7254 /* wptr/rptr are in bytes! */
7255 ring_index = rptr / 4;
7256 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7257 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7258 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7259
7260 switch (src_id) {
7261 case 1: /* D1 vblank/vline */
7262 switch (src_data) {
7263 case 0: /* D1 vblank */
7264 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7265 if (rdev->irq.crtc_vblank_int[0]) {
7266 drm_handle_vblank(rdev->ddev, 0);
7267 rdev->pm.vblank_sync = true;
7268 wake_up(&rdev->irq.vblank_queue);
7269 }
7270 if (atomic_read(&rdev->irq.pflip[0]))
7271 radeon_crtc_handle_flip(rdev, 0);
7272 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7273 DRM_DEBUG("IH: D1 vblank\n");
7274 }
7275 break;
7276 case 1: /* D1 vline */
7277 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7278 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7279 DRM_DEBUG("IH: D1 vline\n");
7280 }
7281 break;
7282 default:
7283 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7284 break;
7285 }
7286 break;
7287 case 2: /* D2 vblank/vline */
7288 switch (src_data) {
7289 case 0: /* D2 vblank */
7290 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7291 if (rdev->irq.crtc_vblank_int[1]) {
7292 drm_handle_vblank(rdev->ddev, 1);
7293 rdev->pm.vblank_sync = true;
7294 wake_up(&rdev->irq.vblank_queue);
7295 }
7296 if (atomic_read(&rdev->irq.pflip[1]))
7297 radeon_crtc_handle_flip(rdev, 1);
7298 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7299 DRM_DEBUG("IH: D2 vblank\n");
7300 }
7301 break;
7302 case 1: /* D2 vline */
7303 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7304 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7305 DRM_DEBUG("IH: D2 vline\n");
7306 }
7307 break;
7308 default:
7309 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7310 break;
7311 }
7312 break;
7313 case 3: /* D3 vblank/vline */
7314 switch (src_data) {
7315 case 0: /* D3 vblank */
7316 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7317 if (rdev->irq.crtc_vblank_int[2]) {
7318 drm_handle_vblank(rdev->ddev, 2);
7319 rdev->pm.vblank_sync = true;
7320 wake_up(&rdev->irq.vblank_queue);
7321 }
7322 if (atomic_read(&rdev->irq.pflip[2]))
7323 radeon_crtc_handle_flip(rdev, 2);
7324 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7325 DRM_DEBUG("IH: D3 vblank\n");
7326 }
7327 break;
7328 case 1: /* D3 vline */
7329 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7330 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7331 DRM_DEBUG("IH: D3 vline\n");
7332 }
7333 break;
7334 default:
7335 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7336 break;
7337 }
7338 break;
7339 case 4: /* D4 vblank/vline */
7340 switch (src_data) {
7341 case 0: /* D4 vblank */
7342 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7343 if (rdev->irq.crtc_vblank_int[3]) {
7344 drm_handle_vblank(rdev->ddev, 3);
7345 rdev->pm.vblank_sync = true;
7346 wake_up(&rdev->irq.vblank_queue);
7347 }
7348 if (atomic_read(&rdev->irq.pflip[3]))
7349 radeon_crtc_handle_flip(rdev, 3);
7350 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7351 DRM_DEBUG("IH: D4 vblank\n");
7352 }
7353 break;
7354 case 1: /* D4 vline */
7355 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7356 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7357 DRM_DEBUG("IH: D4 vline\n");
7358 }
7359 break;
7360 default:
7361 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7362 break;
7363 }
7364 break;
7365 case 5: /* D5 vblank/vline */
7366 switch (src_data) {
7367 case 0: /* D5 vblank */
7368 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7369 if (rdev->irq.crtc_vblank_int[4]) {
7370 drm_handle_vblank(rdev->ddev, 4);
7371 rdev->pm.vblank_sync = true;
7372 wake_up(&rdev->irq.vblank_queue);
7373 }
7374 if (atomic_read(&rdev->irq.pflip[4]))
7375 radeon_crtc_handle_flip(rdev, 4);
7376 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7377 DRM_DEBUG("IH: D5 vblank\n");
7378 }
7379 break;
7380 case 1: /* D5 vline */
7381 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7382 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7383 DRM_DEBUG("IH: D5 vline\n");
7384 }
7385 break;
7386 default:
7387 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7388 break;
7389 }
7390 break;
7391 case 6: /* D6 vblank/vline */
7392 switch (src_data) {
7393 case 0: /* D6 vblank */
7394 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7395 if (rdev->irq.crtc_vblank_int[5]) {
7396 drm_handle_vblank(rdev->ddev, 5);
7397 rdev->pm.vblank_sync = true;
7398 wake_up(&rdev->irq.vblank_queue);
7399 }
7400 if (atomic_read(&rdev->irq.pflip[5]))
7401 radeon_crtc_handle_flip(rdev, 5);
7402 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7403 DRM_DEBUG("IH: D6 vblank\n");
7404 }
7405 break;
7406 case 1: /* D6 vline */
7407 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7408 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7409 DRM_DEBUG("IH: D6 vline\n");
7410 }
7411 break;
7412 default:
7413 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7414 break;
7415 }
7416 break;
7417 case 42: /* HPD hotplug */
7418 switch (src_data) {
7419 case 0:
7420 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7421 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7422 queue_hotplug = true;
7423 DRM_DEBUG("IH: HPD1\n");
7424 }
7425 break;
7426 case 1:
7427 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7428 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7429 queue_hotplug = true;
7430 DRM_DEBUG("IH: HPD2\n");
7431 }
7432 break;
7433 case 2:
7434 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7435 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7436 queue_hotplug = true;
7437 DRM_DEBUG("IH: HPD3\n");
7438 }
7439 break;
7440 case 3:
7441 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7442 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7443 queue_hotplug = true;
7444 DRM_DEBUG("IH: HPD4\n");
7445 }
7446 break;
7447 case 4:
7448 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7449 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7450 queue_hotplug = true;
7451 DRM_DEBUG("IH: HPD5\n");
7452 }
7453 break;
7454 case 5:
7455 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7456 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7457 queue_hotplug = true;
7458 DRM_DEBUG("IH: HPD6\n");
7459 }
7460 break;
7461 default:
7462 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7463 break;
7464 }
7465 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7466 case 124: /* UVD */
7467 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7468 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7469 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7470 case 146:
7471 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7472 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7473 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7474 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7475 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7476 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7477 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7478 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7479 status);
7480 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7481 /* reset addr and status */
7482 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7483 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7484 case 176: /* GFX RB CP_INT */
7485 case 177: /* GFX IB CP_INT */
7486 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7487 break;
7488 case 181: /* CP EOP event */
7489 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7490 /* XXX check the bitfield order! */
7491 me_id = (ring_id & 0x60) >> 5;
7492 pipe_id = (ring_id & 0x18) >> 3;
7493 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7494 switch (me_id) {
7495 case 0:
7496 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7497 break;
7498 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7499 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7500 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7501 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7502 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7503 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7504 break;
7505 }
7506 break;
7507 case 184: /* CP Privileged reg access */
7508 DRM_ERROR("Illegal register access in command stream\n");
7509 /* XXX check the bitfield order! */
7510 me_id = (ring_id & 0x60) >> 5;
7511 pipe_id = (ring_id & 0x18) >> 3;
7512 queue_id = (ring_id & 0x7) >> 0;
7513 switch (me_id) {
7514 case 0:
7515 /* This results in a full GPU reset, but all we need to do is soft
7516 * reset the CP for gfx
7517 */
7518 queue_reset = true;
7519 break;
7520 case 1:
7521 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7522 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7523 break;
7524 case 2:
7525 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7526 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7527 break;
7528 }
7529 break;
7530 case 185: /* CP Privileged inst */
7531 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7532 /* XXX check the bitfield order! */
7533 me_id = (ring_id & 0x60) >> 5;
7534 pipe_id = (ring_id & 0x18) >> 3;
7535 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7536 switch (me_id) {
7537 case 0:
7538 /* This results in a full GPU reset, but all we need to do is soft
7539 * reset the CP for gfx
7540 */
7541 queue_reset = true;
7542 break;
7543 case 1:
7544 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7545 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7546 break;
7547 case 2:
7548 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7549 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7550 break;
7551 }
7552 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7553 case 224: /* SDMA trap event */
7554 /* XXX check the bitfield order! */
7555 me_id = (ring_id & 0x3) >> 0;
7556 queue_id = (ring_id & 0xc) >> 2;
7557 DRM_DEBUG("IH: SDMA trap\n");
7558 switch (me_id) {
7559 case 0:
7560 switch (queue_id) {
7561 case 0:
7562 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7563 break;
7564 case 1:
7565 /* XXX compute */
7566 break;
7567 case 2:
7568 /* XXX compute */
7569 break;
7570 }
7571 break;
7572 case 1:
7573 switch (queue_id) {
7574 case 0:
7575 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7576 break;
7577 case 1:
7578 /* XXX compute */
7579 break;
7580 case 2:
7581 /* XXX compute */
7582 break;
7583 }
7584 break;
7585 }
7586 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7587 case 230: /* thermal low to high */
7588 DRM_DEBUG("IH: thermal low to high\n");
7589 rdev->pm.dpm.thermal.high_to_low = false;
7590 queue_thermal = true;
7591 break;
7592 case 231: /* thermal high to low */
7593 DRM_DEBUG("IH: thermal high to low\n");
7594 rdev->pm.dpm.thermal.high_to_low = true;
7595 queue_thermal = true;
7596 break;
7597 case 233: /* GUI IDLE */
7598 DRM_DEBUG("IH: GUI idle\n");
7599 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7600 case 241: /* SDMA Privileged inst */
7601 case 247: /* SDMA Privileged inst */
7602 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7603 /* XXX check the bitfield order! */
7604 me_id = (ring_id & 0x3) >> 0;
7605 queue_id = (ring_id & 0xc) >> 2;
7606 switch (me_id) {
7607 case 0:
7608 switch (queue_id) {
7609 case 0:
7610 queue_reset = true;
7611 break;
7612 case 1:
7613 /* XXX compute */
7614 queue_reset = true;
7615 break;
7616 case 2:
7617 /* XXX compute */
7618 queue_reset = true;
7619 break;
7620 }
7621 break;
7622 case 1:
7623 switch (queue_id) {
7624 case 0:
7625 queue_reset = true;
7626 break;
7627 case 1:
7628 /* XXX compute */
7629 queue_reset = true;
7630 break;
7631 case 2:
7632 /* XXX compute */
7633 queue_reset = true;
7634 break;
7635 }
7636 break;
7637 }
7638 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7639 default:
7640 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7641 break;
7642 }
7643
7644 /* wptr/rptr are in bytes! */
7645 rptr += 16;
7646 rptr &= rdev->ih.ptr_mask;
7647 }
7648 if (queue_hotplug)
7649 schedule_work(&rdev->hotplug_work);
7650 if (queue_reset)
7651 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7652 if (queue_thermal)
7653 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7654 rdev->ih.rptr = rptr;
7655 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7656 atomic_set(&rdev->ih.lock, 0);
7657
7658 /* make sure wptr hasn't changed while processing */
7659 wptr = cik_get_ih_wptr(rdev);
7660 if (wptr != rptr)
7661 goto restart_ih;
7662
7663 return IRQ_HANDLED;
7664}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7665
7666/*
7667 * startup/shutdown callbacks
7668 */
7669/**
7670 * cik_startup - program the asic to a functional state
7671 *
7672 * @rdev: radeon_device pointer
7673 *
7674 * Programs the asic to a functional state (CIK).
7675 * Called by cik_init() and cik_resume().
7676 * Returns 0 for success, error for failure.
7677 */
7678static int cik_startup(struct radeon_device *rdev)
7679{
7680 struct radeon_ring *ring;
7681 int r;
7682
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7683 /* enable pcie gen2/3 link */
7684 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7685 /* enable aspm */
7686 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7687
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7688 /* scratch needs to be initialized before MC */
7689 r = r600_vram_scratch_init(rdev);
7690 if (r)
7691 return r;
7692
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7693 cik_mc_program(rdev);
7694
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7695 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7696 r = ci_mc_load_microcode(rdev);
7697 if (r) {
7698 DRM_ERROR("Failed to load MC firmware!\n");
7699 return r;
7700 }
7701 }
7702
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7703 r = cik_pcie_gart_enable(rdev);
7704 if (r)
7705 return r;
7706 cik_gpu_init(rdev);
7707
7708 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7709 if (rdev->flags & RADEON_IS_IGP) {
7710 if (rdev->family == CHIP_KAVERI) {
7711 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7712 rdev->rlc.reg_list_size =
7713 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7714 } else {
7715 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7716 rdev->rlc.reg_list_size =
7717 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7718 }
7719 }
7720 rdev->rlc.cs_data = ci_cs_data;
7721 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7722 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7723 if (r) {
7724 DRM_ERROR("Failed to init rlc BOs!\n");
7725 return r;
7726 }
7727
7728 /* allocate wb buffer */
7729 r = radeon_wb_init(rdev);
7730 if (r)
7731 return r;
7732
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7733 /* allocate mec buffers */
7734 r = cik_mec_init(rdev);
7735 if (r) {
7736 DRM_ERROR("Failed to init MEC BOs!\n");
7737 return r;
7738 }
7739
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7740 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7741 if (r) {
7742 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7743 return r;
7744 }
7745
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7746 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7747 if (r) {
7748 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7749 return r;
7750 }
7751
7752 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7753 if (r) {
7754 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7755 return r;
7756 }
7757
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7758 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7759 if (r) {
7760 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7761 return r;
7762 }
7763
7764 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7765 if (r) {
7766 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7767 return r;
7768 }
7769
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7770 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7771 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7772 r = uvd_v4_2_resume(rdev);
7773 if (!r) {
7774 r = radeon_fence_driver_start_ring(rdev,
7775 R600_RING_TYPE_UVD_INDEX);
7776 if (r)
7777 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7778 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7779 }
7780 if (r)
7781 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7782
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7783 /* Enable IRQ */
7784 if (!rdev->irq.installed) {
7785 r = radeon_irq_kms_init(rdev);
7786 if (r)
7787 return r;
7788 }
7789
7790 r = cik_irq_init(rdev);
7791 if (r) {
7792 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7793 radeon_irq_kms_fini(rdev);
7794 return r;
7795 }
7796 cik_irq_set(rdev);
7797
7798 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7799 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7800 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7801 if (r)
7802 return r;
7803
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7804 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7805 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7806 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7807 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7808 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7809 if (r)
7810 return r;
7811 ring->me = 1; /* first MEC */
7812 ring->pipe = 0; /* first pipe */
7813 ring->queue = 0; /* first queue */
7814 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7815
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7816 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7817 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7818 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7819 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7820 if (r)
7821 return r;
7822 /* dGPU only have 1 MEC */
7823 ring->me = 1; /* first MEC */
7824 ring->pipe = 0; /* first pipe */
7825 ring->queue = 1; /* second queue */
7826 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7827
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7828 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7829 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7830 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7831 if (r)
7832 return r;
7833
7834 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7835 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7836 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7837 if (r)
7838 return r;
7839
7840 r = cik_cp_resume(rdev);
7841 if (r)
7842 return r;
7843
7844 r = cik_sdma_resume(rdev);
7845 if (r)
7846 return r;
7847
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7848 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7849 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7850 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7851 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7852 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7853 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7854 if (r)
7855 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7856 }
7857
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7858 r = radeon_ib_pool_init(rdev);
7859 if (r) {
7860 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7861 return r;
7862 }
7863
7864 r = radeon_vm_manager_init(rdev);
7865 if (r) {
7866 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7867 return r;
7868 }
7869
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7870 r = dce6_audio_init(rdev);
7871 if (r)
7872 return r;
7873
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7874 return 0;
7875}
7876
7877/**
7878 * cik_resume - resume the asic to a functional state
7879 *
7880 * @rdev: radeon_device pointer
7881 *
7882 * Programs the asic to a functional state (CIK).
7883 * Called at resume.
7884 * Returns 0 for success, error for failure.
7885 */
7886int cik_resume(struct radeon_device *rdev)
7887{
7888 int r;
7889
7890 /* post card */
7891 atom_asic_init(rdev->mode_info.atom_context);
7892
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7893 /* init golden registers */
7894 cik_init_golden_registers(rdev);
7895
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7896 radeon_pm_resume(rdev);
7897
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7898 rdev->accel_working = true;
7899 r = cik_startup(rdev);
7900 if (r) {
7901 DRM_ERROR("cik startup failed on resume\n");
7902 rdev->accel_working = false;
7903 return r;
7904 }
7905
7906 return r;
7907
7908}
7909
7910/**
7911 * cik_suspend - suspend the asic
7912 *
7913 * @rdev: radeon_device pointer
7914 *
7915 * Bring the chip into a state suitable for suspend (CIK).
7916 * Called at suspend.
7917 * Returns 0 for success.
7918 */
7919int cik_suspend(struct radeon_device *rdev)
7920{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]7921 radeon_pm_suspend(rdev);
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7922 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7923 radeon_vm_manager_fini(rdev);
7924 cik_cp_enable(rdev, false);
7925 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7926 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7927 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7928 cik_fini_pg(rdev);
7929 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7930 cik_irq_suspend(rdev);
7931 radeon_wb_disable(rdev);
7932 cik_pcie_gart_disable(rdev);
7933 return 0;
7934}
7935
7936/* Plan is to move initialization in that function and use
7937 * helper function so that radeon_device_init pretty much
7938 * do nothing more than calling asic specific function. This
7939 * should also allow to remove a bunch of callback function
7940 * like vram_info.
7941 */
7942/**
7943 * cik_init - asic specific driver and hw init
7944 *
7945 * @rdev: radeon_device pointer
7946 *
7947 * Setup asic specific driver variables and program the hw
7948 * to a functional state (CIK).
7949 * Called at driver startup.
7950 * Returns 0 for success, errors for failure.
7951 */
7952int cik_init(struct radeon_device *rdev)
7953{
7954 struct radeon_ring *ring;
7955 int r;
7956
7957 /* Read BIOS */
7958 if (!radeon_get_bios(rdev)) {
7959 if (ASIC_IS_AVIVO(rdev))
7960 return -EINVAL;
7961 }
7962 /* Must be an ATOMBIOS */
7963 if (!rdev->is_atom_bios) {
7964 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7965 return -EINVAL;
7966 }
7967 r = radeon_atombios_init(rdev);
7968 if (r)
7969 return r;
7970
7971 /* Post card if necessary */
7972 if (!radeon_card_posted(rdev)) {
7973 if (!rdev->bios) {
7974 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7975 return -EINVAL;
7976 }
7977 DRM_INFO("GPU not posted. posting now...\n");
7978 atom_asic_init(rdev->mode_info.atom_context);
7979 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7980 /* init golden registers */
7981 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7982 /* Initialize scratch registers */
7983 cik_scratch_init(rdev);
7984 /* Initialize surface registers */
7985 radeon_surface_init(rdev);
7986 /* Initialize clocks */
7987 radeon_get_clock_info(rdev->ddev);
7988
7989 /* Fence driver */
7990 r = radeon_fence_driver_init(rdev);
7991 if (r)
7992 return r;
7993
7994 /* initialize memory controller */
7995 r = cik_mc_init(rdev);
7996 if (r)
7997 return r;
7998 /* Memory manager */
7999 r = radeon_bo_init(rdev);
8000 if (r)
8001 return r;
8002
Alex Deucher01ac8792013-12-18 19:11:27 -0500[diff] [blame]8003 if (rdev->flags & RADEON_IS_IGP) {
8004 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8005 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8006 r = cik_init_microcode(rdev);
8007 if (r) {
8008 DRM_ERROR("Failed to load firmware!\n");
8009 return r;
8010 }
8011 }
8012 } else {
8013 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8014 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8015 !rdev->mc_fw) {
8016 r = cik_init_microcode(rdev);
8017 if (r) {
8018 DRM_ERROR("Failed to load firmware!\n");
8019 return r;
8020 }
8021 }
8022 }
8023
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8024 /* Initialize power management */
8025 radeon_pm_init(rdev);
8026
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8027 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8028 ring->ring_obj = NULL;
8029 r600_ring_init(rdev, ring, 1024 * 1024);
8030
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8031 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8032 ring->ring_obj = NULL;
8033 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8034 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8035 if (r)
8036 return r;
8037
8038 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8039 ring->ring_obj = NULL;
8040 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]8041 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8042 if (r)
8043 return r;
8044
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8045 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8046 ring->ring_obj = NULL;
8047 r600_ring_init(rdev, ring, 256 * 1024);
8048
8049 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8050 ring->ring_obj = NULL;
8051 r600_ring_init(rdev, ring, 256 * 1024);
8052
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8053 r = radeon_uvd_init(rdev);
8054 if (!r) {
8055 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8056 ring->ring_obj = NULL;
8057 r600_ring_init(rdev, ring, 4096);
8058 }
8059
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8060 rdev->ih.ring_obj = NULL;
8061 r600_ih_ring_init(rdev, 64 * 1024);
8062
8063 r = r600_pcie_gart_init(rdev);
8064 if (r)
8065 return r;
8066
8067 rdev->accel_working = true;
8068 r = cik_startup(rdev);
8069 if (r) {
8070 dev_err(rdev->dev, "disabling GPU acceleration\n");
8071 cik_cp_fini(rdev);
8072 cik_sdma_fini(rdev);
8073 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8074 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8075 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8076 radeon_wb_fini(rdev);
8077 radeon_ib_pool_fini(rdev);
8078 radeon_vm_manager_fini(rdev);
8079 radeon_irq_kms_fini(rdev);
8080 cik_pcie_gart_fini(rdev);
8081 rdev->accel_working = false;
8082 }
8083
8084 /* Don't start up if the MC ucode is missing.
8085 * The default clocks and voltages before the MC ucode
8086 * is loaded are not suffient for advanced operations.
8087 */
8088 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8089 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8090 return -EINVAL;
8091 }
8092
8093 return 0;
8094}
8095
8096/**
8097 * cik_fini - asic specific driver and hw fini
8098 *
8099 * @rdev: radeon_device pointer
8100 *
8101 * Tear down the asic specific driver variables and program the hw
8102 * to an idle state (CIK).
8103 * Called at driver unload.
8104 */
8105void cik_fini(struct radeon_device *rdev)
8106{
Alex Deucher6c7bcce2013-12-18 14:07:14 -0500[diff] [blame]8107 radeon_pm_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8108 cik_cp_fini(rdev);
8109 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]8110 cik_fini_pg(rdev);
8111 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8112 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]8113 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]8114 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8115 radeon_wb_fini(rdev);
8116 radeon_vm_manager_fini(rdev);
8117 radeon_ib_pool_fini(rdev);
8118 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]8119 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8120 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]8121 cik_pcie_gart_fini(rdev);
8122 r600_vram_scratch_fini(rdev);
8123 radeon_gem_fini(rdev);
8124 radeon_fence_driver_fini(rdev);
8125 radeon_bo_fini(rdev);
8126 radeon_atombios_fini(rdev);
8127 kfree(rdev->bios);
8128 rdev->bios = NULL;
8129}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8130
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8131void dce8_program_fmt(struct drm_encoder *encoder)
8132{
8133 struct drm_device *dev = encoder->dev;
8134 struct radeon_device *rdev = dev->dev_private;
8135 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8136 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8137 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8138 int bpc = 0;
8139 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8140 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8141
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8142 if (connector) {
8143 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8144 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8145 dither = radeon_connector->dither;
8146 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8147
8148 /* LVDS/eDP FMT is set up by atom */
8149 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8150 return;
8151
8152 /* not needed for analog */
8153 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8154 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8155 return;
8156
8157 if (bpc == 0)
8158 return;
8159
8160 switch (bpc) {
8161 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8162 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8163 /* XXX sort out optimal dither settings */
8164 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8165 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8166 else
8167 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8168 break;
8169 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8170 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8171 /* XXX sort out optimal dither settings */
8172 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8173 FMT_RGB_RANDOM_ENABLE |
8174 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8175 else
8176 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8177 break;
8178 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]8179 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]8180 /* XXX sort out optimal dither settings */
8181 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8182 FMT_RGB_RANDOM_ENABLE |
8183 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8184 else
8185 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8186 break;
8187 default:
8188 /* not needed */
8189 break;
8190 }
8191
8192 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8193}
8194
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8195/* display watermark setup */
8196/**
8197 * dce8_line_buffer_adjust - Set up the line buffer
8198 *
8199 * @rdev: radeon_device pointer
8200 * @radeon_crtc: the selected display controller
8201 * @mode: the current display mode on the selected display
8202 * controller
8203 *
8204 * Setup up the line buffer allocation for
8205 * the selected display controller (CIK).
8206 * Returns the line buffer size in pixels.
8207 */
8208static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8209 struct radeon_crtc *radeon_crtc,
8210 struct drm_display_mode *mode)
8211{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8212 u32 tmp, buffer_alloc, i;
8213 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8214 /*
8215 * Line Buffer Setup
8216 * There are 6 line buffers, one for each display controllers.
8217 * There are 3 partitions per LB. Select the number of partitions
8218 * to enable based on the display width. For display widths larger
8219 * than 4096, you need use to use 2 display controllers and combine
8220 * them using the stereo blender.
8221 */
8222 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8223 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8224 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8225 buffer_alloc = 2;
8226 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8227 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8228 buffer_alloc = 2;
8229 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8230 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8231 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8232 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8233 DRM_DEBUG_KMS("Mode too big for LB!\n");
8234 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8235 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8236 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8237 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8238 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8239 buffer_alloc = 0;
8240 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8241
8242 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8243 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8244
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8245 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8246 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8247 for (i = 0; i < rdev->usec_timeout; i++) {
8248 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8249 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8250 break;
8251 udelay(1);
8252 }
8253
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8254 if (radeon_crtc->base.enabled && mode) {
8255 switch (tmp) {
8256 case 0:
8257 default:
8258 return 4096 * 2;
8259 case 1:
8260 return 1920 * 2;
8261 case 2:
8262 return 2560 * 2;
8263 }
8264 }
8265
8266 /* controller not enabled, so no lb used */
8267 return 0;
8268}
8269
8270/**
8271 * cik_get_number_of_dram_channels - get the number of dram channels
8272 *
8273 * @rdev: radeon_device pointer
8274 *
8275 * Look up the number of video ram channels (CIK).
8276 * Used for display watermark bandwidth calculations
8277 * Returns the number of dram channels
8278 */
8279static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8280{
8281 u32 tmp = RREG32(MC_SHARED_CHMAP);
8282
8283 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8284 case 0:
8285 default:
8286 return 1;
8287 case 1:
8288 return 2;
8289 case 2:
8290 return 4;
8291 case 3:
8292 return 8;
8293 case 4:
8294 return 3;
8295 case 5:
8296 return 6;
8297 case 6:
8298 return 10;
8299 case 7:
8300 return 12;
8301 case 8:
8302 return 16;
8303 }
8304}
8305
8306struct dce8_wm_params {
8307 u32 dram_channels; /* number of dram channels */
8308 u32 yclk; /* bandwidth per dram data pin in kHz */
8309 u32 sclk; /* engine clock in kHz */
8310 u32 disp_clk; /* display clock in kHz */
8311 u32 src_width; /* viewport width */
8312 u32 active_time; /* active display time in ns */
8313 u32 blank_time; /* blank time in ns */
8314 bool interlaced; /* mode is interlaced */
8315 fixed20_12 vsc; /* vertical scale ratio */
8316 u32 num_heads; /* number of active crtcs */
8317 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8318 u32 lb_size; /* line buffer allocated to pipe */
8319 u32 vtaps; /* vertical scaler taps */
8320};
8321
8322/**
8323 * dce8_dram_bandwidth - get the dram bandwidth
8324 *
8325 * @wm: watermark calculation data
8326 *
8327 * Calculate the raw dram bandwidth (CIK).
8328 * Used for display watermark bandwidth calculations
8329 * Returns the dram bandwidth in MBytes/s
8330 */
8331static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8332{
8333 /* Calculate raw DRAM Bandwidth */
8334 fixed20_12 dram_efficiency; /* 0.7 */
8335 fixed20_12 yclk, dram_channels, bandwidth;
8336 fixed20_12 a;
8337
8338 a.full = dfixed_const(1000);
8339 yclk.full = dfixed_const(wm->yclk);
8340 yclk.full = dfixed_div(yclk, a);
8341 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8342 a.full = dfixed_const(10);
8343 dram_efficiency.full = dfixed_const(7);
8344 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8345 bandwidth.full = dfixed_mul(dram_channels, yclk);
8346 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8347
8348 return dfixed_trunc(bandwidth);
8349}
8350
8351/**
8352 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8353 *
8354 * @wm: watermark calculation data
8355 *
8356 * Calculate the dram bandwidth used for display (CIK).
8357 * Used for display watermark bandwidth calculations
8358 * Returns the dram bandwidth for display in MBytes/s
8359 */
8360static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8361{
8362 /* Calculate DRAM Bandwidth and the part allocated to display. */
8363 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8364 fixed20_12 yclk, dram_channels, bandwidth;
8365 fixed20_12 a;
8366
8367 a.full = dfixed_const(1000);
8368 yclk.full = dfixed_const(wm->yclk);
8369 yclk.full = dfixed_div(yclk, a);
8370 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8371 a.full = dfixed_const(10);
8372 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8373 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8374 bandwidth.full = dfixed_mul(dram_channels, yclk);
8375 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8376
8377 return dfixed_trunc(bandwidth);
8378}
8379
8380/**
8381 * dce8_data_return_bandwidth - get the data return bandwidth
8382 *
8383 * @wm: watermark calculation data
8384 *
8385 * Calculate the data return bandwidth used for display (CIK).
8386 * Used for display watermark bandwidth calculations
8387 * Returns the data return bandwidth in MBytes/s
8388 */
8389static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8390{
8391 /* Calculate the display Data return Bandwidth */
8392 fixed20_12 return_efficiency; /* 0.8 */
8393 fixed20_12 sclk, bandwidth;
8394 fixed20_12 a;
8395
8396 a.full = dfixed_const(1000);
8397 sclk.full = dfixed_const(wm->sclk);
8398 sclk.full = dfixed_div(sclk, a);
8399 a.full = dfixed_const(10);
8400 return_efficiency.full = dfixed_const(8);
8401 return_efficiency.full = dfixed_div(return_efficiency, a);
8402 a.full = dfixed_const(32);
8403 bandwidth.full = dfixed_mul(a, sclk);
8404 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8405
8406 return dfixed_trunc(bandwidth);
8407}
8408
8409/**
8410 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8411 *
8412 * @wm: watermark calculation data
8413 *
8414 * Calculate the dmif bandwidth used for display (CIK).
8415 * Used for display watermark bandwidth calculations
8416 * Returns the dmif bandwidth in MBytes/s
8417 */
8418static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8419{
8420 /* Calculate the DMIF Request Bandwidth */
8421 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8422 fixed20_12 disp_clk, bandwidth;
8423 fixed20_12 a, b;
8424
8425 a.full = dfixed_const(1000);
8426 disp_clk.full = dfixed_const(wm->disp_clk);
8427 disp_clk.full = dfixed_div(disp_clk, a);
8428 a.full = dfixed_const(32);
8429 b.full = dfixed_mul(a, disp_clk);
8430
8431 a.full = dfixed_const(10);
8432 disp_clk_request_efficiency.full = dfixed_const(8);
8433 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8434
8435 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8436
8437 return dfixed_trunc(bandwidth);
8438}
8439
8440/**
8441 * dce8_available_bandwidth - get the min available bandwidth
8442 *
8443 * @wm: watermark calculation data
8444 *
8445 * Calculate the min available bandwidth used for display (CIK).
8446 * Used for display watermark bandwidth calculations
8447 * Returns the min available bandwidth in MBytes/s
8448 */
8449static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8450{
8451 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8452 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8453 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8454 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8455
8456 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8457}
8458
8459/**
8460 * dce8_average_bandwidth - get the average available bandwidth
8461 *
8462 * @wm: watermark calculation data
8463 *
8464 * Calculate the average available bandwidth used for display (CIK).
8465 * Used for display watermark bandwidth calculations
8466 * Returns the average available bandwidth in MBytes/s
8467 */
8468static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8469{
8470 /* Calculate the display mode Average Bandwidth
8471 * DisplayMode should contain the source and destination dimensions,
8472 * timing, etc.
8473 */
8474 fixed20_12 bpp;
8475 fixed20_12 line_time;
8476 fixed20_12 src_width;
8477 fixed20_12 bandwidth;
8478 fixed20_12 a;
8479
8480 a.full = dfixed_const(1000);
8481 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8482 line_time.full = dfixed_div(line_time, a);
8483 bpp.full = dfixed_const(wm->bytes_per_pixel);
8484 src_width.full = dfixed_const(wm->src_width);
8485 bandwidth.full = dfixed_mul(src_width, bpp);
8486 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8487 bandwidth.full = dfixed_div(bandwidth, line_time);
8488
8489 return dfixed_trunc(bandwidth);
8490}
8491
8492/**
8493 * dce8_latency_watermark - get the latency watermark
8494 *
8495 * @wm: watermark calculation data
8496 *
8497 * Calculate the latency watermark (CIK).
8498 * Used for display watermark bandwidth calculations
8499 * Returns the latency watermark in ns
8500 */
8501static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8502{
8503 /* First calculate the latency in ns */
8504 u32 mc_latency = 2000; /* 2000 ns. */
8505 u32 available_bandwidth = dce8_available_bandwidth(wm);
8506 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8507 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8508 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8509 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8510 (wm->num_heads * cursor_line_pair_return_time);
8511 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8512 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8513 u32 tmp, dmif_size = 12288;
8514 fixed20_12 a, b, c;
8515
8516 if (wm->num_heads == 0)
8517 return 0;
8518
8519 a.full = dfixed_const(2);
8520 b.full = dfixed_const(1);
8521 if ((wm->vsc.full > a.full) ||
8522 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8523 (wm->vtaps >= 5) ||
8524 ((wm->vsc.full >= a.full) && wm->interlaced))
8525 max_src_lines_per_dst_line = 4;
8526 else
8527 max_src_lines_per_dst_line = 2;
8528
8529 a.full = dfixed_const(available_bandwidth);
8530 b.full = dfixed_const(wm->num_heads);
8531 a.full = dfixed_div(a, b);
8532
8533 b.full = dfixed_const(mc_latency + 512);
8534 c.full = dfixed_const(wm->disp_clk);
8535 b.full = dfixed_div(b, c);
8536
8537 c.full = dfixed_const(dmif_size);
8538 b.full = dfixed_div(c, b);
8539
8540 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8541
8542 b.full = dfixed_const(1000);
8543 c.full = dfixed_const(wm->disp_clk);
8544 b.full = dfixed_div(c, b);
8545 c.full = dfixed_const(wm->bytes_per_pixel);
8546 b.full = dfixed_mul(b, c);
8547
8548 lb_fill_bw = min(tmp, dfixed_trunc(b));
8549
8550 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8551 b.full = dfixed_const(1000);
8552 c.full = dfixed_const(lb_fill_bw);
8553 b.full = dfixed_div(c, b);
8554 a.full = dfixed_div(a, b);
8555 line_fill_time = dfixed_trunc(a);
8556
8557 if (line_fill_time < wm->active_time)
8558 return latency;
8559 else
8560 return latency + (line_fill_time - wm->active_time);
8561
8562}
8563
8564/**
8565 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8566 * average and available dram bandwidth
8567 *
8568 * @wm: watermark calculation data
8569 *
8570 * Check if the display average bandwidth fits in the display
8571 * dram bandwidth (CIK).
8572 * Used for display watermark bandwidth calculations
8573 * Returns true if the display fits, false if not.
8574 */
8575static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8576{
8577 if (dce8_average_bandwidth(wm) <=
8578 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8579 return true;
8580 else
8581 return false;
8582}
8583
8584/**
8585 * dce8_average_bandwidth_vs_available_bandwidth - check
8586 * average and available bandwidth
8587 *
8588 * @wm: watermark calculation data
8589 *
8590 * Check if the display average bandwidth fits in the display
8591 * available bandwidth (CIK).
8592 * Used for display watermark bandwidth calculations
8593 * Returns true if the display fits, false if not.
8594 */
8595static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8596{
8597 if (dce8_average_bandwidth(wm) <=
8598 (dce8_available_bandwidth(wm) / wm->num_heads))
8599 return true;
8600 else
8601 return false;
8602}
8603
8604/**
8605 * dce8_check_latency_hiding - check latency hiding
8606 *
8607 * @wm: watermark calculation data
8608 *
8609 * Check latency hiding (CIK).
8610 * Used for display watermark bandwidth calculations
8611 * Returns true if the display fits, false if not.
8612 */
8613static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8614{
8615 u32 lb_partitions = wm->lb_size / wm->src_width;
8616 u32 line_time = wm->active_time + wm->blank_time;
8617 u32 latency_tolerant_lines;
8618 u32 latency_hiding;
8619 fixed20_12 a;
8620
8621 a.full = dfixed_const(1);
8622 if (wm->vsc.full > a.full)
8623 latency_tolerant_lines = 1;
8624 else {
8625 if (lb_partitions <= (wm->vtaps + 1))
8626 latency_tolerant_lines = 1;
8627 else
8628 latency_tolerant_lines = 2;
8629 }
8630
8631 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8632
8633 if (dce8_latency_watermark(wm) <= latency_hiding)
8634 return true;
8635 else
8636 return false;
8637}
8638
8639/**
8640 * dce8_program_watermarks - program display watermarks
8641 *
8642 * @rdev: radeon_device pointer
8643 * @radeon_crtc: the selected display controller
8644 * @lb_size: line buffer size
8645 * @num_heads: number of display controllers in use
8646 *
8647 * Calculate and program the display watermarks for the
8648 * selected display controller (CIK).
8649 */
8650static void dce8_program_watermarks(struct radeon_device *rdev,
8651 struct radeon_crtc *radeon_crtc,
8652 u32 lb_size, u32 num_heads)
8653{
8654 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8655 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8656 u32 pixel_period;
8657 u32 line_time = 0;
8658 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8659 u32 tmp, wm_mask;
8660
8661 if (radeon_crtc->base.enabled && num_heads && mode) {
8662 pixel_period = 1000000 / (u32)mode->clock;
8663 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8664
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8665 /* watermark for high clocks */
8666 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8667 rdev->pm.dpm_enabled) {
8668 wm_high.yclk =
8669 radeon_dpm_get_mclk(rdev, false) * 10;
8670 wm_high.sclk =
8671 radeon_dpm_get_sclk(rdev, false) * 10;
8672 } else {
8673 wm_high.yclk = rdev->pm.current_mclk * 10;
8674 wm_high.sclk = rdev->pm.current_sclk * 10;
8675 }
8676
8677 wm_high.disp_clk = mode->clock;
8678 wm_high.src_width = mode->crtc_hdisplay;
8679 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8680 wm_high.blank_time = line_time - wm_high.active_time;
8681 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8682 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8683 wm_high.interlaced = true;
8684 wm_high.vsc = radeon_crtc->vsc;
8685 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8686 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8687 wm_high.vtaps = 2;
8688 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8689 wm_high.lb_size = lb_size;
8690 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8691 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8692
8693 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8694 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8695
8696 /* possibly force display priority to high */
8697 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8698 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8699 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8700 !dce8_check_latency_hiding(&wm_high) ||
8701 (rdev->disp_priority == 2)) {
8702 DRM_DEBUG_KMS("force priority to high\n");
8703 }
8704
8705 /* watermark for low clocks */
8706 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8707 rdev->pm.dpm_enabled) {
8708 wm_low.yclk =
8709 radeon_dpm_get_mclk(rdev, true) * 10;
8710 wm_low.sclk =
8711 radeon_dpm_get_sclk(rdev, true) * 10;
8712 } else {
8713 wm_low.yclk = rdev->pm.current_mclk * 10;
8714 wm_low.sclk = rdev->pm.current_sclk * 10;
8715 }
8716
8717 wm_low.disp_clk = mode->clock;
8718 wm_low.src_width = mode->crtc_hdisplay;
8719 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8720 wm_low.blank_time = line_time - wm_low.active_time;
8721 wm_low.interlaced = false;
8722 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8723 wm_low.interlaced = true;
8724 wm_low.vsc = radeon_crtc->vsc;
8725 wm_low.vtaps = 1;
8726 if (radeon_crtc->rmx_type != RMX_OFF)
8727 wm_low.vtaps = 2;
8728 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8729 wm_low.lb_size = lb_size;
8730 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8731 wm_low.num_heads = num_heads;
8732
8733 /* set for low clocks */
8734 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8735
8736 /* possibly force display priority to high */
8737 /* should really do this at mode validation time... */
8738 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8739 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8740 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8741 (rdev->disp_priority == 2)) {
8742 DRM_DEBUG_KMS("force priority to high\n");
8743 }
8744 }
8745
8746 /* select wm A */
8747 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8748 tmp = wm_mask;
8749 tmp &= ~LATENCY_WATERMARK_MASK(3);
8750 tmp |= LATENCY_WATERMARK_MASK(1);
8751 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8752 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8753 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8754 LATENCY_HIGH_WATERMARK(line_time)));
8755 /* select wm B */
8756 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8757 tmp &= ~LATENCY_WATERMARK_MASK(3);
8758 tmp |= LATENCY_WATERMARK_MASK(2);
8759 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8760 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8761 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8762 LATENCY_HIGH_WATERMARK(line_time)));
8763 /* restore original selection */
8764 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8765
8766 /* save values for DPM */
8767 radeon_crtc->line_time = line_time;
8768 radeon_crtc->wm_high = latency_watermark_a;
8769 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8770}
8771
8772/**
8773 * dce8_bandwidth_update - program display watermarks
8774 *
8775 * @rdev: radeon_device pointer
8776 *
8777 * Calculate and program the display watermarks and line
8778 * buffer allocation (CIK).
8779 */
8780void dce8_bandwidth_update(struct radeon_device *rdev)
8781{
8782 struct drm_display_mode *mode = NULL;
8783 u32 num_heads = 0, lb_size;
8784 int i;
8785
8786 radeon_update_display_priority(rdev);
8787
8788 for (i = 0; i < rdev->num_crtc; i++) {
8789 if (rdev->mode_info.crtcs[i]->base.enabled)
8790 num_heads++;
8791 }
8792 for (i = 0; i < rdev->num_crtc; i++) {
8793 mode = &rdev->mode_info.crtcs[i]->base.mode;
8794 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8795 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8796 }
8797}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8798
8799/**
8800 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8801 *
8802 * @rdev: radeon_device pointer
8803 *
8804 * Fetches a GPU clock counter snapshot (SI).
8805 * Returns the 64 bit clock counter snapshot.
8806 */
8807uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8808{
8809 uint64_t clock;
8810
8811 mutex_lock(&rdev->gpu_clock_mutex);
8812 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8813 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8814 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8815 mutex_unlock(&rdev->gpu_clock_mutex);
8816 return clock;
8817}
8818
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8819static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8820 u32 cntl_reg, u32 status_reg)
8821{
8822 int r, i;
8823 struct atom_clock_dividers dividers;
8824 uint32_t tmp;
8825
8826 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8827 clock, false, &dividers);
8828 if (r)
8829 return r;
8830
8831 tmp = RREG32_SMC(cntl_reg);
8832 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8833 tmp |= dividers.post_divider;
8834 WREG32_SMC(cntl_reg, tmp);
8835
8836 for (i = 0; i < 100; i++) {
8837 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8838 break;
8839 mdelay(10);
8840 }
8841 if (i == 100)
8842 return -ETIMEDOUT;
8843
8844 return 0;
8845}
8846
8847int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8848{
8849 int r = 0;
8850
8851 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8852 if (r)
8853 return r;
8854
8855 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8856 return r;
8857}
8858
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8859static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8860{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8861 struct pci_dev *root = rdev->pdev->bus->self;
8862 int bridge_pos, gpu_pos;
8863 u32 speed_cntl, mask, current_data_rate;
8864 int ret, i;
8865 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8866
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8867 if (radeon_pcie_gen2 == 0)
8868 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8869
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8870 if (rdev->flags & RADEON_IS_IGP)
8871 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8872
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8873 if (!(rdev->flags & RADEON_IS_PCIE))
8874 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8875
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8876 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8877 if (ret != 0)
8878 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8879
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8880 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8881 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8882
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8883 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8884 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8885 LC_CURRENT_DATA_RATE_SHIFT;
8886 if (mask & DRM_PCIE_SPEED_80) {
8887 if (current_data_rate == 2) {
8888 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8889 return;
8890 }
8891 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8892 } else if (mask & DRM_PCIE_SPEED_50) {
8893 if (current_data_rate == 1) {
8894 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8895 return;
8896 }
8897 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8898 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8899
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8900 bridge_pos = pci_pcie_cap(root);
8901 if (!bridge_pos)
8902 return;
8903
8904 gpu_pos = pci_pcie_cap(rdev->pdev);
8905 if (!gpu_pos)
8906 return;
8907
8908 if (mask & DRM_PCIE_SPEED_80) {
8909 /* re-try equalization if gen3 is not already enabled */
8910 if (current_data_rate != 2) {
8911 u16 bridge_cfg, gpu_cfg;
8912 u16 bridge_cfg2, gpu_cfg2;
8913 u32 max_lw, current_lw, tmp;
8914
8915 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8916 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8917
8918 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8919 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8920
8921 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8922 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8923
8924 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8925 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8926 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8927
8928 if (current_lw < max_lw) {
8929 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8930 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8931 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8932 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8933 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8934 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8935 }
8936 }
8937
8938 for (i = 0; i < 10; i++) {
8939 /* check status */
8940 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8941 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8942 break;
8943
8944 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8945 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8946
8947 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8948 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8949
8950 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8951 tmp |= LC_SET_QUIESCE;
8952 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8953
8954 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8955 tmp |= LC_REDO_EQ;
8956 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8957
8958 mdelay(100);
8959
8960 /* linkctl */
8961 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8962 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8963 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8964 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8965
8966 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8967 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8968 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8969 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8970
8971 /* linkctl2 */
8972 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8973 tmp16 &= ~((1 << 4) | (7 << 9));
8974 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8975 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8976
8977 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8978 tmp16 &= ~((1 << 4) | (7 << 9));
8979 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8980 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8981
8982 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8983 tmp &= ~LC_SET_QUIESCE;
8984 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8985 }
8986 }
8987 }
8988
8989 /* set the link speed */
8990 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8991 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8992 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8993
8994 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8995 tmp16 &= ~0xf;
8996 if (mask & DRM_PCIE_SPEED_80)
8997 tmp16 |= 3; /* gen3 */
8998 else if (mask & DRM_PCIE_SPEED_50)
8999 tmp16 |= 2; /* gen2 */
9000 else
9001 tmp16 |= 1; /* gen1 */
9002 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9003
9004 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9005 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9006 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9007
9008 for (i = 0; i < rdev->usec_timeout; i++) {
9009 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9010 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9011 break;
9012 udelay(1);
9013 }
9014}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]9015
9016static void cik_program_aspm(struct radeon_device *rdev)
9017{
9018 u32 data, orig;
9019 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9020 bool disable_clkreq = false;
9021
9022 if (radeon_aspm == 0)
9023 return;
9024
9025 /* XXX double check IGPs */
9026 if (rdev->flags & RADEON_IS_IGP)
9027 return;
9028
9029 if (!(rdev->flags & RADEON_IS_PCIE))
9030 return;
9031
9032 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9033 data &= ~LC_XMIT_N_FTS_MASK;
9034 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9035 if (orig != data)
9036 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9037
9038 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9039 data |= LC_GO_TO_RECOVERY;
9040 if (orig != data)
9041 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9042
9043 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9044 data |= P_IGNORE_EDB_ERR;
9045 if (orig != data)
9046 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9047
9048 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9049 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9050 data |= LC_PMI_TO_L1_DIS;
9051 if (!disable_l0s)
9052 data |= LC_L0S_INACTIVITY(7);
9053
9054 if (!disable_l1) {
9055 data |= LC_L1_INACTIVITY(7);
9056 data &= ~LC_PMI_TO_L1_DIS;
9057 if (orig != data)
9058 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9059
9060 if (!disable_plloff_in_l1) {
9061 bool clk_req_support;
9062
9063 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9064 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9065 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9066 if (orig != data)
9067 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9068
9069 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9070 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9071 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9072 if (orig != data)
9073 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9074
9075 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9076 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9077 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9078 if (orig != data)
9079 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9080
9081 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9082 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9083 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9084 if (orig != data)
9085 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9086
9087 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9088 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9089 data |= LC_DYN_LANES_PWR_STATE(3);
9090 if (orig != data)
9091 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9092
9093 if (!disable_clkreq) {
9094 struct pci_dev *root = rdev->pdev->bus->self;
9095 u32 lnkcap;
9096
9097 clk_req_support = false;
9098 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9099 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9100 clk_req_support = true;
9101 } else {
9102 clk_req_support = false;
9103 }
9104
9105 if (clk_req_support) {
9106 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9107 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9108 if (orig != data)
9109 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9110
9111 orig = data = RREG32_SMC(THM_CLK_CNTL);
9112 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9113 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9114 if (orig != data)
9115 WREG32_SMC(THM_CLK_CNTL, data);
9116
9117 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9118 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9119 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9120 if (orig != data)
9121 WREG32_SMC(MISC_CLK_CTRL, data);
9122
9123 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9124 data &= ~BCLK_AS_XCLK;
9125 if (orig != data)
9126 WREG32_SMC(CG_CLKPIN_CNTL, data);
9127
9128 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9129 data &= ~FORCE_BIF_REFCLK_EN;
9130 if (orig != data)
9131 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9132
9133 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9134 data &= ~MPLL_CLKOUT_SEL_MASK;
9135 data |= MPLL_CLKOUT_SEL(4);
9136 if (orig != data)
9137 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9138 }
9139 }
9140 } else {
9141 if (orig != data)
9142 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9143 }
9144
9145 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9146 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9147 if (orig != data)
9148 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9149
9150 if (!disable_l0s) {
9151 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9152 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9153 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9154 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9155 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9156 data &= ~LC_L0S_INACTIVITY_MASK;
9157 if (orig != data)
9158 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9159 }
9160 }
9161 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]9162}