Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 01ac8794a77192236a4b91c33adf4177ac5a21f0 / . / drivers / gpu / drm / radeon / cik.c
blob: 4ddc3e5c2d97188ba702a0b7d803b10edeb9e3bd [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]78static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]80static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]87
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]124
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]130 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]131 u32 r;
132
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]151}
152
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1532 default:
1533 break;
1534 }
1535}
1536
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1563 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1564 *
1565 * Returns the value in the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1566 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1567 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1569{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1572 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1582 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1586 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1587 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1589{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1592 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1594 }
1595}
1596
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
1700static int ci_mc_load_microcode(struct radeon_device *rdev)
1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1802 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1904 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1925 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1932 }
1933
1934out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1952 }
1953 return err;
1954}
1955
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1994 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1995
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2432 }
2433 } else if (num_pipe_configs == 4) {
2434 if (num_rbs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2437 case 0:
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2442 break;
2443 case 1:
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2448 break;
2449 case 2:
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2454 break;
2455 case 3:
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2460 break;
2461 case 4:
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2466 break;
2467 case 5:
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2470 break;
2471 case 6:
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2476 break;
2477 case 7:
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2482 break;
2483 case 8:
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2486 break;
2487 case 9:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2490 break;
2491 case 10:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2496 break;
2497 case 11:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2502 break;
2503 case 12:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2508 break;
2509 case 13:
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2512 break;
2513 case 14:
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518 break;
2519 case 16:
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2524 break;
2525 case 17:
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 break;
2531 case 27:
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2534 break;
2535 case 28:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 29:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 30:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 default:
2554 gb_tile_moden = 0;
2555 break;
2556 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2559 }
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2563 case 0:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2568 break;
2569 case 1:
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2574 break;
2575 case 2:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 break;
2581 case 3:
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2586 break;
2587 case 4:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2592 break;
2593 case 5:
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2596 break;
2597 case 6:
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2602 break;
2603 case 7:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2608 break;
2609 case 8:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2612 break;
2613 case 9:
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2616 break;
2617 case 10:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 11:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 12:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 13:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2638 break;
2639 case 14:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 16:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2650 break;
2651 case 17:
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2656 break;
2657 case 27:
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2660 break;
2661 case 28:
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2666 break;
2667 case 29:
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 break;
2673 case 30:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 default:
2680 gb_tile_moden = 0;
2681 break;
2682 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2685 }
2686 }
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2689 case 0:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2694 break;
2695 case 1:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 2:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 3:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 4:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 5:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 6:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 8:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 9:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 10:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 11:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2754 break;
2755 case 12:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 break;
2761 case 13:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2766 break;
2767 case 14:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2772 break;
2773 default:
2774 gb_tile_moden = 0;
2775 break;
2776 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779 }
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2783 case 0:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2788 break;
2789 case 1:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2794 break;
2795 case 2:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 3:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2806 break;
2807 case 4:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2812 break;
2813 case 5:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2816 break;
2817 case 6:
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2822 break;
2823 case 7:
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2828 break;
2829 case 8:
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2831 break;
2832 case 9:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2835 break;
2836 case 10:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 11:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 12:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 13:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2857 break;
2858 case 14:
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 break;
2864 case 16:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 17:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 27:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2879 break;
2880 case 28:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 29:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 30:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 break;
2898 default:
2899 gb_tile_moden = 0;
2900 break;
2901 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2904 }
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2907 case 0:
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2912 break;
2913 case 1:
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2918 break;
2919 case 2:
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2924 break;
2925 case 3:
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2930 break;
2931 case 4:
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2936 break;
2937 case 5:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 6:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2948 break;
2949 case 8:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 9:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 10:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 11:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 12:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2978 break;
2979 case 13:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 14:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2990 break;
2991 default:
2992 gb_tile_moden = 0;
2993 break;
2994 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2997 }
2998 } else
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3000}
3001
3002/**
3003 * cik_select_se_sh - select which SE, SH to address
3004 *
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3008 *
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3012 */
3013static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3015{
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3017
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3024 else
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3027}
3028
3029/**
3030 * cik_create_bitmask - create a bitmask
3031 *
3032 * @bit_width: length of the mask
3033 *
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3036 */
3037static u32 cik_create_bitmask(u32 bit_width)
3038{
3039 u32 i, mask = 0;
3040
3041 for (i = 0; i < bit_width; i++) {
3042 mask <<= 1;
3043 mask |= 1;
3044 }
3045 return mask;
3046}
3047
3048/**
3049 * cik_select_se_sh - select which SE, SH to address
3050 *
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3055 *
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3058 */
3059static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3060 u32 max_rb_num, u32 se_num,
3061 u32 sh_per_se)
3062{
3063 u32 data, mask;
3064
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3066 if (data & 1)
3067 data &= BACKEND_DISABLE_MASK;
3068 else
3069 data = 0;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3071
3072 data >>= BACKEND_DISABLE_SHIFT;
3073
3074 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3075
3076 return data & mask;
3077}
3078
3079/**
3080 * cik_setup_rb - setup the RBs on the asic
3081 *
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3086 *
3087 * Configures per-SE/SH RB registers (CIK).
3088 */
3089static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
3091 u32 max_rb_num)
3092{
3093 int i, j;
3094 u32 data, mask;
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3097
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
3101 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3104 else
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3106 }
3107 }
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3109
3110 mask = 1;
3111 for (i = 0; i < max_rb_num; i++) {
3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3114 mask <<= 1;
3115 }
3116
3117 for (i = 0; i < se_num; i++) {
3118 cik_select_se_sh(rdev, i, 0xffffffff);
3119 data = 0;
3120 for (j = 0; j < sh_per_se; j++) {
3121 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3122 case 0:
3123 if (j == 0)
3124 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3125 else
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3127 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3128 case 1:
3129 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3130 break;
3131 case 2:
3132 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3133 break;
3134 case 3:
3135 default:
3136 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3137 break;
3138 }
3139 enabled_rbs >>= 2;
3140 }
3141 WREG32(PA_SC_RASTER_CONFIG, data);
3142 }
3143 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3144}
3145
3146/**
3147 * cik_gpu_init - setup the 3D engine
3148 *
3149 * @rdev: radeon_device pointer
3150 *
3151 * Configures the 3D engine and tiling configuration
3152 * registers so that the 3D engine is usable.
3153 */
3154static void cik_gpu_init(struct radeon_device *rdev)
3155{
3156 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3157 u32 mc_shared_chmap, mc_arb_ramcfg;
3158 u32 hdp_host_path_cntl;
3159 u32 tmp;
3160 int i, j;
3161
3162 switch (rdev->family) {
3163 case CHIP_BONAIRE:
3164 rdev->config.cik.max_shader_engines = 2;
3165 rdev->config.cik.max_tile_pipes = 4;
3166 rdev->config.cik.max_cu_per_sh = 7;
3167 rdev->config.cik.max_sh_per_se = 1;
3168 rdev->config.cik.max_backends_per_se = 2;
3169 rdev->config.cik.max_texture_channel_caches = 4;
3170 rdev->config.cik.max_gprs = 256;
3171 rdev->config.cik.max_gs_threads = 32;
3172 rdev->config.cik.max_hw_contexts = 8;
3173
3174 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3175 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3176 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3177 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3178 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3179 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3180 case CHIP_HAWAII:
3181 rdev->config.cik.max_shader_engines = 4;
3182 rdev->config.cik.max_tile_pipes = 16;
3183 rdev->config.cik.max_cu_per_sh = 11;
3184 rdev->config.cik.max_sh_per_se = 1;
3185 rdev->config.cik.max_backends_per_se = 4;
3186 rdev->config.cik.max_texture_channel_caches = 16;
3187 rdev->config.cik.max_gprs = 256;
3188 rdev->config.cik.max_gs_threads = 32;
3189 rdev->config.cik.max_hw_contexts = 8;
3190
3191 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3192 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3193 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3194 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3195 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3196 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3197 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3198 rdev->config.cik.max_shader_engines = 1;
3199 rdev->config.cik.max_tile_pipes = 4;
3200 if ((rdev->pdev->device == 0x1304) ||
3201 (rdev->pdev->device == 0x1305) ||
3202 (rdev->pdev->device == 0x130C) ||
3203 (rdev->pdev->device == 0x130F) ||
3204 (rdev->pdev->device == 0x1310) ||
3205 (rdev->pdev->device == 0x1311) ||
3206 (rdev->pdev->device == 0x131C)) {
3207 rdev->config.cik.max_cu_per_sh = 8;
3208 rdev->config.cik.max_backends_per_se = 2;
3209 } else if ((rdev->pdev->device == 0x1309) ||
3210 (rdev->pdev->device == 0x130A) ||
3211 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3212 (rdev->pdev->device == 0x1313) ||
3213 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3214 rdev->config.cik.max_cu_per_sh = 6;
3215 rdev->config.cik.max_backends_per_se = 2;
3216 } else if ((rdev->pdev->device == 0x1306) ||
3217 (rdev->pdev->device == 0x1307) ||
3218 (rdev->pdev->device == 0x130B) ||
3219 (rdev->pdev->device == 0x130E) ||
3220 (rdev->pdev->device == 0x1315) ||
3221 (rdev->pdev->device == 0x131B)) {
3222 rdev->config.cik.max_cu_per_sh = 4;
3223 rdev->config.cik.max_backends_per_se = 1;
3224 } else {
3225 rdev->config.cik.max_cu_per_sh = 3;
3226 rdev->config.cik.max_backends_per_se = 1;
3227 }
3228 rdev->config.cik.max_sh_per_se = 1;
3229 rdev->config.cik.max_texture_channel_caches = 4;
3230 rdev->config.cik.max_gprs = 256;
3231 rdev->config.cik.max_gs_threads = 16;
3232 rdev->config.cik.max_hw_contexts = 8;
3233
3234 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3235 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3236 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3237 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3238 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3239 break;
3240 case CHIP_KABINI:
3241 default:
3242 rdev->config.cik.max_shader_engines = 1;
3243 rdev->config.cik.max_tile_pipes = 2;
3244 rdev->config.cik.max_cu_per_sh = 2;
3245 rdev->config.cik.max_sh_per_se = 1;
3246 rdev->config.cik.max_backends_per_se = 1;
3247 rdev->config.cik.max_texture_channel_caches = 2;
3248 rdev->config.cik.max_gprs = 256;
3249 rdev->config.cik.max_gs_threads = 16;
3250 rdev->config.cik.max_hw_contexts = 8;
3251
3252 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3253 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3254 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3255 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3256 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3257 break;
3258 }
3259
3260 /* Initialize HDP */
3261 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3262 WREG32((0x2c14 + j), 0x00000000);
3263 WREG32((0x2c18 + j), 0x00000000);
3264 WREG32((0x2c1c + j), 0x00000000);
3265 WREG32((0x2c20 + j), 0x00000000);
3266 WREG32((0x2c24 + j), 0x00000000);
3267 }
3268
3269 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3270
3271 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3272
3273 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3274 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3275
3276 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3277 rdev->config.cik.mem_max_burst_length_bytes = 256;
3278 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3279 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3280 if (rdev->config.cik.mem_row_size_in_kb > 4)
3281 rdev->config.cik.mem_row_size_in_kb = 4;
3282 /* XXX use MC settings? */
3283 rdev->config.cik.shader_engine_tile_size = 32;
3284 rdev->config.cik.num_gpus = 1;
3285 rdev->config.cik.multi_gpu_tile_size = 64;
3286
3287 /* fix up row size */
3288 gb_addr_config &= ~ROW_SIZE_MASK;
3289 switch (rdev->config.cik.mem_row_size_in_kb) {
3290 case 1:
3291 default:
3292 gb_addr_config |= ROW_SIZE(0);
3293 break;
3294 case 2:
3295 gb_addr_config |= ROW_SIZE(1);
3296 break;
3297 case 4:
3298 gb_addr_config |= ROW_SIZE(2);
3299 break;
3300 }
3301
3302 /* setup tiling info dword. gb_addr_config is not adequate since it does
3303 * not have bank info, so create a custom tiling dword.
3304 * bits 3:0 num_pipes
3305 * bits 7:4 num_banks
3306 * bits 11:8 group_size
3307 * bits 15:12 row_size
3308 */
3309 rdev->config.cik.tile_config = 0;
3310 switch (rdev->config.cik.num_tile_pipes) {
3311 case 1:
3312 rdev->config.cik.tile_config |= (0 << 0);
3313 break;
3314 case 2:
3315 rdev->config.cik.tile_config |= (1 << 0);
3316 break;
3317 case 4:
3318 rdev->config.cik.tile_config |= (2 << 0);
3319 break;
3320 case 8:
3321 default:
3322 /* XXX what about 12? */
3323 rdev->config.cik.tile_config |= (3 << 0);
3324 break;
3325 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3326 rdev->config.cik.tile_config |=
3327 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3328 rdev->config.cik.tile_config |=
3329 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3332
3333 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3334 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3335 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3336 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3337 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3338 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3339 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3340 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3341
3342 cik_tiling_mode_table_init(rdev);
3343
3344 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3345 rdev->config.cik.max_sh_per_se,
3346 rdev->config.cik.max_backends_per_se);
3347
3348 /* set HW defaults for 3D engine */
3349 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3350
3351 WREG32(SX_DEBUG_1, 0x20);
3352
3353 WREG32(TA_CNTL_AUX, 0x00010000);
3354
3355 tmp = RREG32(SPI_CONFIG_CNTL);
3356 tmp |= 0x03000000;
3357 WREG32(SPI_CONFIG_CNTL, tmp);
3358
3359 WREG32(SQ_CONFIG, 1);
3360
3361 WREG32(DB_DEBUG, 0);
3362
3363 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3364 tmp |= 0x00000400;
3365 WREG32(DB_DEBUG2, tmp);
3366
3367 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3368 tmp |= 0x00020200;
3369 WREG32(DB_DEBUG3, tmp);
3370
3371 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3372 tmp |= 0x00018208;
3373 WREG32(CB_HW_CONTROL, tmp);
3374
3375 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3376
3377 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3378 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3379 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3380 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3381
3382 WREG32(VGT_NUM_INSTANCES, 1);
3383
3384 WREG32(CP_PERFMON_CNTL, 0);
3385
3386 WREG32(SQ_CONFIG, 0);
3387
3388 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3389 FORCE_EOV_MAX_REZ_CNT(255)));
3390
3391 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3392 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3393
3394 WREG32(VGT_GS_VERTEX_REUSE, 16);
3395 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3396
3397 tmp = RREG32(HDP_MISC_CNTL);
3398 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3399 WREG32(HDP_MISC_CNTL, tmp);
3400
3401 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3402 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3403
3404 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3405 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3406
3407 udelay(50);
3408}
3409
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3410/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3411 * GPU scratch registers helpers function.
3412 */
3413/**
3414 * cik_scratch_init - setup driver info for CP scratch regs
3415 *
3416 * @rdev: radeon_device pointer
3417 *
3418 * Set up the number and offset of the CP scratch registers.
3419 * NOTE: use of CP scratch registers is a legacy inferface and
3420 * is not used by default on newer asics (r6xx+). On newer asics,
3421 * memory buffers are used for fences rather than scratch regs.
3422 */
3423static void cik_scratch_init(struct radeon_device *rdev)
3424{
3425 int i;
3426
3427 rdev->scratch.num_reg = 7;
3428 rdev->scratch.reg_base = SCRATCH_REG0;
3429 for (i = 0; i < rdev->scratch.num_reg; i++) {
3430 rdev->scratch.free[i] = true;
3431 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3432 }
3433}
3434
3435/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3436 * cik_ring_test - basic gfx ring test
3437 *
3438 * @rdev: radeon_device pointer
3439 * @ring: radeon_ring structure holding ring information
3440 *
3441 * Allocate a scratch register and write to it using the gfx ring (CIK).
3442 * Provides a basic gfx ring test to verify that the ring is working.
3443 * Used by cik_cp_gfx_resume();
3444 * Returns 0 on success, error on failure.
3445 */
3446int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3447{
3448 uint32_t scratch;
3449 uint32_t tmp = 0;
3450 unsigned i;
3451 int r;
3452
3453 r = radeon_scratch_get(rdev, &scratch);
3454 if (r) {
3455 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3456 return r;
3457 }
3458 WREG32(scratch, 0xCAFEDEAD);
3459 r = radeon_ring_lock(rdev, ring, 3);
3460 if (r) {
3461 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3462 radeon_scratch_free(rdev, scratch);
3463 return r;
3464 }
3465 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3466 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3467 radeon_ring_write(ring, 0xDEADBEEF);
3468 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3469
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3470 for (i = 0; i < rdev->usec_timeout; i++) {
3471 tmp = RREG32(scratch);
3472 if (tmp == 0xDEADBEEF)
3473 break;
3474 DRM_UDELAY(1);
3475 }
3476 if (i < rdev->usec_timeout) {
3477 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3478 } else {
3479 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3480 ring->idx, scratch, tmp);
3481 r = -EINVAL;
3482 }
3483 radeon_scratch_free(rdev, scratch);
3484 return r;
3485}
3486
3487/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3488 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3489 *
3490 * @rdev: radeon_device pointer
3491 * @fence: radeon fence object
3492 *
3493 * Emits a fence sequnce number on the gfx ring and flushes
3494 * GPU caches.
3495 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3496void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3497 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3498{
3499 struct radeon_ring *ring = &rdev->ring[fence->ring];
3500 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3501
3502 /* EVENT_WRITE_EOP - flush caches, send int */
3503 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3504 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3505 EOP_TC_ACTION_EN |
3506 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3507 EVENT_INDEX(5)));
3508 radeon_ring_write(ring, addr & 0xfffffffc);
3509 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3510 radeon_ring_write(ring, fence->seq);
3511 radeon_ring_write(ring, 0);
3512 /* HDP flush */
3513 /* We should be using the new WAIT_REG_MEM special op packet here
3514 * but it causes the CP to hang
3515 */
3516 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3517 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3518 WRITE_DATA_DST_SEL(0)));
3519 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3520 radeon_ring_write(ring, 0);
3521 radeon_ring_write(ring, 0);
3522}
3523
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3524/**
3525 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3526 *
3527 * @rdev: radeon_device pointer
3528 * @fence: radeon fence object
3529 *
3530 * Emits a fence sequnce number on the compute ring and flushes
3531 * GPU caches.
3532 */
3533void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3534 struct radeon_fence *fence)
3535{
3536 struct radeon_ring *ring = &rdev->ring[fence->ring];
3537 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3538
3539 /* RELEASE_MEM - flush caches, send int */
3540 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3541 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3542 EOP_TC_ACTION_EN |
3543 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3544 EVENT_INDEX(5)));
3545 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3546 radeon_ring_write(ring, addr & 0xfffffffc);
3547 radeon_ring_write(ring, upper_32_bits(addr));
3548 radeon_ring_write(ring, fence->seq);
3549 radeon_ring_write(ring, 0);
3550 /* HDP flush */
3551 /* We should be using the new WAIT_REG_MEM special op packet here
3552 * but it causes the CP to hang
3553 */
3554 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3555 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3556 WRITE_DATA_DST_SEL(0)));
3557 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3558 radeon_ring_write(ring, 0);
3559 radeon_ring_write(ring, 0);
3560}
3561
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3562bool cik_semaphore_ring_emit(struct radeon_device *rdev,
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3563 struct radeon_ring *ring,
3564 struct radeon_semaphore *semaphore,
3565 bool emit_wait)
3566{
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3567/* TODO: figure out why semaphore cause lockups */
3568#if 0
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3569 uint64_t addr = semaphore->gpu_addr;
3570 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3571
3572 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3573 radeon_ring_write(ring, addr & 0xffffffff);
3574 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3575
3576 return true;
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3577#else
3578 return false;
3579#endif
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3580}
3581
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3582/**
3583 * cik_copy_cpdma - copy pages using the CP DMA engine
3584 *
3585 * @rdev: radeon_device pointer
3586 * @src_offset: src GPU address
3587 * @dst_offset: dst GPU address
3588 * @num_gpu_pages: number of GPU pages to xfer
3589 * @fence: radeon fence object
3590 *
3591 * Copy GPU paging using the CP DMA engine (CIK+).
3592 * Used by the radeon ttm implementation to move pages if
3593 * registered as the asic copy callback.
3594 */
3595int cik_copy_cpdma(struct radeon_device *rdev,
3596 uint64_t src_offset, uint64_t dst_offset,
3597 unsigned num_gpu_pages,
3598 struct radeon_fence **fence)
3599{
3600 struct radeon_semaphore *sem = NULL;
3601 int ring_index = rdev->asic->copy.blit_ring_index;
3602 struct radeon_ring *ring = &rdev->ring[ring_index];
3603 u32 size_in_bytes, cur_size_in_bytes, control;
3604 int i, num_loops;
3605 int r = 0;
3606
3607 r = radeon_semaphore_create(rdev, &sem);
3608 if (r) {
3609 DRM_ERROR("radeon: moving bo (%d).\n", r);
3610 return r;
3611 }
3612
3613 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3614 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3615 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3616 if (r) {
3617 DRM_ERROR("radeon: moving bo (%d).\n", r);
3618 radeon_semaphore_free(rdev, &sem, NULL);
3619 return r;
3620 }
3621
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3622 radeon_semaphore_sync_to(sem, *fence);
3623 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3624
3625 for (i = 0; i < num_loops; i++) {
3626 cur_size_in_bytes = size_in_bytes;
3627 if (cur_size_in_bytes > 0x1fffff)
3628 cur_size_in_bytes = 0x1fffff;
3629 size_in_bytes -= cur_size_in_bytes;
3630 control = 0;
3631 if (size_in_bytes == 0)
3632 control |= PACKET3_DMA_DATA_CP_SYNC;
3633 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3634 radeon_ring_write(ring, control);
3635 radeon_ring_write(ring, lower_32_bits(src_offset));
3636 radeon_ring_write(ring, upper_32_bits(src_offset));
3637 radeon_ring_write(ring, lower_32_bits(dst_offset));
3638 radeon_ring_write(ring, upper_32_bits(dst_offset));
3639 radeon_ring_write(ring, cur_size_in_bytes);
3640 src_offset += cur_size_in_bytes;
3641 dst_offset += cur_size_in_bytes;
3642 }
3643
3644 r = radeon_fence_emit(rdev, fence, ring->idx);
3645 if (r) {
3646 radeon_ring_unlock_undo(rdev, ring);
3647 return r;
3648 }
3649
3650 radeon_ring_unlock_commit(rdev, ring);
3651 radeon_semaphore_free(rdev, &sem, *fence);
3652
3653 return r;
3654}
3655
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3656/*
3657 * IB stuff
3658 */
3659/**
3660 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3661 *
3662 * @rdev: radeon_device pointer
3663 * @ib: radeon indirect buffer object
3664 *
3665 * Emits an DE (drawing engine) or CE (constant engine) IB
3666 * on the gfx ring. IBs are usually generated by userspace
3667 * acceleration drivers and submitted to the kernel for
3668 * sheduling on the ring. This function schedules the IB
3669 * on the gfx ring for execution by the GPU.
3670 */
3671void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3672{
3673 struct radeon_ring *ring = &rdev->ring[ib->ring];
3674 u32 header, control = INDIRECT_BUFFER_VALID;
3675
3676 if (ib->is_const_ib) {
3677 /* set switch buffer packet before const IB */
3678 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3679 radeon_ring_write(ring, 0);
3680
3681 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3682 } else {
3683 u32 next_rptr;
3684 if (ring->rptr_save_reg) {
3685 next_rptr = ring->wptr + 3 + 4;
3686 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3687 radeon_ring_write(ring, ((ring->rptr_save_reg -
3688 PACKET3_SET_UCONFIG_REG_START) >> 2));
3689 radeon_ring_write(ring, next_rptr);
3690 } else if (rdev->wb.enabled) {
3691 next_rptr = ring->wptr + 5 + 4;
3692 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3693 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3694 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3695 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3696 radeon_ring_write(ring, next_rptr);
3697 }
3698
3699 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3700 }
3701
3702 control |= ib->length_dw |
3703 (ib->vm ? (ib->vm->id << 24) : 0);
3704
3705 radeon_ring_write(ring, header);
3706 radeon_ring_write(ring,
3707#ifdef __BIG_ENDIAN
3708 (2 << 0) |
3709#endif
3710 (ib->gpu_addr & 0xFFFFFFFC));
3711 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3712 radeon_ring_write(ring, control);
3713}
3714
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3715/**
3716 * cik_ib_test - basic gfx ring IB test
3717 *
3718 * @rdev: radeon_device pointer
3719 * @ring: radeon_ring structure holding ring information
3720 *
3721 * Allocate an IB and execute it on the gfx ring (CIK).
3722 * Provides a basic gfx ring test to verify that IBs are working.
3723 * Returns 0 on success, error on failure.
3724 */
3725int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3726{
3727 struct radeon_ib ib;
3728 uint32_t scratch;
3729 uint32_t tmp = 0;
3730 unsigned i;
3731 int r;
3732
3733 r = radeon_scratch_get(rdev, &scratch);
3734 if (r) {
3735 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3736 return r;
3737 }
3738 WREG32(scratch, 0xCAFEDEAD);
3739 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3740 if (r) {
3741 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3742 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3743 return r;
3744 }
3745 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3746 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3747 ib.ptr[2] = 0xDEADBEEF;
3748 ib.length_dw = 3;
3749 r = radeon_ib_schedule(rdev, &ib, NULL);
3750 if (r) {
3751 radeon_scratch_free(rdev, scratch);
3752 radeon_ib_free(rdev, &ib);
3753 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3754 return r;
3755 }
3756 r = radeon_fence_wait(ib.fence, false);
3757 if (r) {
3758 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3759 radeon_scratch_free(rdev, scratch);
3760 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3761 return r;
3762 }
3763 for (i = 0; i < rdev->usec_timeout; i++) {
3764 tmp = RREG32(scratch);
3765 if (tmp == 0xDEADBEEF)
3766 break;
3767 DRM_UDELAY(1);
3768 }
3769 if (i < rdev->usec_timeout) {
3770 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3771 } else {
3772 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3773 scratch, tmp);
3774 r = -EINVAL;
3775 }
3776 radeon_scratch_free(rdev, scratch);
3777 radeon_ib_free(rdev, &ib);
3778 return r;
3779}
3780
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3781/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3782 * CP.
3783 * On CIK, gfx and compute now have independant command processors.
3784 *
3785 * GFX
3786 * Gfx consists of a single ring and can process both gfx jobs and
3787 * compute jobs. The gfx CP consists of three microengines (ME):
3788 * PFP - Pre-Fetch Parser
3789 * ME - Micro Engine
3790 * CE - Constant Engine
3791 * The PFP and ME make up what is considered the Drawing Engine (DE).
3792 * The CE is an asynchronous engine used for updating buffer desciptors
3793 * used by the DE so that they can be loaded into cache in parallel
3794 * while the DE is processing state update packets.
3795 *
3796 * Compute
3797 * The compute CP consists of two microengines (ME):
3798 * MEC1 - Compute MicroEngine 1
3799 * MEC2 - Compute MicroEngine 2
3800 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3801 * The queues are exposed to userspace and are programmed directly
3802 * by the compute runtime.
3803 */
3804/**
3805 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3806 *
3807 * @rdev: radeon_device pointer
3808 * @enable: enable or disable the MEs
3809 *
3810 * Halts or unhalts the gfx MEs.
3811 */
3812static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3813{
3814 if (enable)
3815 WREG32(CP_ME_CNTL, 0);
3816 else {
3817 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3818 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3819 }
3820 udelay(50);
3821}
3822
3823/**
3824 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3825 *
3826 * @rdev: radeon_device pointer
3827 *
3828 * Loads the gfx PFP, ME, and CE ucode.
3829 * Returns 0 for success, -EINVAL if the ucode is not available.
3830 */
3831static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3832{
3833 const __be32 *fw_data;
3834 int i;
3835
3836 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3837 return -EINVAL;
3838
3839 cik_cp_gfx_enable(rdev, false);
3840
3841 /* PFP */
3842 fw_data = (const __be32 *)rdev->pfp_fw->data;
3843 WREG32(CP_PFP_UCODE_ADDR, 0);
3844 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3845 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3846 WREG32(CP_PFP_UCODE_ADDR, 0);
3847
3848 /* CE */
3849 fw_data = (const __be32 *)rdev->ce_fw->data;
3850 WREG32(CP_CE_UCODE_ADDR, 0);
3851 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3852 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3853 WREG32(CP_CE_UCODE_ADDR, 0);
3854
3855 /* ME */
3856 fw_data = (const __be32 *)rdev->me_fw->data;
3857 WREG32(CP_ME_RAM_WADDR, 0);
3858 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3859 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3860 WREG32(CP_ME_RAM_WADDR, 0);
3861
3862 WREG32(CP_PFP_UCODE_ADDR, 0);
3863 WREG32(CP_CE_UCODE_ADDR, 0);
3864 WREG32(CP_ME_RAM_WADDR, 0);
3865 WREG32(CP_ME_RAM_RADDR, 0);
3866 return 0;
3867}
3868
3869/**
3870 * cik_cp_gfx_start - start the gfx ring
3871 *
3872 * @rdev: radeon_device pointer
3873 *
3874 * Enables the ring and loads the clear state context and other
3875 * packets required to init the ring.
3876 * Returns 0 for success, error for failure.
3877 */
3878static int cik_cp_gfx_start(struct radeon_device *rdev)
3879{
3880 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3881 int r, i;
3882
3883 /* init the CP */
3884 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3885 WREG32(CP_ENDIAN_SWAP, 0);
3886 WREG32(CP_DEVICE_ID, 1);
3887
3888 cik_cp_gfx_enable(rdev, true);
3889
3890 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3891 if (r) {
3892 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3893 return r;
3894 }
3895
3896 /* init the CE partitions. CE only used for gfx on CIK */
3897 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3898 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3899 radeon_ring_write(ring, 0xc000);
3900 radeon_ring_write(ring, 0xc000);
3901
3902 /* setup clear context state */
3903 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3904 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3905
3906 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3907 radeon_ring_write(ring, 0x80000000);
3908 radeon_ring_write(ring, 0x80000000);
3909
3910 for (i = 0; i < cik_default_size; i++)
3911 radeon_ring_write(ring, cik_default_state[i]);
3912
3913 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3914 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3915
3916 /* set clear context state */
3917 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3918 radeon_ring_write(ring, 0);
3919
3920 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3921 radeon_ring_write(ring, 0x00000316);
3922 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3923 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3924
3925 radeon_ring_unlock_commit(rdev, ring);
3926
3927 return 0;
3928}
3929
3930/**
3931 * cik_cp_gfx_fini - stop the gfx ring
3932 *
3933 * @rdev: radeon_device pointer
3934 *
3935 * Stop the gfx ring and tear down the driver ring
3936 * info.
3937 */
3938static void cik_cp_gfx_fini(struct radeon_device *rdev)
3939{
3940 cik_cp_gfx_enable(rdev, false);
3941 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3942}
3943
3944/**
3945 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3946 *
3947 * @rdev: radeon_device pointer
3948 *
3949 * Program the location and size of the gfx ring buffer
3950 * and test it to make sure it's working.
3951 * Returns 0 for success, error for failure.
3952 */
3953static int cik_cp_gfx_resume(struct radeon_device *rdev)
3954{
3955 struct radeon_ring *ring;
3956 u32 tmp;
3957 u32 rb_bufsz;
3958 u64 rb_addr;
3959 int r;
3960
3961 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3962 if (rdev->family != CHIP_HAWAII)
3963 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3964
3965 /* Set the write pointer delay */
3966 WREG32(CP_RB_WPTR_DELAY, 0);
3967
3968 /* set the RB to use vmid 0 */
3969 WREG32(CP_RB_VMID, 0);
3970
3971 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3972
3973 /* ring 0 - compute and gfx */
3974 /* Set ring buffer size */
3975 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3976 rb_bufsz = order_base_2(ring->ring_size / 8);
3977 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3978#ifdef __BIG_ENDIAN
3979 tmp |= BUF_SWAP_32BIT;
3980#endif
3981 WREG32(CP_RB0_CNTL, tmp);
3982
3983 /* Initialize the ring buffer's read and write pointers */
3984 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3985 ring->wptr = 0;
3986 WREG32(CP_RB0_WPTR, ring->wptr);
3987
3988 /* set the wb address wether it's enabled or not */
3989 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3990 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3991
3992 /* scratch register shadowing is no longer supported */
3993 WREG32(SCRATCH_UMSK, 0);
3994
3995 if (!rdev->wb.enabled)
3996 tmp |= RB_NO_UPDATE;
3997
3998 mdelay(1);
3999 WREG32(CP_RB0_CNTL, tmp);
4000
4001 rb_addr = ring->gpu_addr >> 8;
4002 WREG32(CP_RB0_BASE, rb_addr);
4003 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4004
4005 ring->rptr = RREG32(CP_RB0_RPTR);
4006
4007 /* start the ring */
4008 cik_cp_gfx_start(rdev);
4009 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4010 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4011 if (r) {
4012 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4013 return r;
4014 }
4015 return 0;
4016}
4017
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4018u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4019 struct radeon_ring *ring)
4020{
4021 u32 rptr;
4022
4023
4024
4025 if (rdev->wb.enabled) {
4026 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4027 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4028 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4029 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4030 rptr = RREG32(CP_HQD_PQ_RPTR);
4031 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4032 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4033 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4034
4035 return rptr;
4036}
4037
4038u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4039 struct radeon_ring *ring)
4040{
4041 u32 wptr;
4042
4043 if (rdev->wb.enabled) {
4044 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4045 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4046 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4047 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4048 wptr = RREG32(CP_HQD_PQ_WPTR);
4049 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4050 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4051 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4052
4053 return wptr;
4054}
4055
4056void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4057 struct radeon_ring *ring)
4058{
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]4059 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4060 WDOORBELL32(ring->doorbell_index, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4061}
4062
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4063/**
4064 * cik_cp_compute_enable - enable/disable the compute CP MEs
4065 *
4066 * @rdev: radeon_device pointer
4067 * @enable: enable or disable the MEs
4068 *
4069 * Halts or unhalts the compute MEs.
4070 */
4071static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4072{
4073 if (enable)
4074 WREG32(CP_MEC_CNTL, 0);
4075 else
4076 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4077 udelay(50);
4078}
4079
4080/**
4081 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4082 *
4083 * @rdev: radeon_device pointer
4084 *
4085 * Loads the compute MEC1&2 ucode.
4086 * Returns 0 for success, -EINVAL if the ucode is not available.
4087 */
4088static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4089{
4090 const __be32 *fw_data;
4091 int i;
4092
4093 if (!rdev->mec_fw)
4094 return -EINVAL;
4095
4096 cik_cp_compute_enable(rdev, false);
4097
4098 /* MEC1 */
4099 fw_data = (const __be32 *)rdev->mec_fw->data;
4100 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4101 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4102 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4103 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4104
4105 if (rdev->family == CHIP_KAVERI) {
4106 /* MEC2 */
4107 fw_data = (const __be32 *)rdev->mec_fw->data;
4108 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4109 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4110 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4111 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4112 }
4113
4114 return 0;
4115}
4116
4117/**
4118 * cik_cp_compute_start - start the compute queues
4119 *
4120 * @rdev: radeon_device pointer
4121 *
4122 * Enable the compute queues.
4123 * Returns 0 for success, error for failure.
4124 */
4125static int cik_cp_compute_start(struct radeon_device *rdev)
4126{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4127 cik_cp_compute_enable(rdev, true);
4128
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4129 return 0;
4130}
4131
4132/**
4133 * cik_cp_compute_fini - stop the compute queues
4134 *
4135 * @rdev: radeon_device pointer
4136 *
4137 * Stop the compute queues and tear down the driver queue
4138 * info.
4139 */
4140static void cik_cp_compute_fini(struct radeon_device *rdev)
4141{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4142 int i, idx, r;
4143
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4144 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4145
4146 for (i = 0; i < 2; i++) {
4147 if (i == 0)
4148 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4149 else
4150 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4151
4152 if (rdev->ring[idx].mqd_obj) {
4153 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4154 if (unlikely(r != 0))
4155 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4156
4157 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4158 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4159
4160 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4161 rdev->ring[idx].mqd_obj = NULL;
4162 }
4163 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4164}
4165
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4166static void cik_mec_fini(struct radeon_device *rdev)
4167{
4168 int r;
4169
4170 if (rdev->mec.hpd_eop_obj) {
4171 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4172 if (unlikely(r != 0))
4173 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4174 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4175 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4176
4177 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4178 rdev->mec.hpd_eop_obj = NULL;
4179 }
4180}
4181
4182#define MEC_HPD_SIZE 2048
4183
4184static int cik_mec_init(struct radeon_device *rdev)
4185{
4186 int r;
4187 u32 *hpd;
4188
4189 /*
4190 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4191 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4192 */
4193 if (rdev->family == CHIP_KAVERI)
4194 rdev->mec.num_mec = 2;
4195 else
4196 rdev->mec.num_mec = 1;
4197 rdev->mec.num_pipe = 4;
4198 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4199
4200 if (rdev->mec.hpd_eop_obj == NULL) {
4201 r = radeon_bo_create(rdev,
4202 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4203 PAGE_SIZE, true,
4204 RADEON_GEM_DOMAIN_GTT, NULL,
4205 &rdev->mec.hpd_eop_obj);
4206 if (r) {
4207 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4208 return r;
4209 }
4210 }
4211
4212 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4213 if (unlikely(r != 0)) {
4214 cik_mec_fini(rdev);
4215 return r;
4216 }
4217 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4218 &rdev->mec.hpd_eop_gpu_addr);
4219 if (r) {
4220 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4221 cik_mec_fini(rdev);
4222 return r;
4223 }
4224 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4225 if (r) {
4226 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4227 cik_mec_fini(rdev);
4228 return r;
4229 }
4230
4231 /* clear memory. Not sure if this is required or not */
4232 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4233
4234 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4235 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4236
4237 return 0;
4238}
4239
4240struct hqd_registers
4241{
4242 u32 cp_mqd_base_addr;
4243 u32 cp_mqd_base_addr_hi;
4244 u32 cp_hqd_active;
4245 u32 cp_hqd_vmid;
4246 u32 cp_hqd_persistent_state;
4247 u32 cp_hqd_pipe_priority;
4248 u32 cp_hqd_queue_priority;
4249 u32 cp_hqd_quantum;
4250 u32 cp_hqd_pq_base;
4251 u32 cp_hqd_pq_base_hi;
4252 u32 cp_hqd_pq_rptr;
4253 u32 cp_hqd_pq_rptr_report_addr;
4254 u32 cp_hqd_pq_rptr_report_addr_hi;
4255 u32 cp_hqd_pq_wptr_poll_addr;
4256 u32 cp_hqd_pq_wptr_poll_addr_hi;
4257 u32 cp_hqd_pq_doorbell_control;
4258 u32 cp_hqd_pq_wptr;
4259 u32 cp_hqd_pq_control;
4260 u32 cp_hqd_ib_base_addr;
4261 u32 cp_hqd_ib_base_addr_hi;
4262 u32 cp_hqd_ib_rptr;
4263 u32 cp_hqd_ib_control;
4264 u32 cp_hqd_iq_timer;
4265 u32 cp_hqd_iq_rptr;
4266 u32 cp_hqd_dequeue_request;
4267 u32 cp_hqd_dma_offload;
4268 u32 cp_hqd_sema_cmd;
4269 u32 cp_hqd_msg_type;
4270 u32 cp_hqd_atomic0_preop_lo;
4271 u32 cp_hqd_atomic0_preop_hi;
4272 u32 cp_hqd_atomic1_preop_lo;
4273 u32 cp_hqd_atomic1_preop_hi;
4274 u32 cp_hqd_hq_scheduler0;
4275 u32 cp_hqd_hq_scheduler1;
4276 u32 cp_mqd_control;
4277};
4278
4279struct bonaire_mqd
4280{
4281 u32 header;
4282 u32 dispatch_initiator;
4283 u32 dimensions[3];
4284 u32 start_idx[3];
4285 u32 num_threads[3];
4286 u32 pipeline_stat_enable;
4287 u32 perf_counter_enable;
4288 u32 pgm[2];
4289 u32 tba[2];
4290 u32 tma[2];
4291 u32 pgm_rsrc[2];
4292 u32 vmid;
4293 u32 resource_limits;
4294 u32 static_thread_mgmt01[2];
4295 u32 tmp_ring_size;
4296 u32 static_thread_mgmt23[2];
4297 u32 restart[3];
4298 u32 thread_trace_enable;
4299 u32 reserved1;
4300 u32 user_data[16];
4301 u32 vgtcs_invoke_count[2];
4302 struct hqd_registers queue_state;
4303 u32 dequeue_cntr;
4304 u32 interrupt_queue[64];
4305};
4306
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4307/**
4308 * cik_cp_compute_resume - setup the compute queue registers
4309 *
4310 * @rdev: radeon_device pointer
4311 *
4312 * Program the compute queues and test them to make sure they
4313 * are working.
4314 * Returns 0 for success, error for failure.
4315 */
4316static int cik_cp_compute_resume(struct radeon_device *rdev)
4317{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4318 int r, i, idx;
4319 u32 tmp;
4320 bool use_doorbell = true;
4321 u64 hqd_gpu_addr;
4322 u64 mqd_gpu_addr;
4323 u64 eop_gpu_addr;
4324 u64 wb_gpu_addr;
4325 u32 *buf;
4326 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4327
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4328 r = cik_cp_compute_start(rdev);
4329 if (r)
4330 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4331
4332 /* fix up chicken bits */
4333 tmp = RREG32(CP_CPF_DEBUG);
4334 tmp |= (1 << 23);
4335 WREG32(CP_CPF_DEBUG, tmp);
4336
4337 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4338 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4339 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4340 int me = (i < 4) ? 1 : 2;
4341 int pipe = (i < 4) ? i : (i - 4);
4342
4343 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4344
4345 cik_srbm_select(rdev, me, pipe, 0, 0);
4346
4347 /* write the EOP addr */
4348 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4349 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4350
4351 /* set the VMID assigned */
4352 WREG32(CP_HPD_EOP_VMID, 0);
4353
4354 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4355 tmp = RREG32(CP_HPD_EOP_CONTROL);
4356 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4357 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4358 WREG32(CP_HPD_EOP_CONTROL, tmp);
4359 }
4360 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4361 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4362
4363 /* init the queues. Just two for now. */
4364 for (i = 0; i < 2; i++) {
4365 if (i == 0)
4366 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4367 else
4368 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4369
4370 if (rdev->ring[idx].mqd_obj == NULL) {
4371 r = radeon_bo_create(rdev,
4372 sizeof(struct bonaire_mqd),
4373 PAGE_SIZE, true,
4374 RADEON_GEM_DOMAIN_GTT, NULL,
4375 &rdev->ring[idx].mqd_obj);
4376 if (r) {
4377 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4378 return r;
4379 }
4380 }
4381
4382 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4383 if (unlikely(r != 0)) {
4384 cik_cp_compute_fini(rdev);
4385 return r;
4386 }
4387 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4388 &mqd_gpu_addr);
4389 if (r) {
4390 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4391 cik_cp_compute_fini(rdev);
4392 return r;
4393 }
4394 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4395 if (r) {
4396 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4397 cik_cp_compute_fini(rdev);
4398 return r;
4399 }
4400
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4401 /* init the mqd struct */
4402 memset(buf, 0, sizeof(struct bonaire_mqd));
4403
4404 mqd = (struct bonaire_mqd *)buf;
4405 mqd->header = 0xC0310800;
4406 mqd->static_thread_mgmt01[0] = 0xffffffff;
4407 mqd->static_thread_mgmt01[1] = 0xffffffff;
4408 mqd->static_thread_mgmt23[0] = 0xffffffff;
4409 mqd->static_thread_mgmt23[1] = 0xffffffff;
4410
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4411 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4412 cik_srbm_select(rdev, rdev->ring[idx].me,
4413 rdev->ring[idx].pipe,
4414 rdev->ring[idx].queue, 0);
4415
4416 /* disable wptr polling */
4417 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4418 tmp &= ~WPTR_POLL_EN;
4419 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4420
4421 /* enable doorbell? */
4422 mqd->queue_state.cp_hqd_pq_doorbell_control =
4423 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4424 if (use_doorbell)
4425 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4426 else
4427 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4428 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4429 mqd->queue_state.cp_hqd_pq_doorbell_control);
4430
4431 /* disable the queue if it's active */
4432 mqd->queue_state.cp_hqd_dequeue_request = 0;
4433 mqd->queue_state.cp_hqd_pq_rptr = 0;
4434 mqd->queue_state.cp_hqd_pq_wptr= 0;
4435 if (RREG32(CP_HQD_ACTIVE) & 1) {
4436 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4437 for (i = 0; i < rdev->usec_timeout; i++) {
4438 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4439 break;
4440 udelay(1);
4441 }
4442 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4443 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4444 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4445 }
4446
4447 /* set the pointer to the MQD */
4448 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4449 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4450 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4451 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4452 /* set MQD vmid to 0 */
4453 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4454 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4455 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4456
4457 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4458 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4459 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4460 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4461 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4462 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4463
4464 /* set up the HQD, this is similar to CP_RB0_CNTL */
4465 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4466 mqd->queue_state.cp_hqd_pq_control &=
4467 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4468
4469 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4470 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4471 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4472 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4473#ifdef __BIG_ENDIAN
4474 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4475#endif
4476 mqd->queue_state.cp_hqd_pq_control &=
4477 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4478 mqd->queue_state.cp_hqd_pq_control |=
4479 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4480 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4481
4482 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4483 if (i == 0)
4484 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4485 else
4486 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4487 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4488 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4489 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4490 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4491 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4492
4493 /* set the wb address wether it's enabled or not */
4494 if (i == 0)
4495 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4496 else
4497 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4498 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4499 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4500 upper_32_bits(wb_gpu_addr) & 0xffff;
4501 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4502 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4503 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4504 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4505
4506 /* enable the doorbell if requested */
4507 if (use_doorbell) {
4508 mqd->queue_state.cp_hqd_pq_doorbell_control =
4509 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4510 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4511 mqd->queue_state.cp_hqd_pq_doorbell_control |=
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4512 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4513 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4514 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4515 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4516
4517 } else {
4518 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4519 }
4520 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4521 mqd->queue_state.cp_hqd_pq_doorbell_control);
4522
4523 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4524 rdev->ring[idx].wptr = 0;
4525 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4526 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4527 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4528 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4529
4530 /* set the vmid for the queue */
4531 mqd->queue_state.cp_hqd_vmid = 0;
4532 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4533
4534 /* activate the queue */
4535 mqd->queue_state.cp_hqd_active = 1;
4536 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4537
4538 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4539 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4540
4541 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4542 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4543
4544 rdev->ring[idx].ready = true;
4545 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4546 if (r)
4547 rdev->ring[idx].ready = false;
4548 }
4549
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4550 return 0;
4551}
4552
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4553static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4554{
4555 cik_cp_gfx_enable(rdev, enable);
4556 cik_cp_compute_enable(rdev, enable);
4557}
4558
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4559static int cik_cp_load_microcode(struct radeon_device *rdev)
4560{
4561 int r;
4562
4563 r = cik_cp_gfx_load_microcode(rdev);
4564 if (r)
4565 return r;
4566 r = cik_cp_compute_load_microcode(rdev);
4567 if (r)
4568 return r;
4569
4570 return 0;
4571}
4572
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4573static void cik_cp_fini(struct radeon_device *rdev)
4574{
4575 cik_cp_gfx_fini(rdev);
4576 cik_cp_compute_fini(rdev);
4577}
4578
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4579static int cik_cp_resume(struct radeon_device *rdev)
4580{
4581 int r;
4582
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4583 cik_enable_gui_idle_interrupt(rdev, false);
4584
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4585 r = cik_cp_load_microcode(rdev);
4586 if (r)
4587 return r;
4588
4589 r = cik_cp_gfx_resume(rdev);
4590 if (r)
4591 return r;
4592 r = cik_cp_compute_resume(rdev);
4593 if (r)
4594 return r;
4595
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4596 cik_enable_gui_idle_interrupt(rdev, true);
4597
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4598 return 0;
4599}
4600
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4601static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4602{
4603 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4604 RREG32(GRBM_STATUS));
4605 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4606 RREG32(GRBM_STATUS2));
4607 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4608 RREG32(GRBM_STATUS_SE0));
4609 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4610 RREG32(GRBM_STATUS_SE1));
4611 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4612 RREG32(GRBM_STATUS_SE2));
4613 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4614 RREG32(GRBM_STATUS_SE3));
4615 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4616 RREG32(SRBM_STATUS));
4617 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4618 RREG32(SRBM_STATUS2));
4619 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4620 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4621 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4622 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4623 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4624 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4625 RREG32(CP_STALLED_STAT1));
4626 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4627 RREG32(CP_STALLED_STAT2));
4628 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4629 RREG32(CP_STALLED_STAT3));
4630 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4631 RREG32(CP_CPF_BUSY_STAT));
4632 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4633 RREG32(CP_CPF_STALLED_STAT1));
4634 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4635 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4636 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4637 RREG32(CP_CPC_STALLED_STAT1));
4638 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4639}
4640
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4641/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4642 * cik_gpu_check_soft_reset - check which blocks are busy
4643 *
4644 * @rdev: radeon_device pointer
4645 *
4646 * Check which blocks are busy and return the relevant reset
4647 * mask to be used by cik_gpu_soft_reset().
4648 * Returns a mask of the blocks to be reset.
4649 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4650u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4651{
4652 u32 reset_mask = 0;
4653 u32 tmp;
4654
4655 /* GRBM_STATUS */
4656 tmp = RREG32(GRBM_STATUS);
4657 if (tmp & (PA_BUSY | SC_BUSY |
4658 BCI_BUSY | SX_BUSY |
4659 TA_BUSY | VGT_BUSY |
4660 DB_BUSY | CB_BUSY |
4661 GDS_BUSY | SPI_BUSY |
4662 IA_BUSY | IA_BUSY_NO_DMA))
4663 reset_mask |= RADEON_RESET_GFX;
4664
4665 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4666 reset_mask |= RADEON_RESET_CP;
4667
4668 /* GRBM_STATUS2 */
4669 tmp = RREG32(GRBM_STATUS2);
4670 if (tmp & RLC_BUSY)
4671 reset_mask |= RADEON_RESET_RLC;
4672
4673 /* SDMA0_STATUS_REG */
4674 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4675 if (!(tmp & SDMA_IDLE))
4676 reset_mask |= RADEON_RESET_DMA;
4677
4678 /* SDMA1_STATUS_REG */
4679 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4680 if (!(tmp & SDMA_IDLE))
4681 reset_mask |= RADEON_RESET_DMA1;
4682
4683 /* SRBM_STATUS2 */
4684 tmp = RREG32(SRBM_STATUS2);
4685 if (tmp & SDMA_BUSY)
4686 reset_mask |= RADEON_RESET_DMA;
4687
4688 if (tmp & SDMA1_BUSY)
4689 reset_mask |= RADEON_RESET_DMA1;
4690
4691 /* SRBM_STATUS */
4692 tmp = RREG32(SRBM_STATUS);
4693
4694 if (tmp & IH_BUSY)
4695 reset_mask |= RADEON_RESET_IH;
4696
4697 if (tmp & SEM_BUSY)
4698 reset_mask |= RADEON_RESET_SEM;
4699
4700 if (tmp & GRBM_RQ_PENDING)
4701 reset_mask |= RADEON_RESET_GRBM;
4702
4703 if (tmp & VMC_BUSY)
4704 reset_mask |= RADEON_RESET_VMC;
4705
4706 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4707 MCC_BUSY | MCD_BUSY))
4708 reset_mask |= RADEON_RESET_MC;
4709
4710 if (evergreen_is_display_hung(rdev))
4711 reset_mask |= RADEON_RESET_DISPLAY;
4712
4713 /* Skip MC reset as it's mostly likely not hung, just busy */
4714 if (reset_mask & RADEON_RESET_MC) {
4715 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4716 reset_mask &= ~RADEON_RESET_MC;
4717 }
4718
4719 return reset_mask;
4720}
4721
4722/**
4723 * cik_gpu_soft_reset - soft reset GPU
4724 *
4725 * @rdev: radeon_device pointer
4726 * @reset_mask: mask of which blocks to reset
4727 *
4728 * Soft reset the blocks specified in @reset_mask.
4729 */
4730static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4731{
4732 struct evergreen_mc_save save;
4733 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4734 u32 tmp;
4735
4736 if (reset_mask == 0)
4737 return;
4738
4739 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4740
4741 cik_print_gpu_status_regs(rdev);
4742 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4743 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4744 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4745 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4746
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4747 /* disable CG/PG */
4748 cik_fini_pg(rdev);
4749 cik_fini_cg(rdev);
4750
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4751 /* stop the rlc */
4752 cik_rlc_stop(rdev);
4753
4754 /* Disable GFX parsing/prefetching */
4755 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4756
4757 /* Disable MEC parsing/prefetching */
4758 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4759
4760 if (reset_mask & RADEON_RESET_DMA) {
4761 /* sdma0 */
4762 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4763 tmp |= SDMA_HALT;
4764 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4765 }
4766 if (reset_mask & RADEON_RESET_DMA1) {
4767 /* sdma1 */
4768 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4769 tmp |= SDMA_HALT;
4770 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4771 }
4772
4773 evergreen_mc_stop(rdev, &save);
4774 if (evergreen_mc_wait_for_idle(rdev)) {
4775 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4776 }
4777
4778 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4779 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4780
4781 if (reset_mask & RADEON_RESET_CP) {
4782 grbm_soft_reset |= SOFT_RESET_CP;
4783
4784 srbm_soft_reset |= SOFT_RESET_GRBM;
4785 }
4786
4787 if (reset_mask & RADEON_RESET_DMA)
4788 srbm_soft_reset |= SOFT_RESET_SDMA;
4789
4790 if (reset_mask & RADEON_RESET_DMA1)
4791 srbm_soft_reset |= SOFT_RESET_SDMA1;
4792
4793 if (reset_mask & RADEON_RESET_DISPLAY)
4794 srbm_soft_reset |= SOFT_RESET_DC;
4795
4796 if (reset_mask & RADEON_RESET_RLC)
4797 grbm_soft_reset |= SOFT_RESET_RLC;
4798
4799 if (reset_mask & RADEON_RESET_SEM)
4800 srbm_soft_reset |= SOFT_RESET_SEM;
4801
4802 if (reset_mask & RADEON_RESET_IH)
4803 srbm_soft_reset |= SOFT_RESET_IH;
4804
4805 if (reset_mask & RADEON_RESET_GRBM)
4806 srbm_soft_reset |= SOFT_RESET_GRBM;
4807
4808 if (reset_mask & RADEON_RESET_VMC)
4809 srbm_soft_reset |= SOFT_RESET_VMC;
4810
4811 if (!(rdev->flags & RADEON_IS_IGP)) {
4812 if (reset_mask & RADEON_RESET_MC)
4813 srbm_soft_reset |= SOFT_RESET_MC;
4814 }
4815
4816 if (grbm_soft_reset) {
4817 tmp = RREG32(GRBM_SOFT_RESET);
4818 tmp |= grbm_soft_reset;
4819 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4820 WREG32(GRBM_SOFT_RESET, tmp);
4821 tmp = RREG32(GRBM_SOFT_RESET);
4822
4823 udelay(50);
4824
4825 tmp &= ~grbm_soft_reset;
4826 WREG32(GRBM_SOFT_RESET, tmp);
4827 tmp = RREG32(GRBM_SOFT_RESET);
4828 }
4829
4830 if (srbm_soft_reset) {
4831 tmp = RREG32(SRBM_SOFT_RESET);
4832 tmp |= srbm_soft_reset;
4833 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4834 WREG32(SRBM_SOFT_RESET, tmp);
4835 tmp = RREG32(SRBM_SOFT_RESET);
4836
4837 udelay(50);
4838
4839 tmp &= ~srbm_soft_reset;
4840 WREG32(SRBM_SOFT_RESET, tmp);
4841 tmp = RREG32(SRBM_SOFT_RESET);
4842 }
4843
4844 /* Wait a little for things to settle down */
4845 udelay(50);
4846
4847 evergreen_mc_resume(rdev, &save);
4848 udelay(50);
4849
4850 cik_print_gpu_status_regs(rdev);
4851}
4852
4853/**
4854 * cik_asic_reset - soft reset GPU
4855 *
4856 * @rdev: radeon_device pointer
4857 *
4858 * Look up which blocks are hung and attempt
4859 * to reset them.
4860 * Returns 0 for success.
4861 */
4862int cik_asic_reset(struct radeon_device *rdev)
4863{
4864 u32 reset_mask;
4865
4866 reset_mask = cik_gpu_check_soft_reset(rdev);
4867
4868 if (reset_mask)
4869 r600_set_bios_scratch_engine_hung(rdev, true);
4870
4871 cik_gpu_soft_reset(rdev, reset_mask);
4872
4873 reset_mask = cik_gpu_check_soft_reset(rdev);
4874
4875 if (!reset_mask)
4876 r600_set_bios_scratch_engine_hung(rdev, false);
4877
4878 return 0;
4879}
4880
4881/**
4882 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4883 *
4884 * @rdev: radeon_device pointer
4885 * @ring: radeon_ring structure holding ring information
4886 *
4887 * Check if the 3D engine is locked up (CIK).
4888 * Returns true if the engine is locked, false if not.
4889 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4890bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4891{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4892 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4893
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4894 if (!(reset_mask & (RADEON_RESET_GFX |
4895 RADEON_RESET_COMPUTE |
4896 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4897 radeon_ring_lockup_update(ring);
4898 return false;
4899 }
4900 /* force CP activities */
4901 radeon_ring_force_activity(rdev, ring);
4902 return radeon_ring_test_lockup(rdev, ring);
4903}
4904
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4905/* MC */
4906/**
4907 * cik_mc_program - program the GPU memory controller
4908 *
4909 * @rdev: radeon_device pointer
4910 *
4911 * Set the location of vram, gart, and AGP in the GPU's
4912 * physical address space (CIK).
4913 */
4914static void cik_mc_program(struct radeon_device *rdev)
4915{
4916 struct evergreen_mc_save save;
4917 u32 tmp;
4918 int i, j;
4919
4920 /* Initialize HDP */
4921 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4922 WREG32((0x2c14 + j), 0x00000000);
4923 WREG32((0x2c18 + j), 0x00000000);
4924 WREG32((0x2c1c + j), 0x00000000);
4925 WREG32((0x2c20 + j), 0x00000000);
4926 WREG32((0x2c24 + j), 0x00000000);
4927 }
4928 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4929
4930 evergreen_mc_stop(rdev, &save);
4931 if (radeon_mc_wait_for_idle(rdev)) {
4932 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4933 }
4934 /* Lockout access through VGA aperture*/
4935 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4936 /* Update configuration */
4937 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4938 rdev->mc.vram_start >> 12);
4939 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4940 rdev->mc.vram_end >> 12);
4941 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4942 rdev->vram_scratch.gpu_addr >> 12);
4943 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4944 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4945 WREG32(MC_VM_FB_LOCATION, tmp);
4946 /* XXX double check these! */
4947 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4948 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4949 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4950 WREG32(MC_VM_AGP_BASE, 0);
4951 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4952 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4953 if (radeon_mc_wait_for_idle(rdev)) {
4954 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4955 }
4956 evergreen_mc_resume(rdev, &save);
4957 /* we need to own VRAM, so turn off the VGA renderer here
4958 * to stop it overwriting our objects */
4959 rv515_vga_render_disable(rdev);
4960}
4961
4962/**
4963 * cik_mc_init - initialize the memory controller driver params
4964 *
4965 * @rdev: radeon_device pointer
4966 *
4967 * Look up the amount of vram, vram width, and decide how to place
4968 * vram and gart within the GPU's physical address space (CIK).
4969 * Returns 0 for success.
4970 */
4971static int cik_mc_init(struct radeon_device *rdev)
4972{
4973 u32 tmp;
4974 int chansize, numchan;
4975
4976 /* Get VRAM informations */
4977 rdev->mc.vram_is_ddr = true;
4978 tmp = RREG32(MC_ARB_RAMCFG);
4979 if (tmp & CHANSIZE_MASK) {
4980 chansize = 64;
4981 } else {
4982 chansize = 32;
4983 }
4984 tmp = RREG32(MC_SHARED_CHMAP);
4985 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4986 case 0:
4987 default:
4988 numchan = 1;
4989 break;
4990 case 1:
4991 numchan = 2;
4992 break;
4993 case 2:
4994 numchan = 4;
4995 break;
4996 case 3:
4997 numchan = 8;
4998 break;
4999 case 4:
5000 numchan = 3;
5001 break;
5002 case 5:
5003 numchan = 6;
5004 break;
5005 case 6:
5006 numchan = 10;
5007 break;
5008 case 7:
5009 numchan = 12;
5010 break;
5011 case 8:
5012 numchan = 16;
5013 break;
5014 }
5015 rdev->mc.vram_width = numchan * chansize;
5016 /* Could aper size report 0 ? */
5017 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5018 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5019 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5020 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5021 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5022 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5023 si_vram_gtt_location(rdev, &rdev->mc);
5024 radeon_update_bandwidth_info(rdev);
5025
5026 return 0;
5027}
5028
5029/*
5030 * GART
5031 * VMID 0 is the physical GPU addresses as used by the kernel.
5032 * VMIDs 1-15 are used for userspace clients and are handled
5033 * by the radeon vm/hsa code.
5034 */
5035/**
5036 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5037 *
5038 * @rdev: radeon_device pointer
5039 *
5040 * Flush the TLB for the VMID 0 page table (CIK).
5041 */
5042void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5043{
5044 /* flush hdp cache */
5045 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5046
5047 /* bits 0-15 are the VM contexts0-15 */
5048 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5049}
5050
5051/**
5052 * cik_pcie_gart_enable - gart enable
5053 *
5054 * @rdev: radeon_device pointer
5055 *
5056 * This sets up the TLBs, programs the page tables for VMID0,
5057 * sets up the hw for VMIDs 1-15 which are allocated on
5058 * demand, and sets up the global locations for the LDS, GDS,
5059 * and GPUVM for FSA64 clients (CIK).
5060 * Returns 0 for success, errors for failure.
5061 */
5062static int cik_pcie_gart_enable(struct radeon_device *rdev)
5063{
5064 int r, i;
5065
5066 if (rdev->gart.robj == NULL) {
5067 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5068 return -EINVAL;
5069 }
5070 r = radeon_gart_table_vram_pin(rdev);
5071 if (r)
5072 return r;
5073 radeon_gart_restore(rdev);
5074 /* Setup TLB control */
5075 WREG32(MC_VM_MX_L1_TLB_CNTL,
5076 (0xA << 7) |
5077 ENABLE_L1_TLB |
5078 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5079 ENABLE_ADVANCED_DRIVER_MODEL |
5080 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5081 /* Setup L2 cache */
5082 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5083 ENABLE_L2_FRAGMENT_PROCESSING |
5084 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5085 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5086 EFFECTIVE_L2_QUEUE_SIZE(7) |
5087 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5088 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5089 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5090 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5091 /* setup context0 */
5092 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5093 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5094 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5095 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5096 (u32)(rdev->dummy_page.addr >> 12));
5097 WREG32(VM_CONTEXT0_CNTL2, 0);
5098 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5099 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5100
5101 WREG32(0x15D4, 0);
5102 WREG32(0x15D8, 0);
5103 WREG32(0x15DC, 0);
5104
5105 /* empty context1-15 */
5106 /* FIXME start with 4G, once using 2 level pt switch to full
5107 * vm size space
5108 */
5109 /* set vm size, must be a multiple of 4 */
5110 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5111 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5112 for (i = 1; i < 16; i++) {
5113 if (i < 8)
5114 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5115 rdev->gart.table_addr >> 12);
5116 else
5117 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5118 rdev->gart.table_addr >> 12);
5119 }
5120
5121 /* enable context1-15 */
5122 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5123 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5124 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5125 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5126 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5127 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5128 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5129 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5130 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5131 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5132 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5133 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5134 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5135 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5136 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5137 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5138
5139 /* TC cache setup ??? */
5140 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5141 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5142 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5143
5144 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5145 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5146 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5147 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5148 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5149
5150 WREG32(TC_CFG_L1_VOLATILE, 0);
5151 WREG32(TC_CFG_L2_VOLATILE, 0);
5152
5153 if (rdev->family == CHIP_KAVERI) {
5154 u32 tmp = RREG32(CHUB_CONTROL);
5155 tmp &= ~BYPASS_VM;
5156 WREG32(CHUB_CONTROL, tmp);
5157 }
5158
5159 /* XXX SH_MEM regs */
5160 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5161 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5162 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5163 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5164 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5165 WREG32(SH_MEM_CONFIG, 0);
5166 WREG32(SH_MEM_APE1_BASE, 1);
5167 WREG32(SH_MEM_APE1_LIMIT, 0);
5168 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5169 /* SDMA GFX */
5170 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5171 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5172 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5173 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5174 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5175 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5176 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5177 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5178
5179 cik_pcie_gart_tlb_flush(rdev);
5180 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5181 (unsigned)(rdev->mc.gtt_size >> 20),
5182 (unsigned long long)rdev->gart.table_addr);
5183 rdev->gart.ready = true;
5184 return 0;
5185}
5186
5187/**
5188 * cik_pcie_gart_disable - gart disable
5189 *
5190 * @rdev: radeon_device pointer
5191 *
5192 * This disables all VM page table (CIK).
5193 */
5194static void cik_pcie_gart_disable(struct radeon_device *rdev)
5195{
5196 /* Disable all tables */
5197 WREG32(VM_CONTEXT0_CNTL, 0);
5198 WREG32(VM_CONTEXT1_CNTL, 0);
5199 /* Setup TLB control */
5200 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5201 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5202 /* Setup L2 cache */
5203 WREG32(VM_L2_CNTL,
5204 ENABLE_L2_FRAGMENT_PROCESSING |
5205 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5206 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5207 EFFECTIVE_L2_QUEUE_SIZE(7) |
5208 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5209 WREG32(VM_L2_CNTL2, 0);
5210 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5211 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5212 radeon_gart_table_vram_unpin(rdev);
5213}
5214
5215/**
5216 * cik_pcie_gart_fini - vm fini callback
5217 *
5218 * @rdev: radeon_device pointer
5219 *
5220 * Tears down the driver GART/VM setup (CIK).
5221 */
5222static void cik_pcie_gart_fini(struct radeon_device *rdev)
5223{
5224 cik_pcie_gart_disable(rdev);
5225 radeon_gart_table_vram_free(rdev);
5226 radeon_gart_fini(rdev);
5227}
5228
5229/* vm parser */
5230/**
5231 * cik_ib_parse - vm ib_parse callback
5232 *
5233 * @rdev: radeon_device pointer
5234 * @ib: indirect buffer pointer
5235 *
5236 * CIK uses hw IB checking so this is a nop (CIK).
5237 */
5238int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5239{
5240 return 0;
5241}
5242
5243/*
5244 * vm
5245 * VMID 0 is the physical GPU addresses as used by the kernel.
5246 * VMIDs 1-15 are used for userspace clients and are handled
5247 * by the radeon vm/hsa code.
5248 */
5249/**
5250 * cik_vm_init - cik vm init callback
5251 *
5252 * @rdev: radeon_device pointer
5253 *
5254 * Inits cik specific vm parameters (number of VMs, base of vram for
5255 * VMIDs 1-15) (CIK).
5256 * Returns 0 for success.
5257 */
5258int cik_vm_init(struct radeon_device *rdev)
5259{
5260 /* number of VMs */
5261 rdev->vm_manager.nvm = 16;
5262 /* base offset of vram pages */
5263 if (rdev->flags & RADEON_IS_IGP) {
5264 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5265 tmp <<= 22;
5266 rdev->vm_manager.vram_base_offset = tmp;
5267 } else
5268 rdev->vm_manager.vram_base_offset = 0;
5269
5270 return 0;
5271}
5272
5273/**
5274 * cik_vm_fini - cik vm fini callback
5275 *
5276 * @rdev: radeon_device pointer
5277 *
5278 * Tear down any asic specific VM setup (CIK).
5279 */
5280void cik_vm_fini(struct radeon_device *rdev)
5281{
5282}
5283
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5284/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5285 * cik_vm_decode_fault - print human readable fault info
5286 *
5287 * @rdev: radeon_device pointer
5288 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5289 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5290 *
5291 * Print human readable fault information (CIK).
5292 */
5293static void cik_vm_decode_fault(struct radeon_device *rdev,
5294 u32 status, u32 addr, u32 mc_client)
5295{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5296 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5297 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5298 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5299 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5300 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5301
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5302 if (rdev->family == CHIP_HAWAII)
5303 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5304 else
5305 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5306
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5307 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5308 protections, vmid, addr,
5309 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5310 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5311}
5312
5313/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5314 * cik_vm_flush - cik vm flush using the CP
5315 *
5316 * @rdev: radeon_device pointer
5317 *
5318 * Update the page table base and flush the VM TLB
5319 * using the CP (CIK).
5320 */
5321void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5322{
5323 struct radeon_ring *ring = &rdev->ring[ridx];
5324
5325 if (vm == NULL)
5326 return;
5327
5328 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5329 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5330 WRITE_DATA_DST_SEL(0)));
5331 if (vm->id < 8) {
5332 radeon_ring_write(ring,
5333 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5334 } else {
5335 radeon_ring_write(ring,
5336 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5337 }
5338 radeon_ring_write(ring, 0);
5339 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5340
5341 /* update SH_MEM_* regs */
5342 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5343 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5344 WRITE_DATA_DST_SEL(0)));
5345 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5346 radeon_ring_write(ring, 0);
5347 radeon_ring_write(ring, VMID(vm->id));
5348
5349 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5350 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5351 WRITE_DATA_DST_SEL(0)));
5352 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5353 radeon_ring_write(ring, 0);
5354
5355 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5356 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5357 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5358 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5359
5360 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5361 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5362 WRITE_DATA_DST_SEL(0)));
5363 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5364 radeon_ring_write(ring, 0);
5365 radeon_ring_write(ring, VMID(0));
5366
5367 /* HDP flush */
5368 /* We should be using the WAIT_REG_MEM packet here like in
5369 * cik_fence_ring_emit(), but it causes the CP to hang in this
5370 * context...
5371 */
5372 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5373 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5374 WRITE_DATA_DST_SEL(0)));
5375 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5376 radeon_ring_write(ring, 0);
5377 radeon_ring_write(ring, 0);
5378
5379 /* bits 0-15 are the VM contexts0-15 */
5380 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5381 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5382 WRITE_DATA_DST_SEL(0)));
5383 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5384 radeon_ring_write(ring, 0);
5385 radeon_ring_write(ring, 1 << vm->id);
5386
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5387 /* compute doesn't have PFP */
5388 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5389 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5390 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5391 radeon_ring_write(ring, 0x0);
5392 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5393}
5394
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5395/*
5396 * RLC
5397 * The RLC is a multi-purpose microengine that handles a
5398 * variety of functions, the most important of which is
5399 * the interrupt controller.
5400 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5401static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5402 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5403{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5404 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5405
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5406 if (enable)
5407 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5408 else
5409 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5410 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5411}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5412
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5413static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5414{
5415 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5416
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5417 tmp = RREG32(RLC_LB_CNTL);
5418 if (enable)
5419 tmp |= LOAD_BALANCE_ENABLE;
5420 else
5421 tmp &= ~LOAD_BALANCE_ENABLE;
5422 WREG32(RLC_LB_CNTL, tmp);
5423}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5424
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5425static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5426{
5427 u32 i, j, k;
5428 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5429
5430 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5431 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5432 cik_select_se_sh(rdev, i, j);
5433 for (k = 0; k < rdev->usec_timeout; k++) {
5434 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5435 break;
5436 udelay(1);
5437 }
5438 }
5439 }
5440 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5441
5442 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5443 for (k = 0; k < rdev->usec_timeout; k++) {
5444 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5445 break;
5446 udelay(1);
5447 }
5448}
5449
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5450static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5451{
5452 u32 tmp;
5453
5454 tmp = RREG32(RLC_CNTL);
5455 if (tmp != rlc)
5456 WREG32(RLC_CNTL, rlc);
5457}
5458
5459static u32 cik_halt_rlc(struct radeon_device *rdev)
5460{
5461 u32 data, orig;
5462
5463 orig = data = RREG32(RLC_CNTL);
5464
5465 if (data & RLC_ENABLE) {
5466 u32 i;
5467
5468 data &= ~RLC_ENABLE;
5469 WREG32(RLC_CNTL, data);
5470
5471 for (i = 0; i < rdev->usec_timeout; i++) {
5472 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5473 break;
5474 udelay(1);
5475 }
5476
5477 cik_wait_for_rlc_serdes(rdev);
5478 }
5479
5480 return orig;
5481}
5482
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5483void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5484{
5485 u32 tmp, i, mask;
5486
5487 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5488 WREG32(RLC_GPR_REG2, tmp);
5489
5490 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5491 for (i = 0; i < rdev->usec_timeout; i++) {
5492 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5493 break;
5494 udelay(1);
5495 }
5496
5497 for (i = 0; i < rdev->usec_timeout; i++) {
5498 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5499 break;
5500 udelay(1);
5501 }
5502}
5503
5504void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5505{
5506 u32 tmp;
5507
5508 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5509 WREG32(RLC_GPR_REG2, tmp);
5510}
5511
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5512/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5513 * cik_rlc_stop - stop the RLC ME
5514 *
5515 * @rdev: radeon_device pointer
5516 *
5517 * Halt the RLC ME (MicroEngine) (CIK).
5518 */
5519static void cik_rlc_stop(struct radeon_device *rdev)
5520{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5521 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5522
5523 cik_enable_gui_idle_interrupt(rdev, false);
5524
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5525 cik_wait_for_rlc_serdes(rdev);
5526}
5527
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5528/**
5529 * cik_rlc_start - start the RLC ME
5530 *
5531 * @rdev: radeon_device pointer
5532 *
5533 * Unhalt the RLC ME (MicroEngine) (CIK).
5534 */
5535static void cik_rlc_start(struct radeon_device *rdev)
5536{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5537 WREG32(RLC_CNTL, RLC_ENABLE);
5538
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5539 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5540
5541 udelay(50);
5542}
5543
5544/**
5545 * cik_rlc_resume - setup the RLC hw
5546 *
5547 * @rdev: radeon_device pointer
5548 *
5549 * Initialize the RLC registers, load the ucode,
5550 * and start the RLC (CIK).
5551 * Returns 0 for success, -EINVAL if the ucode is not available.
5552 */
5553static int cik_rlc_resume(struct radeon_device *rdev)
5554{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5555 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5556 const __be32 *fw_data;
5557
5558 if (!rdev->rlc_fw)
5559 return -EINVAL;
5560
5561 switch (rdev->family) {
5562 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5563 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5564 default:
5565 size = BONAIRE_RLC_UCODE_SIZE;
5566 break;
5567 case CHIP_KAVERI:
5568 size = KV_RLC_UCODE_SIZE;
5569 break;
5570 case CHIP_KABINI:
5571 size = KB_RLC_UCODE_SIZE;
5572 break;
5573 }
5574
5575 cik_rlc_stop(rdev);
5576
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5577 /* disable CG */
5578 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5579 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5580
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5581 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5582
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5583 cik_init_pg(rdev);
5584
5585 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5586
5587 WREG32(RLC_LB_CNTR_INIT, 0);
5588 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5589
5590 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5591 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5592 WREG32(RLC_LB_PARAMS, 0x00600408);
5593 WREG32(RLC_LB_CNTL, 0x80000004);
5594
5595 WREG32(RLC_MC_CNTL, 0);
5596 WREG32(RLC_UCODE_CNTL, 0);
5597
5598 fw_data = (const __be32 *)rdev->rlc_fw->data;
5599 WREG32(RLC_GPM_UCODE_ADDR, 0);
5600 for (i = 0; i < size; i++)
5601 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5602 WREG32(RLC_GPM_UCODE_ADDR, 0);
5603
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5604 /* XXX - find out what chips support lbpw */
5605 cik_enable_lbpw(rdev, false);
5606
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5607 if (rdev->family == CHIP_BONAIRE)
5608 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5609
5610 cik_rlc_start(rdev);
5611
5612 return 0;
5613}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5614
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5615static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5616{
5617 u32 data, orig, tmp, tmp2;
5618
5619 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5620
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5621 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5622 cik_enable_gui_idle_interrupt(rdev, true);
5623
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5624 tmp = cik_halt_rlc(rdev);
5625
5626 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5627 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5628 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5629 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5630 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5631
5632 cik_update_rlc(rdev, tmp);
5633
5634 data |= CGCG_EN | CGLS_EN;
5635 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5636 cik_enable_gui_idle_interrupt(rdev, false);
5637
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5638 RREG32(CB_CGTT_SCLK_CTRL);
5639 RREG32(CB_CGTT_SCLK_CTRL);
5640 RREG32(CB_CGTT_SCLK_CTRL);
5641 RREG32(CB_CGTT_SCLK_CTRL);
5642
5643 data &= ~(CGCG_EN | CGLS_EN);
5644 }
5645
5646 if (orig != data)
5647 WREG32(RLC_CGCG_CGLS_CTRL, data);
5648
5649}
5650
5651static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5652{
5653 u32 data, orig, tmp = 0;
5654
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5655 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5656 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5657 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5658 orig = data = RREG32(CP_MEM_SLP_CNTL);
5659 data |= CP_MEM_LS_EN;
5660 if (orig != data)
5661 WREG32(CP_MEM_SLP_CNTL, data);
5662 }
5663 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5664
5665 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5666 data &= 0xfffffffd;
5667 if (orig != data)
5668 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5669
5670 tmp = cik_halt_rlc(rdev);
5671
5672 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5673 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5674 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5675 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5676 WREG32(RLC_SERDES_WR_CTRL, data);
5677
5678 cik_update_rlc(rdev, tmp);
5679
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5680 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5681 orig = data = RREG32(CGTS_SM_CTRL_REG);
5682 data &= ~SM_MODE_MASK;
5683 data |= SM_MODE(0x2);
5684 data |= SM_MODE_ENABLE;
5685 data &= ~CGTS_OVERRIDE;
5686 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5687 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5688 data &= ~CGTS_LS_OVERRIDE;
5689 data &= ~ON_MONITOR_ADD_MASK;
5690 data |= ON_MONITOR_ADD_EN;
5691 data |= ON_MONITOR_ADD(0x96);
5692 if (orig != data)
5693 WREG32(CGTS_SM_CTRL_REG, data);
5694 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5695 } else {
5696 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5697 data |= 0x00000002;
5698 if (orig != data)
5699 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5700
5701 data = RREG32(RLC_MEM_SLP_CNTL);
5702 if (data & RLC_MEM_LS_EN) {
5703 data &= ~RLC_MEM_LS_EN;
5704 WREG32(RLC_MEM_SLP_CNTL, data);
5705 }
5706
5707 data = RREG32(CP_MEM_SLP_CNTL);
5708 if (data & CP_MEM_LS_EN) {
5709 data &= ~CP_MEM_LS_EN;
5710 WREG32(CP_MEM_SLP_CNTL, data);
5711 }
5712
5713 orig = data = RREG32(CGTS_SM_CTRL_REG);
5714 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5715 if (orig != data)
5716 WREG32(CGTS_SM_CTRL_REG, data);
5717
5718 tmp = cik_halt_rlc(rdev);
5719
5720 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5721 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5722 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5723 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5724 WREG32(RLC_SERDES_WR_CTRL, data);
5725
5726 cik_update_rlc(rdev, tmp);
5727 }
5728}
5729
5730static const u32 mc_cg_registers[] =
5731{
5732 MC_HUB_MISC_HUB_CG,
5733 MC_HUB_MISC_SIP_CG,
5734 MC_HUB_MISC_VM_CG,
5735 MC_XPB_CLK_GAT,
5736 ATC_MISC_CG,
5737 MC_CITF_MISC_WR_CG,
5738 MC_CITF_MISC_RD_CG,
5739 MC_CITF_MISC_VM_CG,
5740 VM_L2_CG,
5741};
5742
5743static void cik_enable_mc_ls(struct radeon_device *rdev,
5744 bool enable)
5745{
5746 int i;
5747 u32 orig, data;
5748
5749 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5750 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5751 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5752 data |= MC_LS_ENABLE;
5753 else
5754 data &= ~MC_LS_ENABLE;
5755 if (data != orig)
5756 WREG32(mc_cg_registers[i], data);
5757 }
5758}
5759
5760static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5761 bool enable)
5762{
5763 int i;
5764 u32 orig, data;
5765
5766 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5767 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5768 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5769 data |= MC_CG_ENABLE;
5770 else
5771 data &= ~MC_CG_ENABLE;
5772 if (data != orig)
5773 WREG32(mc_cg_registers[i], data);
5774 }
5775}
5776
5777static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5778 bool enable)
5779{
5780 u32 orig, data;
5781
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5782 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5783 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5784 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5785 } else {
5786 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5787 data |= 0xff000000;
5788 if (data != orig)
5789 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5790
5791 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5792 data |= 0xff000000;
5793 if (data != orig)
5794 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5795 }
5796}
5797
5798static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5799 bool enable)
5800{
5801 u32 orig, data;
5802
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5803 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5804 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5805 data |= 0x100;
5806 if (orig != data)
5807 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5808
5809 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5810 data |= 0x100;
5811 if (orig != data)
5812 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5813 } else {
5814 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5815 data &= ~0x100;
5816 if (orig != data)
5817 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5818
5819 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5820 data &= ~0x100;
5821 if (orig != data)
5822 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5823 }
5824}
5825
5826static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5827 bool enable)
5828{
5829 u32 orig, data;
5830
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5831 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5832 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5833 data = 0xfff;
5834 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5835
5836 orig = data = RREG32(UVD_CGC_CTRL);
5837 data |= DCM;
5838 if (orig != data)
5839 WREG32(UVD_CGC_CTRL, data);
5840 } else {
5841 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5842 data &= ~0xfff;
5843 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5844
5845 orig = data = RREG32(UVD_CGC_CTRL);
5846 data &= ~DCM;
5847 if (orig != data)
5848 WREG32(UVD_CGC_CTRL, data);
5849 }
5850}
5851
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5852static void cik_enable_bif_mgls(struct radeon_device *rdev,
5853 bool enable)
5854{
5855 u32 orig, data;
5856
5857 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5858
5859 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5860 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5861 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5862 else
5863 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5864 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5865
5866 if (orig != data)
5867 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5868}
5869
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5870static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5871 bool enable)
5872{
5873 u32 orig, data;
5874
5875 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5876
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5877 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5878 data &= ~CLOCK_GATING_DIS;
5879 else
5880 data |= CLOCK_GATING_DIS;
5881
5882 if (orig != data)
5883 WREG32(HDP_HOST_PATH_CNTL, data);
5884}
5885
5886static void cik_enable_hdp_ls(struct radeon_device *rdev,
5887 bool enable)
5888{
5889 u32 orig, data;
5890
5891 orig = data = RREG32(HDP_MEM_POWER_LS);
5892
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5893 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5894 data |= HDP_LS_ENABLE;
5895 else
5896 data &= ~HDP_LS_ENABLE;
5897
5898 if (orig != data)
5899 WREG32(HDP_MEM_POWER_LS, data);
5900}
5901
5902void cik_update_cg(struct radeon_device *rdev,
5903 u32 block, bool enable)
5904{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5905
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5906 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5907 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5908 /* order matters! */
5909 if (enable) {
5910 cik_enable_mgcg(rdev, true);
5911 cik_enable_cgcg(rdev, true);
5912 } else {
5913 cik_enable_cgcg(rdev, false);
5914 cik_enable_mgcg(rdev, false);
5915 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5916 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5917 }
5918
5919 if (block & RADEON_CG_BLOCK_MC) {
5920 if (!(rdev->flags & RADEON_IS_IGP)) {
5921 cik_enable_mc_mgcg(rdev, enable);
5922 cik_enable_mc_ls(rdev, enable);
5923 }
5924 }
5925
5926 if (block & RADEON_CG_BLOCK_SDMA) {
5927 cik_enable_sdma_mgcg(rdev, enable);
5928 cik_enable_sdma_mgls(rdev, enable);
5929 }
5930
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5931 if (block & RADEON_CG_BLOCK_BIF) {
5932 cik_enable_bif_mgls(rdev, enable);
5933 }
5934
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5935 if (block & RADEON_CG_BLOCK_UVD) {
5936 if (rdev->has_uvd)
5937 cik_enable_uvd_mgcg(rdev, enable);
5938 }
5939
5940 if (block & RADEON_CG_BLOCK_HDP) {
5941 cik_enable_hdp_mgcg(rdev, enable);
5942 cik_enable_hdp_ls(rdev, enable);
5943 }
5944}
5945
5946static void cik_init_cg(struct radeon_device *rdev)
5947{
5948
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5949 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5950
5951 if (rdev->has_uvd)
5952 si_init_uvd_internal_cg(rdev);
5953
5954 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5955 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5956 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5957 RADEON_CG_BLOCK_UVD |
5958 RADEON_CG_BLOCK_HDP), true);
5959}
5960
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5961static void cik_fini_cg(struct radeon_device *rdev)
5962{
5963 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5964 RADEON_CG_BLOCK_SDMA |
5965 RADEON_CG_BLOCK_BIF |
5966 RADEON_CG_BLOCK_UVD |
5967 RADEON_CG_BLOCK_HDP), false);
5968
5969 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5970}
5971
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5972static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5973 bool enable)
5974{
5975 u32 data, orig;
5976
5977 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5978 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5979 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5980 else
5981 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5982 if (orig != data)
5983 WREG32(RLC_PG_CNTL, data);
5984}
5985
5986static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5987 bool enable)
5988{
5989 u32 data, orig;
5990
5991 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5992 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5993 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5994 else
5995 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5996 if (orig != data)
5997 WREG32(RLC_PG_CNTL, data);
5998}
5999
6000static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6001{
6002 u32 data, orig;
6003
6004 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6005 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6006 data &= ~DISABLE_CP_PG;
6007 else
6008 data |= DISABLE_CP_PG;
6009 if (orig != data)
6010 WREG32(RLC_PG_CNTL, data);
6011}
6012
6013static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6014{
6015 u32 data, orig;
6016
6017 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6018 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6019 data &= ~DISABLE_GDS_PG;
6020 else
6021 data |= DISABLE_GDS_PG;
6022 if (orig != data)
6023 WREG32(RLC_PG_CNTL, data);
6024}
6025
6026#define CP_ME_TABLE_SIZE 96
6027#define CP_ME_TABLE_OFFSET 2048
6028#define CP_MEC_TABLE_OFFSET 4096
6029
6030void cik_init_cp_pg_table(struct radeon_device *rdev)
6031{
6032 const __be32 *fw_data;
6033 volatile u32 *dst_ptr;
6034 int me, i, max_me = 4;
6035 u32 bo_offset = 0;
6036 u32 table_offset;
6037
6038 if (rdev->family == CHIP_KAVERI)
6039 max_me = 5;
6040
6041 if (rdev->rlc.cp_table_ptr == NULL)
6042 return;
6043
6044 /* write the cp table buffer */
6045 dst_ptr = rdev->rlc.cp_table_ptr;
6046 for (me = 0; me < max_me; me++) {
6047 if (me == 0) {
6048 fw_data = (const __be32 *)rdev->ce_fw->data;
6049 table_offset = CP_ME_TABLE_OFFSET;
6050 } else if (me == 1) {
6051 fw_data = (const __be32 *)rdev->pfp_fw->data;
6052 table_offset = CP_ME_TABLE_OFFSET;
6053 } else if (me == 2) {
6054 fw_data = (const __be32 *)rdev->me_fw->data;
6055 table_offset = CP_ME_TABLE_OFFSET;
6056 } else {
6057 fw_data = (const __be32 *)rdev->mec_fw->data;
6058 table_offset = CP_MEC_TABLE_OFFSET;
6059 }
6060
6061 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6062 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6063 }
6064 bo_offset += CP_ME_TABLE_SIZE;
6065 }
6066}
6067
6068static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6069 bool enable)
6070{
6071 u32 data, orig;
6072
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6073 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6074 orig = data = RREG32(RLC_PG_CNTL);
6075 data |= GFX_PG_ENABLE;
6076 if (orig != data)
6077 WREG32(RLC_PG_CNTL, data);
6078
6079 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6080 data |= AUTO_PG_EN;
6081 if (orig != data)
6082 WREG32(RLC_AUTO_PG_CTRL, data);
6083 } else {
6084 orig = data = RREG32(RLC_PG_CNTL);
6085 data &= ~GFX_PG_ENABLE;
6086 if (orig != data)
6087 WREG32(RLC_PG_CNTL, data);
6088
6089 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6090 data &= ~AUTO_PG_EN;
6091 if (orig != data)
6092 WREG32(RLC_AUTO_PG_CTRL, data);
6093
6094 data = RREG32(DB_RENDER_CONTROL);
6095 }
6096}
6097
6098static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6099{
6100 u32 mask = 0, tmp, tmp1;
6101 int i;
6102
6103 cik_select_se_sh(rdev, se, sh);
6104 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6105 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6106 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6107
6108 tmp &= 0xffff0000;
6109
6110 tmp |= tmp1;
6111 tmp >>= 16;
6112
6113 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6114 mask <<= 1;
6115 mask |= 1;
6116 }
6117
6118 return (~tmp) & mask;
6119}
6120
6121static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6122{
6123 u32 i, j, k, active_cu_number = 0;
6124 u32 mask, counter, cu_bitmap;
6125 u32 tmp = 0;
6126
6127 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6128 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6129 mask = 1;
6130 cu_bitmap = 0;
6131 counter = 0;
6132 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6133 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6134 if (counter < 2)
6135 cu_bitmap |= mask;
6136 counter ++;
6137 }
6138 mask <<= 1;
6139 }
6140
6141 active_cu_number += counter;
6142 tmp |= (cu_bitmap << (i * 16 + j * 8));
6143 }
6144 }
6145
6146 WREG32(RLC_PG_AO_CU_MASK, tmp);
6147
6148 tmp = RREG32(RLC_MAX_PG_CU);
6149 tmp &= ~MAX_PU_CU_MASK;
6150 tmp |= MAX_PU_CU(active_cu_number);
6151 WREG32(RLC_MAX_PG_CU, tmp);
6152}
6153
6154static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6155 bool enable)
6156{
6157 u32 data, orig;
6158
6159 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6160 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6161 data |= STATIC_PER_CU_PG_ENABLE;
6162 else
6163 data &= ~STATIC_PER_CU_PG_ENABLE;
6164 if (orig != data)
6165 WREG32(RLC_PG_CNTL, data);
6166}
6167
6168static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6169 bool enable)
6170{
6171 u32 data, orig;
6172
6173 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6174 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6175 data |= DYN_PER_CU_PG_ENABLE;
6176 else
6177 data &= ~DYN_PER_CU_PG_ENABLE;
6178 if (orig != data)
6179 WREG32(RLC_PG_CNTL, data);
6180}
6181
6182#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6183#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6184
6185static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6186{
6187 u32 data, orig;
6188 u32 i;
6189
6190 if (rdev->rlc.cs_data) {
6191 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6192 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6193 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6194 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6195 } else {
6196 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6197 for (i = 0; i < 3; i++)
6198 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6199 }
6200 if (rdev->rlc.reg_list) {
6201 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6202 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6203 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6204 }
6205
6206 orig = data = RREG32(RLC_PG_CNTL);
6207 data |= GFX_PG_SRC;
6208 if (orig != data)
6209 WREG32(RLC_PG_CNTL, data);
6210
6211 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6212 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6213
6214 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6215 data &= ~IDLE_POLL_COUNT_MASK;
6216 data |= IDLE_POLL_COUNT(0x60);
6217 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6218
6219 data = 0x10101010;
6220 WREG32(RLC_PG_DELAY, data);
6221
6222 data = RREG32(RLC_PG_DELAY_2);
6223 data &= ~0xff;
6224 data |= 0x3;
6225 WREG32(RLC_PG_DELAY_2, data);
6226
6227 data = RREG32(RLC_AUTO_PG_CTRL);
6228 data &= ~GRBM_REG_SGIT_MASK;
6229 data |= GRBM_REG_SGIT(0x700);
6230 WREG32(RLC_AUTO_PG_CTRL, data);
6231
6232}
6233
6234static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6235{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6236 cik_enable_gfx_cgpg(rdev, enable);
6237 cik_enable_gfx_static_mgpg(rdev, enable);
6238 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6239}
6240
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6241u32 cik_get_csb_size(struct radeon_device *rdev)
6242{
6243 u32 count = 0;
6244 const struct cs_section_def *sect = NULL;
6245 const struct cs_extent_def *ext = NULL;
6246
6247 if (rdev->rlc.cs_data == NULL)
6248 return 0;
6249
6250 /* begin clear state */
6251 count += 2;
6252 /* context control state */
6253 count += 3;
6254
6255 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6256 for (ext = sect->section; ext->extent != NULL; ++ext) {
6257 if (sect->id == SECT_CONTEXT)
6258 count += 2 + ext->reg_count;
6259 else
6260 return 0;
6261 }
6262 }
6263 /* pa_sc_raster_config/pa_sc_raster_config1 */
6264 count += 4;
6265 /* end clear state */
6266 count += 2;
6267 /* clear state */
6268 count += 2;
6269
6270 return count;
6271}
6272
6273void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6274{
6275 u32 count = 0, i;
6276 const struct cs_section_def *sect = NULL;
6277 const struct cs_extent_def *ext = NULL;
6278
6279 if (rdev->rlc.cs_data == NULL)
6280 return;
6281 if (buffer == NULL)
6282 return;
6283
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6284 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6285 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6286
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6287 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6288 buffer[count++] = cpu_to_le32(0x80000000);
6289 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6290
6291 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6292 for (ext = sect->section; ext->extent != NULL; ++ext) {
6293 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6294 buffer[count++] =
6295 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6296 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6297 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6298 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6299 } else {
6300 return;
6301 }
6302 }
6303 }
6304
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6305 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6306 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6307 switch (rdev->family) {
6308 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6309 buffer[count++] = cpu_to_le32(0x16000012);
6310 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6311 break;
6312 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6313 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6314 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6315 break;
6316 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6317 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6318 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6319 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame]6320 case CHIP_HAWAII:
6321 buffer[count++] = 0x3a00161a;
6322 buffer[count++] = 0x0000002e;
6323 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6324 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6325 buffer[count++] = cpu_to_le32(0x00000000);
6326 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6327 break;
6328 }
6329
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6330 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6331 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6332
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6333 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6334 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6335}
6336
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6337static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6338{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6339 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6340 cik_enable_sck_slowdown_on_pu(rdev, true);
6341 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6342 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6343 cik_init_gfx_cgpg(rdev);
6344 cik_enable_cp_pg(rdev, true);
6345 cik_enable_gds_pg(rdev, true);
6346 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6347 cik_init_ao_cu_mask(rdev);
6348 cik_update_gfx_pg(rdev, true);
6349 }
6350}
6351
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6352static void cik_fini_pg(struct radeon_device *rdev)
6353{
6354 if (rdev->pg_flags) {
6355 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6356 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6357 cik_enable_cp_pg(rdev, false);
6358 cik_enable_gds_pg(rdev, false);
6359 }
6360 }
6361}
6362
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6363/*
6364 * Interrupts
6365 * Starting with r6xx, interrupts are handled via a ring buffer.
6366 * Ring buffers are areas of GPU accessible memory that the GPU
6367 * writes interrupt vectors into and the host reads vectors out of.
6368 * There is a rptr (read pointer) that determines where the
6369 * host is currently reading, and a wptr (write pointer)
6370 * which determines where the GPU has written. When the
6371 * pointers are equal, the ring is idle. When the GPU
6372 * writes vectors to the ring buffer, it increments the
6373 * wptr. When there is an interrupt, the host then starts
6374 * fetching commands and processing them until the pointers are
6375 * equal again at which point it updates the rptr.
6376 */
6377
6378/**
6379 * cik_enable_interrupts - Enable the interrupt ring buffer
6380 *
6381 * @rdev: radeon_device pointer
6382 *
6383 * Enable the interrupt ring buffer (CIK).
6384 */
6385static void cik_enable_interrupts(struct radeon_device *rdev)
6386{
6387 u32 ih_cntl = RREG32(IH_CNTL);
6388 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6389
6390 ih_cntl |= ENABLE_INTR;
6391 ih_rb_cntl |= IH_RB_ENABLE;
6392 WREG32(IH_CNTL, ih_cntl);
6393 WREG32(IH_RB_CNTL, ih_rb_cntl);
6394 rdev->ih.enabled = true;
6395}
6396
6397/**
6398 * cik_disable_interrupts - Disable the interrupt ring buffer
6399 *
6400 * @rdev: radeon_device pointer
6401 *
6402 * Disable the interrupt ring buffer (CIK).
6403 */
6404static void cik_disable_interrupts(struct radeon_device *rdev)
6405{
6406 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6407 u32 ih_cntl = RREG32(IH_CNTL);
6408
6409 ih_rb_cntl &= ~IH_RB_ENABLE;
6410 ih_cntl &= ~ENABLE_INTR;
6411 WREG32(IH_RB_CNTL, ih_rb_cntl);
6412 WREG32(IH_CNTL, ih_cntl);
6413 /* set rptr, wptr to 0 */
6414 WREG32(IH_RB_RPTR, 0);
6415 WREG32(IH_RB_WPTR, 0);
6416 rdev->ih.enabled = false;
6417 rdev->ih.rptr = 0;
6418}
6419
6420/**
6421 * cik_disable_interrupt_state - Disable all interrupt sources
6422 *
6423 * @rdev: radeon_device pointer
6424 *
6425 * Clear all interrupt enable bits used by the driver (CIK).
6426 */
6427static void cik_disable_interrupt_state(struct radeon_device *rdev)
6428{
6429 u32 tmp;
6430
6431 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6432 tmp = RREG32(CP_INT_CNTL_RING0) &
6433 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6434 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6435 /* sdma */
6436 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6437 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6438 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6439 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6440 /* compute queues */
6441 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6442 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6443 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6444 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6445 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6446 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6447 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6448 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6449 /* grbm */
6450 WREG32(GRBM_INT_CNTL, 0);
6451 /* vline/vblank, etc. */
6452 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6453 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6454 if (rdev->num_crtc >= 4) {
6455 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6456 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6457 }
6458 if (rdev->num_crtc >= 6) {
6459 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6460 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6461 }
6462
6463 /* dac hotplug */
6464 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6465
6466 /* digital hotplug */
6467 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6468 WREG32(DC_HPD1_INT_CONTROL, tmp);
6469 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6470 WREG32(DC_HPD2_INT_CONTROL, tmp);
6471 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6472 WREG32(DC_HPD3_INT_CONTROL, tmp);
6473 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6474 WREG32(DC_HPD4_INT_CONTROL, tmp);
6475 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6476 WREG32(DC_HPD5_INT_CONTROL, tmp);
6477 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6478 WREG32(DC_HPD6_INT_CONTROL, tmp);
6479
6480}
6481
6482/**
6483 * cik_irq_init - init and enable the interrupt ring
6484 *
6485 * @rdev: radeon_device pointer
6486 *
6487 * Allocate a ring buffer for the interrupt controller,
6488 * enable the RLC, disable interrupts, enable the IH
6489 * ring buffer and enable it (CIK).
6490 * Called at device load and reume.
6491 * Returns 0 for success, errors for failure.
6492 */
6493static int cik_irq_init(struct radeon_device *rdev)
6494{
6495 int ret = 0;
6496 int rb_bufsz;
6497 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6498
6499 /* allocate ring */
6500 ret = r600_ih_ring_alloc(rdev);
6501 if (ret)
6502 return ret;
6503
6504 /* disable irqs */
6505 cik_disable_interrupts(rdev);
6506
6507 /* init rlc */
6508 ret = cik_rlc_resume(rdev);
6509 if (ret) {
6510 r600_ih_ring_fini(rdev);
6511 return ret;
6512 }
6513
6514 /* setup interrupt control */
6515 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6516 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6517 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6518 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6519 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6520 */
6521 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6522 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6523 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6524 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6525
6526 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6527 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6528
6529 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6530 IH_WPTR_OVERFLOW_CLEAR |
6531 (rb_bufsz << 1));
6532
6533 if (rdev->wb.enabled)
6534 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6535
6536 /* set the writeback address whether it's enabled or not */
6537 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6538 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6539
6540 WREG32(IH_RB_CNTL, ih_rb_cntl);
6541
6542 /* set rptr, wptr to 0 */
6543 WREG32(IH_RB_RPTR, 0);
6544 WREG32(IH_RB_WPTR, 0);
6545
6546 /* Default settings for IH_CNTL (disabled at first) */
6547 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6548 /* RPTR_REARM only works if msi's are enabled */
6549 if (rdev->msi_enabled)
6550 ih_cntl |= RPTR_REARM;
6551 WREG32(IH_CNTL, ih_cntl);
6552
6553 /* force the active interrupt state to all disabled */
6554 cik_disable_interrupt_state(rdev);
6555
6556 pci_set_master(rdev->pdev);
6557
6558 /* enable irqs */
6559 cik_enable_interrupts(rdev);
6560
6561 return ret;
6562}
6563
6564/**
6565 * cik_irq_set - enable/disable interrupt sources
6566 *
6567 * @rdev: radeon_device pointer
6568 *
6569 * Enable interrupt sources on the GPU (vblanks, hpd,
6570 * etc.) (CIK).
6571 * Returns 0 for success, errors for failure.
6572 */
6573int cik_irq_set(struct radeon_device *rdev)
6574{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6575 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6576 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6577 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6578 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6579 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6580 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6581 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6582 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6583
6584 if (!rdev->irq.installed) {
6585 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6586 return -EINVAL;
6587 }
6588 /* don't enable anything if the ih is disabled */
6589 if (!rdev->ih.enabled) {
6590 cik_disable_interrupts(rdev);
6591 /* force the active interrupt state to all disabled */
6592 cik_disable_interrupt_state(rdev);
6593 return 0;
6594 }
6595
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6596 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6597 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6598 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6599
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6600 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6601 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6602 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6604 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6605 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6606
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6607 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6608 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6609
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6610 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6611 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6612 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6616 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6617 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6618
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6619 if (rdev->flags & RADEON_IS_IGP)
6620 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6621 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6622 else
6623 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6624 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6625
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6626 /* enable CP interrupts on all rings */
6627 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6628 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6629 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6630 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6631 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6632 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6633 DRM_DEBUG("si_irq_set: sw int cp1\n");
6634 if (ring->me == 1) {
6635 switch (ring->pipe) {
6636 case 0:
6637 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6638 break;
6639 case 1:
6640 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6641 break;
6642 case 2:
6643 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6644 break;
6645 case 3:
6646 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6647 break;
6648 default:
6649 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6650 break;
6651 }
6652 } else if (ring->me == 2) {
6653 switch (ring->pipe) {
6654 case 0:
6655 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6656 break;
6657 case 1:
6658 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6659 break;
6660 case 2:
6661 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6662 break;
6663 case 3:
6664 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6665 break;
6666 default:
6667 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6668 break;
6669 }
6670 } else {
6671 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6672 }
6673 }
6674 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6675 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6676 DRM_DEBUG("si_irq_set: sw int cp2\n");
6677 if (ring->me == 1) {
6678 switch (ring->pipe) {
6679 case 0:
6680 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6681 break;
6682 case 1:
6683 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6684 break;
6685 case 2:
6686 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6687 break;
6688 case 3:
6689 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6690 break;
6691 default:
6692 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6693 break;
6694 }
6695 } else if (ring->me == 2) {
6696 switch (ring->pipe) {
6697 case 0:
6698 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6699 break;
6700 case 1:
6701 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6702 break;
6703 case 2:
6704 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6705 break;
6706 case 3:
6707 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6708 break;
6709 default:
6710 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6711 break;
6712 }
6713 } else {
6714 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6715 }
6716 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6717
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6718 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6719 DRM_DEBUG("cik_irq_set: sw int dma\n");
6720 dma_cntl |= TRAP_ENABLE;
6721 }
6722
6723 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6724 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6725 dma_cntl1 |= TRAP_ENABLE;
6726 }
6727
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6728 if (rdev->irq.crtc_vblank_int[0] ||
6729 atomic_read(&rdev->irq.pflip[0])) {
6730 DRM_DEBUG("cik_irq_set: vblank 0\n");
6731 crtc1 |= VBLANK_INTERRUPT_MASK;
6732 }
6733 if (rdev->irq.crtc_vblank_int[1] ||
6734 atomic_read(&rdev->irq.pflip[1])) {
6735 DRM_DEBUG("cik_irq_set: vblank 1\n");
6736 crtc2 |= VBLANK_INTERRUPT_MASK;
6737 }
6738 if (rdev->irq.crtc_vblank_int[2] ||
6739 atomic_read(&rdev->irq.pflip[2])) {
6740 DRM_DEBUG("cik_irq_set: vblank 2\n");
6741 crtc3 |= VBLANK_INTERRUPT_MASK;
6742 }
6743 if (rdev->irq.crtc_vblank_int[3] ||
6744 atomic_read(&rdev->irq.pflip[3])) {
6745 DRM_DEBUG("cik_irq_set: vblank 3\n");
6746 crtc4 |= VBLANK_INTERRUPT_MASK;
6747 }
6748 if (rdev->irq.crtc_vblank_int[4] ||
6749 atomic_read(&rdev->irq.pflip[4])) {
6750 DRM_DEBUG("cik_irq_set: vblank 4\n");
6751 crtc5 |= VBLANK_INTERRUPT_MASK;
6752 }
6753 if (rdev->irq.crtc_vblank_int[5] ||
6754 atomic_read(&rdev->irq.pflip[5])) {
6755 DRM_DEBUG("cik_irq_set: vblank 5\n");
6756 crtc6 |= VBLANK_INTERRUPT_MASK;
6757 }
6758 if (rdev->irq.hpd[0]) {
6759 DRM_DEBUG("cik_irq_set: hpd 1\n");
6760 hpd1 |= DC_HPDx_INT_EN;
6761 }
6762 if (rdev->irq.hpd[1]) {
6763 DRM_DEBUG("cik_irq_set: hpd 2\n");
6764 hpd2 |= DC_HPDx_INT_EN;
6765 }
6766 if (rdev->irq.hpd[2]) {
6767 DRM_DEBUG("cik_irq_set: hpd 3\n");
6768 hpd3 |= DC_HPDx_INT_EN;
6769 }
6770 if (rdev->irq.hpd[3]) {
6771 DRM_DEBUG("cik_irq_set: hpd 4\n");
6772 hpd4 |= DC_HPDx_INT_EN;
6773 }
6774 if (rdev->irq.hpd[4]) {
6775 DRM_DEBUG("cik_irq_set: hpd 5\n");
6776 hpd5 |= DC_HPDx_INT_EN;
6777 }
6778 if (rdev->irq.hpd[5]) {
6779 DRM_DEBUG("cik_irq_set: hpd 6\n");
6780 hpd6 |= DC_HPDx_INT_EN;
6781 }
6782
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6783 if (rdev->irq.dpm_thermal) {
6784 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6785 if (rdev->flags & RADEON_IS_IGP)
6786 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6787 else
6788 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6789 }
6790
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6791 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6792
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6793 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6794 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6795
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6796 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6797 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6798 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6799 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6800 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6801 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6802 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6803 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6804
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6805 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6806
6807 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6808 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6809 if (rdev->num_crtc >= 4) {
6810 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6811 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6812 }
6813 if (rdev->num_crtc >= 6) {
6814 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6815 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6816 }
6817
6818 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6819 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6820 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6821 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6822 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6823 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6824
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6825 if (rdev->flags & RADEON_IS_IGP)
6826 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6827 else
6828 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6829
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6830 return 0;
6831}
6832
6833/**
6834 * cik_irq_ack - ack interrupt sources
6835 *
6836 * @rdev: radeon_device pointer
6837 *
6838 * Ack interrupt sources on the GPU (vblanks, hpd,
6839 * etc.) (CIK). Certain interrupts sources are sw
6840 * generated and do not require an explicit ack.
6841 */
6842static inline void cik_irq_ack(struct radeon_device *rdev)
6843{
6844 u32 tmp;
6845
6846 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6847 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6848 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6849 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6850 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6851 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6852 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6853
6854 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6855 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6856 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6857 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6858 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6859 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6860 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6861 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6862
6863 if (rdev->num_crtc >= 4) {
6864 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6865 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6866 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6867 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6868 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6869 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6870 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6871 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6872 }
6873
6874 if (rdev->num_crtc >= 6) {
6875 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6876 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6877 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6878 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6879 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6880 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6881 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6882 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6883 }
6884
6885 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6886 tmp = RREG32(DC_HPD1_INT_CONTROL);
6887 tmp |= DC_HPDx_INT_ACK;
6888 WREG32(DC_HPD1_INT_CONTROL, tmp);
6889 }
6890 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6891 tmp = RREG32(DC_HPD2_INT_CONTROL);
6892 tmp |= DC_HPDx_INT_ACK;
6893 WREG32(DC_HPD2_INT_CONTROL, tmp);
6894 }
6895 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6896 tmp = RREG32(DC_HPD3_INT_CONTROL);
6897 tmp |= DC_HPDx_INT_ACK;
6898 WREG32(DC_HPD3_INT_CONTROL, tmp);
6899 }
6900 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6901 tmp = RREG32(DC_HPD4_INT_CONTROL);
6902 tmp |= DC_HPDx_INT_ACK;
6903 WREG32(DC_HPD4_INT_CONTROL, tmp);
6904 }
6905 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6906 tmp = RREG32(DC_HPD5_INT_CONTROL);
6907 tmp |= DC_HPDx_INT_ACK;
6908 WREG32(DC_HPD5_INT_CONTROL, tmp);
6909 }
6910 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6911 tmp = RREG32(DC_HPD5_INT_CONTROL);
6912 tmp |= DC_HPDx_INT_ACK;
6913 WREG32(DC_HPD6_INT_CONTROL, tmp);
6914 }
6915}
6916
6917/**
6918 * cik_irq_disable - disable interrupts
6919 *
6920 * @rdev: radeon_device pointer
6921 *
6922 * Disable interrupts on the hw (CIK).
6923 */
6924static void cik_irq_disable(struct radeon_device *rdev)
6925{
6926 cik_disable_interrupts(rdev);
6927 /* Wait and acknowledge irq */
6928 mdelay(1);
6929 cik_irq_ack(rdev);
6930 cik_disable_interrupt_state(rdev);
6931}
6932
6933/**
6934 * cik_irq_disable - disable interrupts for suspend
6935 *
6936 * @rdev: radeon_device pointer
6937 *
6938 * Disable interrupts and stop the RLC (CIK).
6939 * Used for suspend.
6940 */
6941static void cik_irq_suspend(struct radeon_device *rdev)
6942{
6943 cik_irq_disable(rdev);
6944 cik_rlc_stop(rdev);
6945}
6946
6947/**
6948 * cik_irq_fini - tear down interrupt support
6949 *
6950 * @rdev: radeon_device pointer
6951 *
6952 * Disable interrupts on the hw and free the IH ring
6953 * buffer (CIK).
6954 * Used for driver unload.
6955 */
6956static void cik_irq_fini(struct radeon_device *rdev)
6957{
6958 cik_irq_suspend(rdev);
6959 r600_ih_ring_fini(rdev);
6960}
6961
6962/**
6963 * cik_get_ih_wptr - get the IH ring buffer wptr
6964 *
6965 * @rdev: radeon_device pointer
6966 *
6967 * Get the IH ring buffer wptr from either the register
6968 * or the writeback memory buffer (CIK). Also check for
6969 * ring buffer overflow and deal with it.
6970 * Used by cik_irq_process().
6971 * Returns the value of the wptr.
6972 */
6973static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6974{
6975 u32 wptr, tmp;
6976
6977 if (rdev->wb.enabled)
6978 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6979 else
6980 wptr = RREG32(IH_RB_WPTR);
6981
6982 if (wptr & RB_OVERFLOW) {
6983 /* When a ring buffer overflow happen start parsing interrupt
6984 * from the last not overwritten vector (wptr + 16). Hopefully
6985 * this should allow us to catchup.
6986 */
6987 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6988 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6989 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6990 tmp = RREG32(IH_RB_CNTL);
6991 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6992 WREG32(IH_RB_CNTL, tmp);
6993 }
6994 return (wptr & rdev->ih.ptr_mask);
6995}
6996
6997/* CIK IV Ring
6998 * Each IV ring entry is 128 bits:
6999 * [7:0] - interrupt source id
7000 * [31:8] - reserved
7001 * [59:32] - interrupt source data
7002 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7003 * [71:64] - RINGID
7004 * CP:
7005 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7006 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7007 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7008 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7009 * PIPE_ID - ME0 0=3D
7010 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7011 * SDMA:
7012 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7013 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7014 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7015 * [79:72] - VMID
7016 * [95:80] - PASID
7017 * [127:96] - reserved
7018 */
7019/**
7020 * cik_irq_process - interrupt handler
7021 *
7022 * @rdev: radeon_device pointer
7023 *
7024 * Interrupt hander (CIK). Walk the IH ring,
7025 * ack interrupts and schedule work to handle
7026 * interrupt events.
7027 * Returns irq process return code.
7028 */
7029int cik_irq_process(struct radeon_device *rdev)
7030{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7031 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7032 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7033 u32 wptr;
7034 u32 rptr;
7035 u32 src_id, src_data, ring_id;
7036 u8 me_id, pipe_id, queue_id;
7037 u32 ring_index;
7038 bool queue_hotplug = false;
7039 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7040 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7041 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7042
7043 if (!rdev->ih.enabled || rdev->shutdown)
7044 return IRQ_NONE;
7045
7046 wptr = cik_get_ih_wptr(rdev);
7047
7048restart_ih:
7049 /* is somebody else already processing irqs? */
7050 if (atomic_xchg(&rdev->ih.lock, 1))
7051 return IRQ_NONE;
7052
7053 rptr = rdev->ih.rptr;
7054 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7055
7056 /* Order reading of wptr vs. reading of IH ring data */
7057 rmb();
7058
7059 /* display interrupts */
7060 cik_irq_ack(rdev);
7061
7062 while (rptr != wptr) {
7063 /* wptr/rptr are in bytes! */
7064 ring_index = rptr / 4;
7065 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7066 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7067 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7068
7069 switch (src_id) {
7070 case 1: /* D1 vblank/vline */
7071 switch (src_data) {
7072 case 0: /* D1 vblank */
7073 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7074 if (rdev->irq.crtc_vblank_int[0]) {
7075 drm_handle_vblank(rdev->ddev, 0);
7076 rdev->pm.vblank_sync = true;
7077 wake_up(&rdev->irq.vblank_queue);
7078 }
7079 if (atomic_read(&rdev->irq.pflip[0]))
7080 radeon_crtc_handle_flip(rdev, 0);
7081 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7082 DRM_DEBUG("IH: D1 vblank\n");
7083 }
7084 break;
7085 case 1: /* D1 vline */
7086 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7087 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7088 DRM_DEBUG("IH: D1 vline\n");
7089 }
7090 break;
7091 default:
7092 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7093 break;
7094 }
7095 break;
7096 case 2: /* D2 vblank/vline */
7097 switch (src_data) {
7098 case 0: /* D2 vblank */
7099 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7100 if (rdev->irq.crtc_vblank_int[1]) {
7101 drm_handle_vblank(rdev->ddev, 1);
7102 rdev->pm.vblank_sync = true;
7103 wake_up(&rdev->irq.vblank_queue);
7104 }
7105 if (atomic_read(&rdev->irq.pflip[1]))
7106 radeon_crtc_handle_flip(rdev, 1);
7107 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7108 DRM_DEBUG("IH: D2 vblank\n");
7109 }
7110 break;
7111 case 1: /* D2 vline */
7112 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7113 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7114 DRM_DEBUG("IH: D2 vline\n");
7115 }
7116 break;
7117 default:
7118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7119 break;
7120 }
7121 break;
7122 case 3: /* D3 vblank/vline */
7123 switch (src_data) {
7124 case 0: /* D3 vblank */
7125 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7126 if (rdev->irq.crtc_vblank_int[2]) {
7127 drm_handle_vblank(rdev->ddev, 2);
7128 rdev->pm.vblank_sync = true;
7129 wake_up(&rdev->irq.vblank_queue);
7130 }
7131 if (atomic_read(&rdev->irq.pflip[2]))
7132 radeon_crtc_handle_flip(rdev, 2);
7133 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7134 DRM_DEBUG("IH: D3 vblank\n");
7135 }
7136 break;
7137 case 1: /* D3 vline */
7138 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7139 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7140 DRM_DEBUG("IH: D3 vline\n");
7141 }
7142 break;
7143 default:
7144 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7145 break;
7146 }
7147 break;
7148 case 4: /* D4 vblank/vline */
7149 switch (src_data) {
7150 case 0: /* D4 vblank */
7151 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7152 if (rdev->irq.crtc_vblank_int[3]) {
7153 drm_handle_vblank(rdev->ddev, 3);
7154 rdev->pm.vblank_sync = true;
7155 wake_up(&rdev->irq.vblank_queue);
7156 }
7157 if (atomic_read(&rdev->irq.pflip[3]))
7158 radeon_crtc_handle_flip(rdev, 3);
7159 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7160 DRM_DEBUG("IH: D4 vblank\n");
7161 }
7162 break;
7163 case 1: /* D4 vline */
7164 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7165 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7166 DRM_DEBUG("IH: D4 vline\n");
7167 }
7168 break;
7169 default:
7170 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7171 break;
7172 }
7173 break;
7174 case 5: /* D5 vblank/vline */
7175 switch (src_data) {
7176 case 0: /* D5 vblank */
7177 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7178 if (rdev->irq.crtc_vblank_int[4]) {
7179 drm_handle_vblank(rdev->ddev, 4);
7180 rdev->pm.vblank_sync = true;
7181 wake_up(&rdev->irq.vblank_queue);
7182 }
7183 if (atomic_read(&rdev->irq.pflip[4]))
7184 radeon_crtc_handle_flip(rdev, 4);
7185 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7186 DRM_DEBUG("IH: D5 vblank\n");
7187 }
7188 break;
7189 case 1: /* D5 vline */
7190 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7191 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7192 DRM_DEBUG("IH: D5 vline\n");
7193 }
7194 break;
7195 default:
7196 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7197 break;
7198 }
7199 break;
7200 case 6: /* D6 vblank/vline */
7201 switch (src_data) {
7202 case 0: /* D6 vblank */
7203 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7204 if (rdev->irq.crtc_vblank_int[5]) {
7205 drm_handle_vblank(rdev->ddev, 5);
7206 rdev->pm.vblank_sync = true;
7207 wake_up(&rdev->irq.vblank_queue);
7208 }
7209 if (atomic_read(&rdev->irq.pflip[5]))
7210 radeon_crtc_handle_flip(rdev, 5);
7211 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7212 DRM_DEBUG("IH: D6 vblank\n");
7213 }
7214 break;
7215 case 1: /* D6 vline */
7216 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7217 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7218 DRM_DEBUG("IH: D6 vline\n");
7219 }
7220 break;
7221 default:
7222 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7223 break;
7224 }
7225 break;
7226 case 42: /* HPD hotplug */
7227 switch (src_data) {
7228 case 0:
7229 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7230 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7231 queue_hotplug = true;
7232 DRM_DEBUG("IH: HPD1\n");
7233 }
7234 break;
7235 case 1:
7236 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7237 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7238 queue_hotplug = true;
7239 DRM_DEBUG("IH: HPD2\n");
7240 }
7241 break;
7242 case 2:
7243 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7244 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7245 queue_hotplug = true;
7246 DRM_DEBUG("IH: HPD3\n");
7247 }
7248 break;
7249 case 3:
7250 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7251 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7252 queue_hotplug = true;
7253 DRM_DEBUG("IH: HPD4\n");
7254 }
7255 break;
7256 case 4:
7257 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7258 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7259 queue_hotplug = true;
7260 DRM_DEBUG("IH: HPD5\n");
7261 }
7262 break;
7263 case 5:
7264 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7265 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7266 queue_hotplug = true;
7267 DRM_DEBUG("IH: HPD6\n");
7268 }
7269 break;
7270 default:
7271 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7272 break;
7273 }
7274 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7275 case 124: /* UVD */
7276 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7277 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7278 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7279 case 146:
7280 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7281 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7282 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7283 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7284 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7285 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7286 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7287 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7288 status);
7289 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7290 /* reset addr and status */
7291 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7292 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7293 case 176: /* GFX RB CP_INT */
7294 case 177: /* GFX IB CP_INT */
7295 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7296 break;
7297 case 181: /* CP EOP event */
7298 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7299 /* XXX check the bitfield order! */
7300 me_id = (ring_id & 0x60) >> 5;
7301 pipe_id = (ring_id & 0x18) >> 3;
7302 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7303 switch (me_id) {
7304 case 0:
7305 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7306 break;
7307 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7308 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7309 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7310 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7311 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7312 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7313 break;
7314 }
7315 break;
7316 case 184: /* CP Privileged reg access */
7317 DRM_ERROR("Illegal register access in command stream\n");
7318 /* XXX check the bitfield order! */
7319 me_id = (ring_id & 0x60) >> 5;
7320 pipe_id = (ring_id & 0x18) >> 3;
7321 queue_id = (ring_id & 0x7) >> 0;
7322 switch (me_id) {
7323 case 0:
7324 /* This results in a full GPU reset, but all we need to do is soft
7325 * reset the CP for gfx
7326 */
7327 queue_reset = true;
7328 break;
7329 case 1:
7330 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7331 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7332 break;
7333 case 2:
7334 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7335 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7336 break;
7337 }
7338 break;
7339 case 185: /* CP Privileged inst */
7340 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7341 /* XXX check the bitfield order! */
7342 me_id = (ring_id & 0x60) >> 5;
7343 pipe_id = (ring_id & 0x18) >> 3;
7344 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7345 switch (me_id) {
7346 case 0:
7347 /* This results in a full GPU reset, but all we need to do is soft
7348 * reset the CP for gfx
7349 */
7350 queue_reset = true;
7351 break;
7352 case 1:
7353 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7354 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7355 break;
7356 case 2:
7357 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7358 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7359 break;
7360 }
7361 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7362 case 224: /* SDMA trap event */
7363 /* XXX check the bitfield order! */
7364 me_id = (ring_id & 0x3) >> 0;
7365 queue_id = (ring_id & 0xc) >> 2;
7366 DRM_DEBUG("IH: SDMA trap\n");
7367 switch (me_id) {
7368 case 0:
7369 switch (queue_id) {
7370 case 0:
7371 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7372 break;
7373 case 1:
7374 /* XXX compute */
7375 break;
7376 case 2:
7377 /* XXX compute */
7378 break;
7379 }
7380 break;
7381 case 1:
7382 switch (queue_id) {
7383 case 0:
7384 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7385 break;
7386 case 1:
7387 /* XXX compute */
7388 break;
7389 case 2:
7390 /* XXX compute */
7391 break;
7392 }
7393 break;
7394 }
7395 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7396 case 230: /* thermal low to high */
7397 DRM_DEBUG("IH: thermal low to high\n");
7398 rdev->pm.dpm.thermal.high_to_low = false;
7399 queue_thermal = true;
7400 break;
7401 case 231: /* thermal high to low */
7402 DRM_DEBUG("IH: thermal high to low\n");
7403 rdev->pm.dpm.thermal.high_to_low = true;
7404 queue_thermal = true;
7405 break;
7406 case 233: /* GUI IDLE */
7407 DRM_DEBUG("IH: GUI idle\n");
7408 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7409 case 241: /* SDMA Privileged inst */
7410 case 247: /* SDMA Privileged inst */
7411 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7412 /* XXX check the bitfield order! */
7413 me_id = (ring_id & 0x3) >> 0;
7414 queue_id = (ring_id & 0xc) >> 2;
7415 switch (me_id) {
7416 case 0:
7417 switch (queue_id) {
7418 case 0:
7419 queue_reset = true;
7420 break;
7421 case 1:
7422 /* XXX compute */
7423 queue_reset = true;
7424 break;
7425 case 2:
7426 /* XXX compute */
7427 queue_reset = true;
7428 break;
7429 }
7430 break;
7431 case 1:
7432 switch (queue_id) {
7433 case 0:
7434 queue_reset = true;
7435 break;
7436 case 1:
7437 /* XXX compute */
7438 queue_reset = true;
7439 break;
7440 case 2:
7441 /* XXX compute */
7442 queue_reset = true;
7443 break;
7444 }
7445 break;
7446 }
7447 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7448 default:
7449 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7450 break;
7451 }
7452
7453 /* wptr/rptr are in bytes! */
7454 rptr += 16;
7455 rptr &= rdev->ih.ptr_mask;
7456 }
7457 if (queue_hotplug)
7458 schedule_work(&rdev->hotplug_work);
7459 if (queue_reset)
7460 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7461 if (queue_thermal)
7462 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7463 rdev->ih.rptr = rptr;
7464 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7465 atomic_set(&rdev->ih.lock, 0);
7466
7467 /* make sure wptr hasn't changed while processing */
7468 wptr = cik_get_ih_wptr(rdev);
7469 if (wptr != rptr)
7470 goto restart_ih;
7471
7472 return IRQ_HANDLED;
7473}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7474
7475/*
7476 * startup/shutdown callbacks
7477 */
7478/**
7479 * cik_startup - program the asic to a functional state
7480 *
7481 * @rdev: radeon_device pointer
7482 *
7483 * Programs the asic to a functional state (CIK).
7484 * Called by cik_init() and cik_resume().
7485 * Returns 0 for success, error for failure.
7486 */
7487static int cik_startup(struct radeon_device *rdev)
7488{
7489 struct radeon_ring *ring;
7490 int r;
7491
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7492 /* enable pcie gen2/3 link */
7493 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7494 /* enable aspm */
7495 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7496
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7497 /* scratch needs to be initialized before MC */
7498 r = r600_vram_scratch_init(rdev);
7499 if (r)
7500 return r;
7501
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7502 cik_mc_program(rdev);
7503
Alex Deucher01ac8792013-12-18 19:11:27 -0500[diff] [blame^]7504 if (!(rdev->flags & RADEON_IS_IGP)) {
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7505 r = ci_mc_load_microcode(rdev);
7506 if (r) {
7507 DRM_ERROR("Failed to load MC firmware!\n");
7508 return r;
7509 }
7510 }
7511
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7512 r = cik_pcie_gart_enable(rdev);
7513 if (r)
7514 return r;
7515 cik_gpu_init(rdev);
7516
7517 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7518 if (rdev->flags & RADEON_IS_IGP) {
7519 if (rdev->family == CHIP_KAVERI) {
7520 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7521 rdev->rlc.reg_list_size =
7522 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7523 } else {
7524 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7525 rdev->rlc.reg_list_size =
7526 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7527 }
7528 }
7529 rdev->rlc.cs_data = ci_cs_data;
7530 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7531 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7532 if (r) {
7533 DRM_ERROR("Failed to init rlc BOs!\n");
7534 return r;
7535 }
7536
7537 /* allocate wb buffer */
7538 r = radeon_wb_init(rdev);
7539 if (r)
7540 return r;
7541
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7542 /* allocate mec buffers */
7543 r = cik_mec_init(rdev);
7544 if (r) {
7545 DRM_ERROR("Failed to init MEC BOs!\n");
7546 return r;
7547 }
7548
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7549 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7550 if (r) {
7551 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7552 return r;
7553 }
7554
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7555 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7556 if (r) {
7557 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7558 return r;
7559 }
7560
7561 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7562 if (r) {
7563 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7564 return r;
7565 }
7566
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7567 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7568 if (r) {
7569 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7570 return r;
7571 }
7572
7573 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7574 if (r) {
7575 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7576 return r;
7577 }
7578
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7579 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7580 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7581 r = uvd_v4_2_resume(rdev);
7582 if (!r) {
7583 r = radeon_fence_driver_start_ring(rdev,
7584 R600_RING_TYPE_UVD_INDEX);
7585 if (r)
7586 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7587 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7588 }
7589 if (r)
7590 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7591
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7592 /* Enable IRQ */
7593 if (!rdev->irq.installed) {
7594 r = radeon_irq_kms_init(rdev);
7595 if (r)
7596 return r;
7597 }
7598
7599 r = cik_irq_init(rdev);
7600 if (r) {
7601 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7602 radeon_irq_kms_fini(rdev);
7603 return r;
7604 }
7605 cik_irq_set(rdev);
7606
7607 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7608 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7609 CP_RB0_RPTR, CP_RB0_WPTR,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7610 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7611 if (r)
7612 return r;
7613
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7614 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7615 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7616 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7617 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7618 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7619 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7620 if (r)
7621 return r;
7622 ring->me = 1; /* first MEC */
7623 ring->pipe = 0; /* first pipe */
7624 ring->queue = 0; /* first queue */
7625 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7626
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7627 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7628 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7629 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7630 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7631 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7632 if (r)
7633 return r;
7634 /* dGPU only have 1 MEC */
7635 ring->me = 1; /* first MEC */
7636 ring->pipe = 0; /* first pipe */
7637 ring->queue = 1; /* second queue */
7638 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7639
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7640 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7641 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7642 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7643 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7644 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7645 if (r)
7646 return r;
7647
7648 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7649 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7650 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7651 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7652 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7653 if (r)
7654 return r;
7655
7656 r = cik_cp_resume(rdev);
7657 if (r)
7658 return r;
7659
7660 r = cik_sdma_resume(rdev);
7661 if (r)
7662 return r;
7663
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7664 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7665 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7666 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7667 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7668 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7669 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7670 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7671 if (r)
7672 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7673 }
7674
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7675 r = radeon_ib_pool_init(rdev);
7676 if (r) {
7677 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7678 return r;
7679 }
7680
7681 r = radeon_vm_manager_init(rdev);
7682 if (r) {
7683 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7684 return r;
7685 }
7686
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7687 r = dce6_audio_init(rdev);
7688 if (r)
7689 return r;
7690
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7691 return 0;
7692}
7693
7694/**
7695 * cik_resume - resume the asic to a functional state
7696 *
7697 * @rdev: radeon_device pointer
7698 *
7699 * Programs the asic to a functional state (CIK).
7700 * Called at resume.
7701 * Returns 0 for success, error for failure.
7702 */
7703int cik_resume(struct radeon_device *rdev)
7704{
7705 int r;
7706
7707 /* post card */
7708 atom_asic_init(rdev->mode_info.atom_context);
7709
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7710 /* init golden registers */
7711 cik_init_golden_registers(rdev);
7712
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7713 rdev->accel_working = true;
7714 r = cik_startup(rdev);
7715 if (r) {
7716 DRM_ERROR("cik startup failed on resume\n");
7717 rdev->accel_working = false;
7718 return r;
7719 }
7720
7721 return r;
7722
7723}
7724
7725/**
7726 * cik_suspend - suspend the asic
7727 *
7728 * @rdev: radeon_device pointer
7729 *
7730 * Bring the chip into a state suitable for suspend (CIK).
7731 * Called at suspend.
7732 * Returns 0 for success.
7733 */
7734int cik_suspend(struct radeon_device *rdev)
7735{
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7736 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7737 radeon_vm_manager_fini(rdev);
7738 cik_cp_enable(rdev, false);
7739 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7740 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7741 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7742 cik_fini_pg(rdev);
7743 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7744 cik_irq_suspend(rdev);
7745 radeon_wb_disable(rdev);
7746 cik_pcie_gart_disable(rdev);
7747 return 0;
7748}
7749
7750/* Plan is to move initialization in that function and use
7751 * helper function so that radeon_device_init pretty much
7752 * do nothing more than calling asic specific function. This
7753 * should also allow to remove a bunch of callback function
7754 * like vram_info.
7755 */
7756/**
7757 * cik_init - asic specific driver and hw init
7758 *
7759 * @rdev: radeon_device pointer
7760 *
7761 * Setup asic specific driver variables and program the hw
7762 * to a functional state (CIK).
7763 * Called at driver startup.
7764 * Returns 0 for success, errors for failure.
7765 */
7766int cik_init(struct radeon_device *rdev)
7767{
7768 struct radeon_ring *ring;
7769 int r;
7770
7771 /* Read BIOS */
7772 if (!radeon_get_bios(rdev)) {
7773 if (ASIC_IS_AVIVO(rdev))
7774 return -EINVAL;
7775 }
7776 /* Must be an ATOMBIOS */
7777 if (!rdev->is_atom_bios) {
7778 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7779 return -EINVAL;
7780 }
7781 r = radeon_atombios_init(rdev);
7782 if (r)
7783 return r;
7784
7785 /* Post card if necessary */
7786 if (!radeon_card_posted(rdev)) {
7787 if (!rdev->bios) {
7788 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7789 return -EINVAL;
7790 }
7791 DRM_INFO("GPU not posted. posting now...\n");
7792 atom_asic_init(rdev->mode_info.atom_context);
7793 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7794 /* init golden registers */
7795 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7796 /* Initialize scratch registers */
7797 cik_scratch_init(rdev);
7798 /* Initialize surface registers */
7799 radeon_surface_init(rdev);
7800 /* Initialize clocks */
7801 radeon_get_clock_info(rdev->ddev);
7802
7803 /* Fence driver */
7804 r = radeon_fence_driver_init(rdev);
7805 if (r)
7806 return r;
7807
7808 /* initialize memory controller */
7809 r = cik_mc_init(rdev);
7810 if (r)
7811 return r;
7812 /* Memory manager */
7813 r = radeon_bo_init(rdev);
7814 if (r)
7815 return r;
7816
Alex Deucher01ac8792013-12-18 19:11:27 -0500[diff] [blame^]7817 if (rdev->flags & RADEON_IS_IGP) {
7818 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7819 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7820 r = cik_init_microcode(rdev);
7821 if (r) {
7822 DRM_ERROR("Failed to load firmware!\n");
7823 return r;
7824 }
7825 }
7826 } else {
7827 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7828 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7829 !rdev->mc_fw) {
7830 r = cik_init_microcode(rdev);
7831 if (r) {
7832 DRM_ERROR("Failed to load firmware!\n");
7833 return r;
7834 }
7835 }
7836 }
7837
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7838 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7839 ring->ring_obj = NULL;
7840 r600_ring_init(rdev, ring, 1024 * 1024);
7841
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7842 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7843 ring->ring_obj = NULL;
7844 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]7845 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7846 if (r)
7847 return r;
7848
7849 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7850 ring->ring_obj = NULL;
7851 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]7852 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7853 if (r)
7854 return r;
7855
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7856 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7857 ring->ring_obj = NULL;
7858 r600_ring_init(rdev, ring, 256 * 1024);
7859
7860 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7861 ring->ring_obj = NULL;
7862 r600_ring_init(rdev, ring, 256 * 1024);
7863
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7864 r = radeon_uvd_init(rdev);
7865 if (!r) {
7866 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7867 ring->ring_obj = NULL;
7868 r600_ring_init(rdev, ring, 4096);
7869 }
7870
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7871 rdev->ih.ring_obj = NULL;
7872 r600_ih_ring_init(rdev, 64 * 1024);
7873
7874 r = r600_pcie_gart_init(rdev);
7875 if (r)
7876 return r;
7877
7878 rdev->accel_working = true;
7879 r = cik_startup(rdev);
7880 if (r) {
7881 dev_err(rdev->dev, "disabling GPU acceleration\n");
7882 cik_cp_fini(rdev);
7883 cik_sdma_fini(rdev);
7884 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7885 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7886 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7887 radeon_wb_fini(rdev);
7888 radeon_ib_pool_fini(rdev);
7889 radeon_vm_manager_fini(rdev);
7890 radeon_irq_kms_fini(rdev);
7891 cik_pcie_gart_fini(rdev);
7892 rdev->accel_working = false;
7893 }
7894
7895 /* Don't start up if the MC ucode is missing.
7896 * The default clocks and voltages before the MC ucode
7897 * is loaded are not suffient for advanced operations.
7898 */
7899 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7900 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7901 return -EINVAL;
7902 }
7903
7904 return 0;
7905}
7906
7907/**
7908 * cik_fini - asic specific driver and hw fini
7909 *
7910 * @rdev: radeon_device pointer
7911 *
7912 * Tear down the asic specific driver variables and program the hw
7913 * to an idle state (CIK).
7914 * Called at driver unload.
7915 */
7916void cik_fini(struct radeon_device *rdev)
7917{
7918 cik_cp_fini(rdev);
7919 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7920 cik_fini_pg(rdev);
7921 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7922 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7923 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7924 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7925 radeon_wb_fini(rdev);
7926 radeon_vm_manager_fini(rdev);
7927 radeon_ib_pool_fini(rdev);
7928 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7929 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7930 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7931 cik_pcie_gart_fini(rdev);
7932 r600_vram_scratch_fini(rdev);
7933 radeon_gem_fini(rdev);
7934 radeon_fence_driver_fini(rdev);
7935 radeon_bo_fini(rdev);
7936 radeon_atombios_fini(rdev);
7937 kfree(rdev->bios);
7938 rdev->bios = NULL;
7939}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7940
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7941void dce8_program_fmt(struct drm_encoder *encoder)
7942{
7943 struct drm_device *dev = encoder->dev;
7944 struct radeon_device *rdev = dev->dev_private;
7945 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7946 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7947 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7948 int bpc = 0;
7949 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7950 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7951
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7952 if (connector) {
7953 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7954 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7955 dither = radeon_connector->dither;
7956 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7957
7958 /* LVDS/eDP FMT is set up by atom */
7959 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7960 return;
7961
7962 /* not needed for analog */
7963 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7964 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7965 return;
7966
7967 if (bpc == 0)
7968 return;
7969
7970 switch (bpc) {
7971 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7972 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7973 /* XXX sort out optimal dither settings */
7974 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7975 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7976 else
7977 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7978 break;
7979 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7980 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7981 /* XXX sort out optimal dither settings */
7982 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7983 FMT_RGB_RANDOM_ENABLE |
7984 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7985 else
7986 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7987 break;
7988 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7989 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7990 /* XXX sort out optimal dither settings */
7991 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7992 FMT_RGB_RANDOM_ENABLE |
7993 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7994 else
7995 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7996 break;
7997 default:
7998 /* not needed */
7999 break;
8000 }
8001
8002 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8003}
8004
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8005/* display watermark setup */
8006/**
8007 * dce8_line_buffer_adjust - Set up the line buffer
8008 *
8009 * @rdev: radeon_device pointer
8010 * @radeon_crtc: the selected display controller
8011 * @mode: the current display mode on the selected display
8012 * controller
8013 *
8014 * Setup up the line buffer allocation for
8015 * the selected display controller (CIK).
8016 * Returns the line buffer size in pixels.
8017 */
8018static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8019 struct radeon_crtc *radeon_crtc,
8020 struct drm_display_mode *mode)
8021{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8022 u32 tmp, buffer_alloc, i;
8023 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8024 /*
8025 * Line Buffer Setup
8026 * There are 6 line buffers, one for each display controllers.
8027 * There are 3 partitions per LB. Select the number of partitions
8028 * to enable based on the display width. For display widths larger
8029 * than 4096, you need use to use 2 display controllers and combine
8030 * them using the stereo blender.
8031 */
8032 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8033 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8034 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8035 buffer_alloc = 2;
8036 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8037 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8038 buffer_alloc = 2;
8039 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8040 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8041 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8042 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8043 DRM_DEBUG_KMS("Mode too big for LB!\n");
8044 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8045 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8046 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8047 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8048 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8049 buffer_alloc = 0;
8050 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8051
8052 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8053 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8054
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8055 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8056 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8057 for (i = 0; i < rdev->usec_timeout; i++) {
8058 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8059 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8060 break;
8061 udelay(1);
8062 }
8063
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8064 if (radeon_crtc->base.enabled && mode) {
8065 switch (tmp) {
8066 case 0:
8067 default:
8068 return 4096 * 2;
8069 case 1:
8070 return 1920 * 2;
8071 case 2:
8072 return 2560 * 2;
8073 }
8074 }
8075
8076 /* controller not enabled, so no lb used */
8077 return 0;
8078}
8079
8080/**
8081 * cik_get_number_of_dram_channels - get the number of dram channels
8082 *
8083 * @rdev: radeon_device pointer
8084 *
8085 * Look up the number of video ram channels (CIK).
8086 * Used for display watermark bandwidth calculations
8087 * Returns the number of dram channels
8088 */
8089static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8090{
8091 u32 tmp = RREG32(MC_SHARED_CHMAP);
8092
8093 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8094 case 0:
8095 default:
8096 return 1;
8097 case 1:
8098 return 2;
8099 case 2:
8100 return 4;
8101 case 3:
8102 return 8;
8103 case 4:
8104 return 3;
8105 case 5:
8106 return 6;
8107 case 6:
8108 return 10;
8109 case 7:
8110 return 12;
8111 case 8:
8112 return 16;
8113 }
8114}
8115
8116struct dce8_wm_params {
8117 u32 dram_channels; /* number of dram channels */
8118 u32 yclk; /* bandwidth per dram data pin in kHz */
8119 u32 sclk; /* engine clock in kHz */
8120 u32 disp_clk; /* display clock in kHz */
8121 u32 src_width; /* viewport width */
8122 u32 active_time; /* active display time in ns */
8123 u32 blank_time; /* blank time in ns */
8124 bool interlaced; /* mode is interlaced */
8125 fixed20_12 vsc; /* vertical scale ratio */
8126 u32 num_heads; /* number of active crtcs */
8127 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8128 u32 lb_size; /* line buffer allocated to pipe */
8129 u32 vtaps; /* vertical scaler taps */
8130};
8131
8132/**
8133 * dce8_dram_bandwidth - get the dram bandwidth
8134 *
8135 * @wm: watermark calculation data
8136 *
8137 * Calculate the raw dram bandwidth (CIK).
8138 * Used for display watermark bandwidth calculations
8139 * Returns the dram bandwidth in MBytes/s
8140 */
8141static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8142{
8143 /* Calculate raw DRAM Bandwidth */
8144 fixed20_12 dram_efficiency; /* 0.7 */
8145 fixed20_12 yclk, dram_channels, bandwidth;
8146 fixed20_12 a;
8147
8148 a.full = dfixed_const(1000);
8149 yclk.full = dfixed_const(wm->yclk);
8150 yclk.full = dfixed_div(yclk, a);
8151 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8152 a.full = dfixed_const(10);
8153 dram_efficiency.full = dfixed_const(7);
8154 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8155 bandwidth.full = dfixed_mul(dram_channels, yclk);
8156 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8157
8158 return dfixed_trunc(bandwidth);
8159}
8160
8161/**
8162 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8163 *
8164 * @wm: watermark calculation data
8165 *
8166 * Calculate the dram bandwidth used for display (CIK).
8167 * Used for display watermark bandwidth calculations
8168 * Returns the dram bandwidth for display in MBytes/s
8169 */
8170static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8171{
8172 /* Calculate DRAM Bandwidth and the part allocated to display. */
8173 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8174 fixed20_12 yclk, dram_channels, bandwidth;
8175 fixed20_12 a;
8176
8177 a.full = dfixed_const(1000);
8178 yclk.full = dfixed_const(wm->yclk);
8179 yclk.full = dfixed_div(yclk, a);
8180 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8181 a.full = dfixed_const(10);
8182 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8183 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8184 bandwidth.full = dfixed_mul(dram_channels, yclk);
8185 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8186
8187 return dfixed_trunc(bandwidth);
8188}
8189
8190/**
8191 * dce8_data_return_bandwidth - get the data return bandwidth
8192 *
8193 * @wm: watermark calculation data
8194 *
8195 * Calculate the data return bandwidth used for display (CIK).
8196 * Used for display watermark bandwidth calculations
8197 * Returns the data return bandwidth in MBytes/s
8198 */
8199static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8200{
8201 /* Calculate the display Data return Bandwidth */
8202 fixed20_12 return_efficiency; /* 0.8 */
8203 fixed20_12 sclk, bandwidth;
8204 fixed20_12 a;
8205
8206 a.full = dfixed_const(1000);
8207 sclk.full = dfixed_const(wm->sclk);
8208 sclk.full = dfixed_div(sclk, a);
8209 a.full = dfixed_const(10);
8210 return_efficiency.full = dfixed_const(8);
8211 return_efficiency.full = dfixed_div(return_efficiency, a);
8212 a.full = dfixed_const(32);
8213 bandwidth.full = dfixed_mul(a, sclk);
8214 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8215
8216 return dfixed_trunc(bandwidth);
8217}
8218
8219/**
8220 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8221 *
8222 * @wm: watermark calculation data
8223 *
8224 * Calculate the dmif bandwidth used for display (CIK).
8225 * Used for display watermark bandwidth calculations
8226 * Returns the dmif bandwidth in MBytes/s
8227 */
8228static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8229{
8230 /* Calculate the DMIF Request Bandwidth */
8231 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8232 fixed20_12 disp_clk, bandwidth;
8233 fixed20_12 a, b;
8234
8235 a.full = dfixed_const(1000);
8236 disp_clk.full = dfixed_const(wm->disp_clk);
8237 disp_clk.full = dfixed_div(disp_clk, a);
8238 a.full = dfixed_const(32);
8239 b.full = dfixed_mul(a, disp_clk);
8240
8241 a.full = dfixed_const(10);
8242 disp_clk_request_efficiency.full = dfixed_const(8);
8243 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8244
8245 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8246
8247 return dfixed_trunc(bandwidth);
8248}
8249
8250/**
8251 * dce8_available_bandwidth - get the min available bandwidth
8252 *
8253 * @wm: watermark calculation data
8254 *
8255 * Calculate the min available bandwidth used for display (CIK).
8256 * Used for display watermark bandwidth calculations
8257 * Returns the min available bandwidth in MBytes/s
8258 */
8259static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8260{
8261 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8262 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8263 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8264 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8265
8266 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8267}
8268
8269/**
8270 * dce8_average_bandwidth - get the average available bandwidth
8271 *
8272 * @wm: watermark calculation data
8273 *
8274 * Calculate the average available bandwidth used for display (CIK).
8275 * Used for display watermark bandwidth calculations
8276 * Returns the average available bandwidth in MBytes/s
8277 */
8278static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8279{
8280 /* Calculate the display mode Average Bandwidth
8281 * DisplayMode should contain the source and destination dimensions,
8282 * timing, etc.
8283 */
8284 fixed20_12 bpp;
8285 fixed20_12 line_time;
8286 fixed20_12 src_width;
8287 fixed20_12 bandwidth;
8288 fixed20_12 a;
8289
8290 a.full = dfixed_const(1000);
8291 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8292 line_time.full = dfixed_div(line_time, a);
8293 bpp.full = dfixed_const(wm->bytes_per_pixel);
8294 src_width.full = dfixed_const(wm->src_width);
8295 bandwidth.full = dfixed_mul(src_width, bpp);
8296 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8297 bandwidth.full = dfixed_div(bandwidth, line_time);
8298
8299 return dfixed_trunc(bandwidth);
8300}
8301
8302/**
8303 * dce8_latency_watermark - get the latency watermark
8304 *
8305 * @wm: watermark calculation data
8306 *
8307 * Calculate the latency watermark (CIK).
8308 * Used for display watermark bandwidth calculations
8309 * Returns the latency watermark in ns
8310 */
8311static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8312{
8313 /* First calculate the latency in ns */
8314 u32 mc_latency = 2000; /* 2000 ns. */
8315 u32 available_bandwidth = dce8_available_bandwidth(wm);
8316 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8317 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8318 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8319 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8320 (wm->num_heads * cursor_line_pair_return_time);
8321 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8322 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8323 u32 tmp, dmif_size = 12288;
8324 fixed20_12 a, b, c;
8325
8326 if (wm->num_heads == 0)
8327 return 0;
8328
8329 a.full = dfixed_const(2);
8330 b.full = dfixed_const(1);
8331 if ((wm->vsc.full > a.full) ||
8332 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8333 (wm->vtaps >= 5) ||
8334 ((wm->vsc.full >= a.full) && wm->interlaced))
8335 max_src_lines_per_dst_line = 4;
8336 else
8337 max_src_lines_per_dst_line = 2;
8338
8339 a.full = dfixed_const(available_bandwidth);
8340 b.full = dfixed_const(wm->num_heads);
8341 a.full = dfixed_div(a, b);
8342
8343 b.full = dfixed_const(mc_latency + 512);
8344 c.full = dfixed_const(wm->disp_clk);
8345 b.full = dfixed_div(b, c);
8346
8347 c.full = dfixed_const(dmif_size);
8348 b.full = dfixed_div(c, b);
8349
8350 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8351
8352 b.full = dfixed_const(1000);
8353 c.full = dfixed_const(wm->disp_clk);
8354 b.full = dfixed_div(c, b);
8355 c.full = dfixed_const(wm->bytes_per_pixel);
8356 b.full = dfixed_mul(b, c);
8357
8358 lb_fill_bw = min(tmp, dfixed_trunc(b));
8359
8360 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8361 b.full = dfixed_const(1000);
8362 c.full = dfixed_const(lb_fill_bw);
8363 b.full = dfixed_div(c, b);
8364 a.full = dfixed_div(a, b);
8365 line_fill_time = dfixed_trunc(a);
8366
8367 if (line_fill_time < wm->active_time)
8368 return latency;
8369 else
8370 return latency + (line_fill_time - wm->active_time);
8371
8372}
8373
8374/**
8375 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8376 * average and available dram bandwidth
8377 *
8378 * @wm: watermark calculation data
8379 *
8380 * Check if the display average bandwidth fits in the display
8381 * dram bandwidth (CIK).
8382 * Used for display watermark bandwidth calculations
8383 * Returns true if the display fits, false if not.
8384 */
8385static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8386{
8387 if (dce8_average_bandwidth(wm) <=
8388 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8389 return true;
8390 else
8391 return false;
8392}
8393
8394/**
8395 * dce8_average_bandwidth_vs_available_bandwidth - check
8396 * average and available bandwidth
8397 *
8398 * @wm: watermark calculation data
8399 *
8400 * Check if the display average bandwidth fits in the display
8401 * available bandwidth (CIK).
8402 * Used for display watermark bandwidth calculations
8403 * Returns true if the display fits, false if not.
8404 */
8405static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8406{
8407 if (dce8_average_bandwidth(wm) <=
8408 (dce8_available_bandwidth(wm) / wm->num_heads))
8409 return true;
8410 else
8411 return false;
8412}
8413
8414/**
8415 * dce8_check_latency_hiding - check latency hiding
8416 *
8417 * @wm: watermark calculation data
8418 *
8419 * Check latency hiding (CIK).
8420 * Used for display watermark bandwidth calculations
8421 * Returns true if the display fits, false if not.
8422 */
8423static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8424{
8425 u32 lb_partitions = wm->lb_size / wm->src_width;
8426 u32 line_time = wm->active_time + wm->blank_time;
8427 u32 latency_tolerant_lines;
8428 u32 latency_hiding;
8429 fixed20_12 a;
8430
8431 a.full = dfixed_const(1);
8432 if (wm->vsc.full > a.full)
8433 latency_tolerant_lines = 1;
8434 else {
8435 if (lb_partitions <= (wm->vtaps + 1))
8436 latency_tolerant_lines = 1;
8437 else
8438 latency_tolerant_lines = 2;
8439 }
8440
8441 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8442
8443 if (dce8_latency_watermark(wm) <= latency_hiding)
8444 return true;
8445 else
8446 return false;
8447}
8448
8449/**
8450 * dce8_program_watermarks - program display watermarks
8451 *
8452 * @rdev: radeon_device pointer
8453 * @radeon_crtc: the selected display controller
8454 * @lb_size: line buffer size
8455 * @num_heads: number of display controllers in use
8456 *
8457 * Calculate and program the display watermarks for the
8458 * selected display controller (CIK).
8459 */
8460static void dce8_program_watermarks(struct radeon_device *rdev,
8461 struct radeon_crtc *radeon_crtc,
8462 u32 lb_size, u32 num_heads)
8463{
8464 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8465 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8466 u32 pixel_period;
8467 u32 line_time = 0;
8468 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8469 u32 tmp, wm_mask;
8470
8471 if (radeon_crtc->base.enabled && num_heads && mode) {
8472 pixel_period = 1000000 / (u32)mode->clock;
8473 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8474
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8475 /* watermark for high clocks */
8476 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8477 rdev->pm.dpm_enabled) {
8478 wm_high.yclk =
8479 radeon_dpm_get_mclk(rdev, false) * 10;
8480 wm_high.sclk =
8481 radeon_dpm_get_sclk(rdev, false) * 10;
8482 } else {
8483 wm_high.yclk = rdev->pm.current_mclk * 10;
8484 wm_high.sclk = rdev->pm.current_sclk * 10;
8485 }
8486
8487 wm_high.disp_clk = mode->clock;
8488 wm_high.src_width = mode->crtc_hdisplay;
8489 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8490 wm_high.blank_time = line_time - wm_high.active_time;
8491 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8492 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8493 wm_high.interlaced = true;
8494 wm_high.vsc = radeon_crtc->vsc;
8495 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8496 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8497 wm_high.vtaps = 2;
8498 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8499 wm_high.lb_size = lb_size;
8500 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8501 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8502
8503 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8504 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8505
8506 /* possibly force display priority to high */
8507 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8508 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8509 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8510 !dce8_check_latency_hiding(&wm_high) ||
8511 (rdev->disp_priority == 2)) {
8512 DRM_DEBUG_KMS("force priority to high\n");
8513 }
8514
8515 /* watermark for low clocks */
8516 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8517 rdev->pm.dpm_enabled) {
8518 wm_low.yclk =
8519 radeon_dpm_get_mclk(rdev, true) * 10;
8520 wm_low.sclk =
8521 radeon_dpm_get_sclk(rdev, true) * 10;
8522 } else {
8523 wm_low.yclk = rdev->pm.current_mclk * 10;
8524 wm_low.sclk = rdev->pm.current_sclk * 10;
8525 }
8526
8527 wm_low.disp_clk = mode->clock;
8528 wm_low.src_width = mode->crtc_hdisplay;
8529 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8530 wm_low.blank_time = line_time - wm_low.active_time;
8531 wm_low.interlaced = false;
8532 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8533 wm_low.interlaced = true;
8534 wm_low.vsc = radeon_crtc->vsc;
8535 wm_low.vtaps = 1;
8536 if (radeon_crtc->rmx_type != RMX_OFF)
8537 wm_low.vtaps = 2;
8538 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8539 wm_low.lb_size = lb_size;
8540 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8541 wm_low.num_heads = num_heads;
8542
8543 /* set for low clocks */
8544 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8545
8546 /* possibly force display priority to high */
8547 /* should really do this at mode validation time... */
8548 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8549 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8550 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8551 (rdev->disp_priority == 2)) {
8552 DRM_DEBUG_KMS("force priority to high\n");
8553 }
8554 }
8555
8556 /* select wm A */
8557 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8558 tmp = wm_mask;
8559 tmp &= ~LATENCY_WATERMARK_MASK(3);
8560 tmp |= LATENCY_WATERMARK_MASK(1);
8561 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8562 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8563 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8564 LATENCY_HIGH_WATERMARK(line_time)));
8565 /* select wm B */
8566 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8567 tmp &= ~LATENCY_WATERMARK_MASK(3);
8568 tmp |= LATENCY_WATERMARK_MASK(2);
8569 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8570 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8571 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8572 LATENCY_HIGH_WATERMARK(line_time)));
8573 /* restore original selection */
8574 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8575
8576 /* save values for DPM */
8577 radeon_crtc->line_time = line_time;
8578 radeon_crtc->wm_high = latency_watermark_a;
8579 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8580}
8581
8582/**
8583 * dce8_bandwidth_update - program display watermarks
8584 *
8585 * @rdev: radeon_device pointer
8586 *
8587 * Calculate and program the display watermarks and line
8588 * buffer allocation (CIK).
8589 */
8590void dce8_bandwidth_update(struct radeon_device *rdev)
8591{
8592 struct drm_display_mode *mode = NULL;
8593 u32 num_heads = 0, lb_size;
8594 int i;
8595
8596 radeon_update_display_priority(rdev);
8597
8598 for (i = 0; i < rdev->num_crtc; i++) {
8599 if (rdev->mode_info.crtcs[i]->base.enabled)
8600 num_heads++;
8601 }
8602 for (i = 0; i < rdev->num_crtc; i++) {
8603 mode = &rdev->mode_info.crtcs[i]->base.mode;
8604 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8605 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8606 }
8607}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8608
8609/**
8610 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8611 *
8612 * @rdev: radeon_device pointer
8613 *
8614 * Fetches a GPU clock counter snapshot (SI).
8615 * Returns the 64 bit clock counter snapshot.
8616 */
8617uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8618{
8619 uint64_t clock;
8620
8621 mutex_lock(&rdev->gpu_clock_mutex);
8622 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8623 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8624 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8625 mutex_unlock(&rdev->gpu_clock_mutex);
8626 return clock;
8627}
8628
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8629static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8630 u32 cntl_reg, u32 status_reg)
8631{
8632 int r, i;
8633 struct atom_clock_dividers dividers;
8634 uint32_t tmp;
8635
8636 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8637 clock, false, &dividers);
8638 if (r)
8639 return r;
8640
8641 tmp = RREG32_SMC(cntl_reg);
8642 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8643 tmp |= dividers.post_divider;
8644 WREG32_SMC(cntl_reg, tmp);
8645
8646 for (i = 0; i < 100; i++) {
8647 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8648 break;
8649 mdelay(10);
8650 }
8651 if (i == 100)
8652 return -ETIMEDOUT;
8653
8654 return 0;
8655}
8656
8657int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8658{
8659 int r = 0;
8660
8661 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8662 if (r)
8663 return r;
8664
8665 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8666 return r;
8667}
8668
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8669static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8670{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8671 struct pci_dev *root = rdev->pdev->bus->self;
8672 int bridge_pos, gpu_pos;
8673 u32 speed_cntl, mask, current_data_rate;
8674 int ret, i;
8675 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8676
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8677 if (radeon_pcie_gen2 == 0)
8678 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8679
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8680 if (rdev->flags & RADEON_IS_IGP)
8681 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8682
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8683 if (!(rdev->flags & RADEON_IS_PCIE))
8684 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8685
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8686 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8687 if (ret != 0)
8688 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8689
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8690 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8691 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8692
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8693 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8694 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8695 LC_CURRENT_DATA_RATE_SHIFT;
8696 if (mask & DRM_PCIE_SPEED_80) {
8697 if (current_data_rate == 2) {
8698 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8699 return;
8700 }
8701 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8702 } else if (mask & DRM_PCIE_SPEED_50) {
8703 if (current_data_rate == 1) {
8704 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8705 return;
8706 }
8707 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8708 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8709
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8710 bridge_pos = pci_pcie_cap(root);
8711 if (!bridge_pos)
8712 return;
8713
8714 gpu_pos = pci_pcie_cap(rdev->pdev);
8715 if (!gpu_pos)
8716 return;
8717
8718 if (mask & DRM_PCIE_SPEED_80) {
8719 /* re-try equalization if gen3 is not already enabled */
8720 if (current_data_rate != 2) {
8721 u16 bridge_cfg, gpu_cfg;
8722 u16 bridge_cfg2, gpu_cfg2;
8723 u32 max_lw, current_lw, tmp;
8724
8725 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8726 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8727
8728 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8729 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8730
8731 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8732 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8733
8734 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8735 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8736 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8737
8738 if (current_lw < max_lw) {
8739 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8740 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8741 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8742 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8743 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8744 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8745 }
8746 }
8747
8748 for (i = 0; i < 10; i++) {
8749 /* check status */
8750 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8751 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8752 break;
8753
8754 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8755 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8756
8757 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8758 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8759
8760 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8761 tmp |= LC_SET_QUIESCE;
8762 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8763
8764 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8765 tmp |= LC_REDO_EQ;
8766 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8767
8768 mdelay(100);
8769
8770 /* linkctl */
8771 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8772 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8773 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8774 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8775
8776 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8777 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8778 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8779 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8780
8781 /* linkctl2 */
8782 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8783 tmp16 &= ~((1 << 4) | (7 << 9));
8784 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8785 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8786
8787 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8788 tmp16 &= ~((1 << 4) | (7 << 9));
8789 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8790 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8791
8792 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8793 tmp &= ~LC_SET_QUIESCE;
8794 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8795 }
8796 }
8797 }
8798
8799 /* set the link speed */
8800 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8801 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8802 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8803
8804 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8805 tmp16 &= ~0xf;
8806 if (mask & DRM_PCIE_SPEED_80)
8807 tmp16 |= 3; /* gen3 */
8808 else if (mask & DRM_PCIE_SPEED_50)
8809 tmp16 |= 2; /* gen2 */
8810 else
8811 tmp16 |= 1; /* gen1 */
8812 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8813
8814 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8815 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8816 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8817
8818 for (i = 0; i < rdev->usec_timeout; i++) {
8819 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8820 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8821 break;
8822 udelay(1);
8823 }
8824}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]8825
8826static void cik_program_aspm(struct radeon_device *rdev)
8827{
8828 u32 data, orig;
8829 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8830 bool disable_clkreq = false;
8831
8832 if (radeon_aspm == 0)
8833 return;
8834
8835 /* XXX double check IGPs */
8836 if (rdev->flags & RADEON_IS_IGP)
8837 return;
8838
8839 if (!(rdev->flags & RADEON_IS_PCIE))
8840 return;
8841
8842 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8843 data &= ~LC_XMIT_N_FTS_MASK;
8844 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8845 if (orig != data)
8846 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8847
8848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8849 data |= LC_GO_TO_RECOVERY;
8850 if (orig != data)
8851 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8852
8853 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8854 data |= P_IGNORE_EDB_ERR;
8855 if (orig != data)
8856 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8857
8858 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8859 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8860 data |= LC_PMI_TO_L1_DIS;
8861 if (!disable_l0s)
8862 data |= LC_L0S_INACTIVITY(7);
8863
8864 if (!disable_l1) {
8865 data |= LC_L1_INACTIVITY(7);
8866 data &= ~LC_PMI_TO_L1_DIS;
8867 if (orig != data)
8868 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8869
8870 if (!disable_plloff_in_l1) {
8871 bool clk_req_support;
8872
8873 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8874 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8875 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8876 if (orig != data)
8877 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8878
8879 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8880 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8881 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8882 if (orig != data)
8883 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8884
8885 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8886 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8887 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8888 if (orig != data)
8889 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8890
8891 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8892 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8893 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8894 if (orig != data)
8895 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8896
8897 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8898 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8899 data |= LC_DYN_LANES_PWR_STATE(3);
8900 if (orig != data)
8901 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8902
8903 if (!disable_clkreq) {
8904 struct pci_dev *root = rdev->pdev->bus->self;
8905 u32 lnkcap;
8906
8907 clk_req_support = false;
8908 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8909 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8910 clk_req_support = true;
8911 } else {
8912 clk_req_support = false;
8913 }
8914
8915 if (clk_req_support) {
8916 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8917 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8918 if (orig != data)
8919 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8920
8921 orig = data = RREG32_SMC(THM_CLK_CNTL);
8922 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8923 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8924 if (orig != data)
8925 WREG32_SMC(THM_CLK_CNTL, data);
8926
8927 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8928 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8929 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8930 if (orig != data)
8931 WREG32_SMC(MISC_CLK_CTRL, data);
8932
8933 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8934 data &= ~BCLK_AS_XCLK;
8935 if (orig != data)
8936 WREG32_SMC(CG_CLKPIN_CNTL, data);
8937
8938 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8939 data &= ~FORCE_BIF_REFCLK_EN;
8940 if (orig != data)
8941 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8942
8943 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8944 data &= ~MPLL_CLKOUT_SEL_MASK;
8945 data |= MPLL_CLKOUT_SEL(4);
8946 if (orig != data)
8947 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8948 }
8949 }
8950 } else {
8951 if (orig != data)
8952 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8953 }
8954
8955 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8956 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8957 if (orig != data)
8958 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8959
8960 if (!disable_l0s) {
8961 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8962 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8963 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8964 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8965 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8966 data &= ~LC_L0S_INACTIVITY_MASK;
8967 if (orig != data)
8968 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8969 }
8970 }
8971 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8972}