Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / 439a1cfffe2c1a06e5a6394ccd5d18a8e89b15d3 / . / drivers / gpu / drm / radeon / cik.c
blob: e950fabd7f5e474ab3e60371f669e31805077785 [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]64
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]73extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]78static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]80static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]87
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]124
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]130 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]131 u32 r;
132
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]151}
152
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1532 default:
1533 break;
1534 }
1535}
1536
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1563 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1564 *
1565 * Returns the value in the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1566 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1567 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1569{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1572 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1582 * @index: doorbell index
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1586 * requested doorbell index (CIK).
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1587 */
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1589{
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1592 } else {
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1594 }
1595}
1596
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
1700static int ci_mc_load_microcode(struct radeon_device *rdev)
1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1802 break;
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1904 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1925 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1932 }
1933
1934out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1952 }
1953 return err;
1954}
1955
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1994 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1995
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2432 }
2433 } else if (num_pipe_configs == 4) {
2434 if (num_rbs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2437 case 0:
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2442 break;
2443 case 1:
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2448 break;
2449 case 2:
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2454 break;
2455 case 3:
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2460 break;
2461 case 4:
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2466 break;
2467 case 5:
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2470 break;
2471 case 6:
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2476 break;
2477 case 7:
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2482 break;
2483 case 8:
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2486 break;
2487 case 9:
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2490 break;
2491 case 10:
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2496 break;
2497 case 11:
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2502 break;
2503 case 12:
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2508 break;
2509 case 13:
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2512 break;
2513 case 14:
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2518 break;
2519 case 16:
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2524 break;
2525 case 17:
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 break;
2531 case 27:
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2534 break;
2535 case 28:
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540 break;
2541 case 29:
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546 break;
2547 case 30:
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552 break;
2553 default:
2554 gb_tile_moden = 0;
2555 break;
2556 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2559 }
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2563 case 0:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2568 break;
2569 case 1:
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2574 break;
2575 case 2:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2580 break;
2581 case 3:
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2586 break;
2587 case 4:
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2592 break;
2593 case 5:
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2596 break;
2597 case 6:
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2602 break;
2603 case 7:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2608 break;
2609 case 8:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2612 break;
2613 case 9:
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2616 break;
2617 case 10:
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2622 break;
2623 case 11:
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2628 break;
2629 case 12:
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2634 break;
2635 case 13:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2638 break;
2639 case 14:
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2644 break;
2645 case 16:
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2650 break;
2651 case 17:
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2656 break;
2657 case 27:
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2660 break;
2661 case 28:
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2666 break;
2667 case 29:
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 break;
2673 case 30:
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 break;
2679 default:
2680 gb_tile_moden = 0;
2681 break;
2682 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2685 }
2686 }
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2689 case 0:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2694 break;
2695 case 1:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 2:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 3:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 4:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 5:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 6:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 8:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 break;
2737 case 9:
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2742 break;
2743 case 10:
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 break;
2749 case 11:
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2754 break;
2755 case 12:
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 break;
2761 case 13:
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2766 break;
2767 case 14:
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2772 break;
2773 default:
2774 gb_tile_moden = 0;
2775 break;
2776 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2779 }
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2783 case 0:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2788 break;
2789 case 1:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2794 break;
2795 case 2:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2800 break;
2801 case 3:
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2806 break;
2807 case 4:
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2812 break;
2813 case 5:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2816 break;
2817 case 6:
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2822 break;
2823 case 7:
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2828 break;
2829 case 8:
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2831 break;
2832 case 9:
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2835 break;
2836 case 10:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 11:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 12:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 13:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2857 break;
2858 case 14:
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2863 break;
2864 case 16:
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2869 break;
2870 case 17:
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2875 break;
2876 case 27:
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2879 break;
2880 case 28:
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 break;
2886 case 29:
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2891 break;
2892 case 30:
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 break;
2898 default:
2899 gb_tile_moden = 0;
2900 break;
2901 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2904 }
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2907 case 0:
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2912 break;
2913 case 1:
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2918 break;
2919 case 2:
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2924 break;
2925 case 3:
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2930 break;
2931 case 4:
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2936 break;
2937 case 5:
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942 break;
2943 case 6:
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2948 break;
2949 case 8:
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2954 break;
2955 case 9:
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 break;
2961 case 10:
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2966 break;
2967 case 11:
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 break;
2973 case 12:
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2978 break;
2979 case 13:
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 break;
2985 case 14:
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2990 break;
2991 default:
2992 gb_tile_moden = 0;
2993 break;
2994 }
Michel Dänzer32f79a82013-11-18 18:26:00 +0900[diff] [blame]2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2997 }
2998 } else
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3000}
3001
3002/**
3003 * cik_select_se_sh - select which SE, SH to address
3004 *
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3008 *
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3012 */
3013static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3015{
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3017
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3024 else
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3027}
3028
3029/**
3030 * cik_create_bitmask - create a bitmask
3031 *
3032 * @bit_width: length of the mask
3033 *
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3036 */
3037static u32 cik_create_bitmask(u32 bit_width)
3038{
3039 u32 i, mask = 0;
3040
3041 for (i = 0; i < bit_width; i++) {
3042 mask <<= 1;
3043 mask |= 1;
3044 }
3045 return mask;
3046}
3047
3048/**
3049 * cik_select_se_sh - select which SE, SH to address
3050 *
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3055 *
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3058 */
3059static u32 cik_get_rb_disabled(struct radeon_device *rdev,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3060 u32 max_rb_num_per_se,
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3061 u32 sh_per_se)
3062{
3063 u32 data, mask;
3064
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3066 if (data & 1)
3067 data &= BACKEND_DISABLE_MASK;
3068 else
3069 data = 0;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3071
3072 data >>= BACKEND_DISABLE_SHIFT;
3073
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3074 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3075
3076 return data & mask;
3077}
3078
3079/**
3080 * cik_setup_rb - setup the RBs on the asic
3081 *
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3086 *
3087 * Configures per-SE/SH RB registers (CIK).
3088 */
3089static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3091 u32 max_rb_num_per_se)
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3092{
3093 int i, j;
3094 u32 data, mask;
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3097
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3101 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3104 else
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3106 }
3107 }
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3109
3110 mask = 1;
Marek Olšák9fadb352013-12-22 02:18:00 +0100[diff] [blame]3111 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3114 mask <<= 1;
3115 }
3116
Marek Olšák439a1cf2013-12-22 02:18:01 +0100[diff] [blame^]3117 rdev->config.cik.backend_enable_mask = enabled_rbs;
3118
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3119 for (i = 0; i < se_num; i++) {
3120 cik_select_se_sh(rdev, i, 0xffffffff);
3121 data = 0;
3122 for (j = 0; j < sh_per_se; j++) {
3123 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame]3124 case 0:
3125 if (j == 0)
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3127 else
3128 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3129 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3130 case 1:
3131 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3132 break;
3133 case 2:
3134 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3135 break;
3136 case 3:
3137 default:
3138 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3139 break;
3140 }
3141 enabled_rbs >>= 2;
3142 }
3143 WREG32(PA_SC_RASTER_CONFIG, data);
3144 }
3145 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3146}
3147
3148/**
3149 * cik_gpu_init - setup the 3D engine
3150 *
3151 * @rdev: radeon_device pointer
3152 *
3153 * Configures the 3D engine and tiling configuration
3154 * registers so that the 3D engine is usable.
3155 */
3156static void cik_gpu_init(struct radeon_device *rdev)
3157{
3158 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3159 u32 mc_shared_chmap, mc_arb_ramcfg;
3160 u32 hdp_host_path_cntl;
3161 u32 tmp;
3162 int i, j;
3163
3164 switch (rdev->family) {
3165 case CHIP_BONAIRE:
3166 rdev->config.cik.max_shader_engines = 2;
3167 rdev->config.cik.max_tile_pipes = 4;
3168 rdev->config.cik.max_cu_per_sh = 7;
3169 rdev->config.cik.max_sh_per_se = 1;
3170 rdev->config.cik.max_backends_per_se = 2;
3171 rdev->config.cik.max_texture_channel_caches = 4;
3172 rdev->config.cik.max_gprs = 256;
3173 rdev->config.cik.max_gs_threads = 32;
3174 rdev->config.cik.max_hw_contexts = 8;
3175
3176 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3177 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3178 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3181 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3182 case CHIP_HAWAII:
3183 rdev->config.cik.max_shader_engines = 4;
3184 rdev->config.cik.max_tile_pipes = 16;
3185 rdev->config.cik.max_cu_per_sh = 11;
3186 rdev->config.cik.max_sh_per_se = 1;
3187 rdev->config.cik.max_backends_per_se = 4;
3188 rdev->config.cik.max_texture_channel_caches = 16;
3189 rdev->config.cik.max_gprs = 256;
3190 rdev->config.cik.max_gs_threads = 32;
3191 rdev->config.cik.max_hw_contexts = 8;
3192
3193 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3194 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3195 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3198 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3199 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3200 rdev->config.cik.max_shader_engines = 1;
3201 rdev->config.cik.max_tile_pipes = 4;
3202 if ((rdev->pdev->device == 0x1304) ||
3203 (rdev->pdev->device == 0x1305) ||
3204 (rdev->pdev->device == 0x130C) ||
3205 (rdev->pdev->device == 0x130F) ||
3206 (rdev->pdev->device == 0x1310) ||
3207 (rdev->pdev->device == 0x1311) ||
3208 (rdev->pdev->device == 0x131C)) {
3209 rdev->config.cik.max_cu_per_sh = 8;
3210 rdev->config.cik.max_backends_per_se = 2;
3211 } else if ((rdev->pdev->device == 0x1309) ||
3212 (rdev->pdev->device == 0x130A) ||
3213 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3214 (rdev->pdev->device == 0x1313) ||
3215 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3216 rdev->config.cik.max_cu_per_sh = 6;
3217 rdev->config.cik.max_backends_per_se = 2;
3218 } else if ((rdev->pdev->device == 0x1306) ||
3219 (rdev->pdev->device == 0x1307) ||
3220 (rdev->pdev->device == 0x130B) ||
3221 (rdev->pdev->device == 0x130E) ||
3222 (rdev->pdev->device == 0x1315) ||
3223 (rdev->pdev->device == 0x131B)) {
3224 rdev->config.cik.max_cu_per_sh = 4;
3225 rdev->config.cik.max_backends_per_se = 1;
3226 } else {
3227 rdev->config.cik.max_cu_per_sh = 3;
3228 rdev->config.cik.max_backends_per_se = 1;
3229 }
3230 rdev->config.cik.max_sh_per_se = 1;
3231 rdev->config.cik.max_texture_channel_caches = 4;
3232 rdev->config.cik.max_gprs = 256;
3233 rdev->config.cik.max_gs_threads = 16;
3234 rdev->config.cik.max_hw_contexts = 8;
3235
3236 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3237 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3238 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3240 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3241 break;
3242 case CHIP_KABINI:
3243 default:
3244 rdev->config.cik.max_shader_engines = 1;
3245 rdev->config.cik.max_tile_pipes = 2;
3246 rdev->config.cik.max_cu_per_sh = 2;
3247 rdev->config.cik.max_sh_per_se = 1;
3248 rdev->config.cik.max_backends_per_se = 1;
3249 rdev->config.cik.max_texture_channel_caches = 2;
3250 rdev->config.cik.max_gprs = 256;
3251 rdev->config.cik.max_gs_threads = 16;
3252 rdev->config.cik.max_hw_contexts = 8;
3253
3254 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3255 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3256 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3257 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3258 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3259 break;
3260 }
3261
3262 /* Initialize HDP */
3263 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3264 WREG32((0x2c14 + j), 0x00000000);
3265 WREG32((0x2c18 + j), 0x00000000);
3266 WREG32((0x2c1c + j), 0x00000000);
3267 WREG32((0x2c20 + j), 0x00000000);
3268 WREG32((0x2c24 + j), 0x00000000);
3269 }
3270
3271 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3272
3273 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3274
3275 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3276 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3277
3278 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3279 rdev->config.cik.mem_max_burst_length_bytes = 256;
3280 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3281 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3282 if (rdev->config.cik.mem_row_size_in_kb > 4)
3283 rdev->config.cik.mem_row_size_in_kb = 4;
3284 /* XXX use MC settings? */
3285 rdev->config.cik.shader_engine_tile_size = 32;
3286 rdev->config.cik.num_gpus = 1;
3287 rdev->config.cik.multi_gpu_tile_size = 64;
3288
3289 /* fix up row size */
3290 gb_addr_config &= ~ROW_SIZE_MASK;
3291 switch (rdev->config.cik.mem_row_size_in_kb) {
3292 case 1:
3293 default:
3294 gb_addr_config |= ROW_SIZE(0);
3295 break;
3296 case 2:
3297 gb_addr_config |= ROW_SIZE(1);
3298 break;
3299 case 4:
3300 gb_addr_config |= ROW_SIZE(2);
3301 break;
3302 }
3303
3304 /* setup tiling info dword. gb_addr_config is not adequate since it does
3305 * not have bank info, so create a custom tiling dword.
3306 * bits 3:0 num_pipes
3307 * bits 7:4 num_banks
3308 * bits 11:8 group_size
3309 * bits 15:12 row_size
3310 */
3311 rdev->config.cik.tile_config = 0;
3312 switch (rdev->config.cik.num_tile_pipes) {
3313 case 1:
3314 rdev->config.cik.tile_config |= (0 << 0);
3315 break;
3316 case 2:
3317 rdev->config.cik.tile_config |= (1 << 0);
3318 break;
3319 case 4:
3320 rdev->config.cik.tile_config |= (2 << 0);
3321 break;
3322 case 8:
3323 default:
3324 /* XXX what about 12? */
3325 rdev->config.cik.tile_config |= (3 << 0);
3326 break;
3327 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3328 rdev->config.cik.tile_config |=
3329 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3332 rdev->config.cik.tile_config |=
3333 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3334
3335 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3336 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3337 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3338 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3339 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3340 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3341 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3342 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3343
3344 cik_tiling_mode_table_init(rdev);
3345
3346 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3347 rdev->config.cik.max_sh_per_se,
3348 rdev->config.cik.max_backends_per_se);
3349
3350 /* set HW defaults for 3D engine */
3351 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3352
3353 WREG32(SX_DEBUG_1, 0x20);
3354
3355 WREG32(TA_CNTL_AUX, 0x00010000);
3356
3357 tmp = RREG32(SPI_CONFIG_CNTL);
3358 tmp |= 0x03000000;
3359 WREG32(SPI_CONFIG_CNTL, tmp);
3360
3361 WREG32(SQ_CONFIG, 1);
3362
3363 WREG32(DB_DEBUG, 0);
3364
3365 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3366 tmp |= 0x00000400;
3367 WREG32(DB_DEBUG2, tmp);
3368
3369 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3370 tmp |= 0x00020200;
3371 WREG32(DB_DEBUG3, tmp);
3372
3373 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3374 tmp |= 0x00018208;
3375 WREG32(CB_HW_CONTROL, tmp);
3376
3377 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3378
3379 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3380 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3381 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3382 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3383
3384 WREG32(VGT_NUM_INSTANCES, 1);
3385
3386 WREG32(CP_PERFMON_CNTL, 0);
3387
3388 WREG32(SQ_CONFIG, 0);
3389
3390 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3391 FORCE_EOV_MAX_REZ_CNT(255)));
3392
3393 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3394 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3395
3396 WREG32(VGT_GS_VERTEX_REUSE, 16);
3397 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3398
3399 tmp = RREG32(HDP_MISC_CNTL);
3400 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3401 WREG32(HDP_MISC_CNTL, tmp);
3402
3403 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3404 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3405
3406 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3407 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3408
3409 udelay(50);
3410}
3411
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3412/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3413 * GPU scratch registers helpers function.
3414 */
3415/**
3416 * cik_scratch_init - setup driver info for CP scratch regs
3417 *
3418 * @rdev: radeon_device pointer
3419 *
3420 * Set up the number and offset of the CP scratch registers.
3421 * NOTE: use of CP scratch registers is a legacy inferface and
3422 * is not used by default on newer asics (r6xx+). On newer asics,
3423 * memory buffers are used for fences rather than scratch regs.
3424 */
3425static void cik_scratch_init(struct radeon_device *rdev)
3426{
3427 int i;
3428
3429 rdev->scratch.num_reg = 7;
3430 rdev->scratch.reg_base = SCRATCH_REG0;
3431 for (i = 0; i < rdev->scratch.num_reg; i++) {
3432 rdev->scratch.free[i] = true;
3433 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3434 }
3435}
3436
3437/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3438 * cik_ring_test - basic gfx ring test
3439 *
3440 * @rdev: radeon_device pointer
3441 * @ring: radeon_ring structure holding ring information
3442 *
3443 * Allocate a scratch register and write to it using the gfx ring (CIK).
3444 * Provides a basic gfx ring test to verify that the ring is working.
3445 * Used by cik_cp_gfx_resume();
3446 * Returns 0 on success, error on failure.
3447 */
3448int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3449{
3450 uint32_t scratch;
3451 uint32_t tmp = 0;
3452 unsigned i;
3453 int r;
3454
3455 r = radeon_scratch_get(rdev, &scratch);
3456 if (r) {
3457 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3458 return r;
3459 }
3460 WREG32(scratch, 0xCAFEDEAD);
3461 r = radeon_ring_lock(rdev, ring, 3);
3462 if (r) {
3463 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3464 radeon_scratch_free(rdev, scratch);
3465 return r;
3466 }
3467 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3468 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3469 radeon_ring_write(ring, 0xDEADBEEF);
3470 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3471
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3472 for (i = 0; i < rdev->usec_timeout; i++) {
3473 tmp = RREG32(scratch);
3474 if (tmp == 0xDEADBEEF)
3475 break;
3476 DRM_UDELAY(1);
3477 }
3478 if (i < rdev->usec_timeout) {
3479 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3480 } else {
3481 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3482 ring->idx, scratch, tmp);
3483 r = -EINVAL;
3484 }
3485 radeon_scratch_free(rdev, scratch);
3486 return r;
3487}
3488
3489/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3490 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3491 *
3492 * @rdev: radeon_device pointer
3493 * @fence: radeon fence object
3494 *
3495 * Emits a fence sequnce number on the gfx ring and flushes
3496 * GPU caches.
3497 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3498void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3499 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3500{
3501 struct radeon_ring *ring = &rdev->ring[fence->ring];
3502 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3503
3504 /* EVENT_WRITE_EOP - flush caches, send int */
3505 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3506 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3507 EOP_TC_ACTION_EN |
3508 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3509 EVENT_INDEX(5)));
3510 radeon_ring_write(ring, addr & 0xfffffffc);
3511 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3512 radeon_ring_write(ring, fence->seq);
3513 radeon_ring_write(ring, 0);
3514 /* HDP flush */
3515 /* We should be using the new WAIT_REG_MEM special op packet here
3516 * but it causes the CP to hang
3517 */
3518 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3519 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3520 WRITE_DATA_DST_SEL(0)));
3521 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3522 radeon_ring_write(ring, 0);
3523 radeon_ring_write(ring, 0);
3524}
3525
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3526/**
3527 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3528 *
3529 * @rdev: radeon_device pointer
3530 * @fence: radeon fence object
3531 *
3532 * Emits a fence sequnce number on the compute ring and flushes
3533 * GPU caches.
3534 */
3535void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3536 struct radeon_fence *fence)
3537{
3538 struct radeon_ring *ring = &rdev->ring[fence->ring];
3539 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3540
3541 /* RELEASE_MEM - flush caches, send int */
3542 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3543 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3544 EOP_TC_ACTION_EN |
3545 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3546 EVENT_INDEX(5)));
3547 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3548 radeon_ring_write(ring, addr & 0xfffffffc);
3549 radeon_ring_write(ring, upper_32_bits(addr));
3550 radeon_ring_write(ring, fence->seq);
3551 radeon_ring_write(ring, 0);
3552 /* HDP flush */
3553 /* We should be using the new WAIT_REG_MEM special op packet here
3554 * but it causes the CP to hang
3555 */
3556 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3557 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3558 WRITE_DATA_DST_SEL(0)));
3559 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3560 radeon_ring_write(ring, 0);
3561 radeon_ring_write(ring, 0);
3562}
3563
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3564bool cik_semaphore_ring_emit(struct radeon_device *rdev,
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3565 struct radeon_ring *ring,
3566 struct radeon_semaphore *semaphore,
3567 bool emit_wait)
3568{
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3569/* TODO: figure out why semaphore cause lockups */
3570#if 0
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3571 uint64_t addr = semaphore->gpu_addr;
3572 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3573
3574 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3575 radeon_ring_write(ring, addr & 0xffffffff);
3576 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3577
3578 return true;
Christian König99b4f252013-11-12 12:58:06 +0100[diff] [blame]3579#else
3580 return false;
3581#endif
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3582}
3583
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3584/**
3585 * cik_copy_cpdma - copy pages using the CP DMA engine
3586 *
3587 * @rdev: radeon_device pointer
3588 * @src_offset: src GPU address
3589 * @dst_offset: dst GPU address
3590 * @num_gpu_pages: number of GPU pages to xfer
3591 * @fence: radeon fence object
3592 *
3593 * Copy GPU paging using the CP DMA engine (CIK+).
3594 * Used by the radeon ttm implementation to move pages if
3595 * registered as the asic copy callback.
3596 */
3597int cik_copy_cpdma(struct radeon_device *rdev,
3598 uint64_t src_offset, uint64_t dst_offset,
3599 unsigned num_gpu_pages,
3600 struct radeon_fence **fence)
3601{
3602 struct radeon_semaphore *sem = NULL;
3603 int ring_index = rdev->asic->copy.blit_ring_index;
3604 struct radeon_ring *ring = &rdev->ring[ring_index];
3605 u32 size_in_bytes, cur_size_in_bytes, control;
3606 int i, num_loops;
3607 int r = 0;
3608
3609 r = radeon_semaphore_create(rdev, &sem);
3610 if (r) {
3611 DRM_ERROR("radeon: moving bo (%d).\n", r);
3612 return r;
3613 }
3614
3615 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3616 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3617 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3618 if (r) {
3619 DRM_ERROR("radeon: moving bo (%d).\n", r);
3620 radeon_semaphore_free(rdev, &sem, NULL);
3621 return r;
3622 }
3623
Christian König1654b812013-11-12 12:58:05 +0100[diff] [blame]3624 radeon_semaphore_sync_to(sem, *fence);
3625 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3626
3627 for (i = 0; i < num_loops; i++) {
3628 cur_size_in_bytes = size_in_bytes;
3629 if (cur_size_in_bytes > 0x1fffff)
3630 cur_size_in_bytes = 0x1fffff;
3631 size_in_bytes -= cur_size_in_bytes;
3632 control = 0;
3633 if (size_in_bytes == 0)
3634 control |= PACKET3_DMA_DATA_CP_SYNC;
3635 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3636 radeon_ring_write(ring, control);
3637 radeon_ring_write(ring, lower_32_bits(src_offset));
3638 radeon_ring_write(ring, upper_32_bits(src_offset));
3639 radeon_ring_write(ring, lower_32_bits(dst_offset));
3640 radeon_ring_write(ring, upper_32_bits(dst_offset));
3641 radeon_ring_write(ring, cur_size_in_bytes);
3642 src_offset += cur_size_in_bytes;
3643 dst_offset += cur_size_in_bytes;
3644 }
3645
3646 r = radeon_fence_emit(rdev, fence, ring->idx);
3647 if (r) {
3648 radeon_ring_unlock_undo(rdev, ring);
3649 return r;
3650 }
3651
3652 radeon_ring_unlock_commit(rdev, ring);
3653 radeon_semaphore_free(rdev, &sem, *fence);
3654
3655 return r;
3656}
3657
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3658/*
3659 * IB stuff
3660 */
3661/**
3662 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3663 *
3664 * @rdev: radeon_device pointer
3665 * @ib: radeon indirect buffer object
3666 *
3667 * Emits an DE (drawing engine) or CE (constant engine) IB
3668 * on the gfx ring. IBs are usually generated by userspace
3669 * acceleration drivers and submitted to the kernel for
3670 * sheduling on the ring. This function schedules the IB
3671 * on the gfx ring for execution by the GPU.
3672 */
3673void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3674{
3675 struct radeon_ring *ring = &rdev->ring[ib->ring];
3676 u32 header, control = INDIRECT_BUFFER_VALID;
3677
3678 if (ib->is_const_ib) {
3679 /* set switch buffer packet before const IB */
3680 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3681 radeon_ring_write(ring, 0);
3682
3683 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3684 } else {
3685 u32 next_rptr;
3686 if (ring->rptr_save_reg) {
3687 next_rptr = ring->wptr + 3 + 4;
3688 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3689 radeon_ring_write(ring, ((ring->rptr_save_reg -
3690 PACKET3_SET_UCONFIG_REG_START) >> 2));
3691 radeon_ring_write(ring, next_rptr);
3692 } else if (rdev->wb.enabled) {
3693 next_rptr = ring->wptr + 5 + 4;
3694 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3695 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3696 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3697 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3698 radeon_ring_write(ring, next_rptr);
3699 }
3700
3701 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3702 }
3703
3704 control |= ib->length_dw |
3705 (ib->vm ? (ib->vm->id << 24) : 0);
3706
3707 radeon_ring_write(ring, header);
3708 radeon_ring_write(ring,
3709#ifdef __BIG_ENDIAN
3710 (2 << 0) |
3711#endif
3712 (ib->gpu_addr & 0xFFFFFFFC));
3713 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3714 radeon_ring_write(ring, control);
3715}
3716
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3717/**
3718 * cik_ib_test - basic gfx ring IB test
3719 *
3720 * @rdev: radeon_device pointer
3721 * @ring: radeon_ring structure holding ring information
3722 *
3723 * Allocate an IB and execute it on the gfx ring (CIK).
3724 * Provides a basic gfx ring test to verify that IBs are working.
3725 * Returns 0 on success, error on failure.
3726 */
3727int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3728{
3729 struct radeon_ib ib;
3730 uint32_t scratch;
3731 uint32_t tmp = 0;
3732 unsigned i;
3733 int r;
3734
3735 r = radeon_scratch_get(rdev, &scratch);
3736 if (r) {
3737 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3738 return r;
3739 }
3740 WREG32(scratch, 0xCAFEDEAD);
3741 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3742 if (r) {
3743 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3744 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3745 return r;
3746 }
3747 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3748 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3749 ib.ptr[2] = 0xDEADBEEF;
3750 ib.length_dw = 3;
3751 r = radeon_ib_schedule(rdev, &ib, NULL);
3752 if (r) {
3753 radeon_scratch_free(rdev, scratch);
3754 radeon_ib_free(rdev, &ib);
3755 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3756 return r;
3757 }
3758 r = radeon_fence_wait(ib.fence, false);
3759 if (r) {
3760 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3761 radeon_scratch_free(rdev, scratch);
3762 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3763 return r;
3764 }
3765 for (i = 0; i < rdev->usec_timeout; i++) {
3766 tmp = RREG32(scratch);
3767 if (tmp == 0xDEADBEEF)
3768 break;
3769 DRM_UDELAY(1);
3770 }
3771 if (i < rdev->usec_timeout) {
3772 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3773 } else {
3774 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3775 scratch, tmp);
3776 r = -EINVAL;
3777 }
3778 radeon_scratch_free(rdev, scratch);
3779 radeon_ib_free(rdev, &ib);
3780 return r;
3781}
3782
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3783/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3784 * CP.
3785 * On CIK, gfx and compute now have independant command processors.
3786 *
3787 * GFX
3788 * Gfx consists of a single ring and can process both gfx jobs and
3789 * compute jobs. The gfx CP consists of three microengines (ME):
3790 * PFP - Pre-Fetch Parser
3791 * ME - Micro Engine
3792 * CE - Constant Engine
3793 * The PFP and ME make up what is considered the Drawing Engine (DE).
3794 * The CE is an asynchronous engine used for updating buffer desciptors
3795 * used by the DE so that they can be loaded into cache in parallel
3796 * while the DE is processing state update packets.
3797 *
3798 * Compute
3799 * The compute CP consists of two microengines (ME):
3800 * MEC1 - Compute MicroEngine 1
3801 * MEC2 - Compute MicroEngine 2
3802 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3803 * The queues are exposed to userspace and are programmed directly
3804 * by the compute runtime.
3805 */
3806/**
3807 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3808 *
3809 * @rdev: radeon_device pointer
3810 * @enable: enable or disable the MEs
3811 *
3812 * Halts or unhalts the gfx MEs.
3813 */
3814static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3815{
3816 if (enable)
3817 WREG32(CP_ME_CNTL, 0);
3818 else {
3819 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3820 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3821 }
3822 udelay(50);
3823}
3824
3825/**
3826 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3827 *
3828 * @rdev: radeon_device pointer
3829 *
3830 * Loads the gfx PFP, ME, and CE ucode.
3831 * Returns 0 for success, -EINVAL if the ucode is not available.
3832 */
3833static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3834{
3835 const __be32 *fw_data;
3836 int i;
3837
3838 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3839 return -EINVAL;
3840
3841 cik_cp_gfx_enable(rdev, false);
3842
3843 /* PFP */
3844 fw_data = (const __be32 *)rdev->pfp_fw->data;
3845 WREG32(CP_PFP_UCODE_ADDR, 0);
3846 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3847 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3848 WREG32(CP_PFP_UCODE_ADDR, 0);
3849
3850 /* CE */
3851 fw_data = (const __be32 *)rdev->ce_fw->data;
3852 WREG32(CP_CE_UCODE_ADDR, 0);
3853 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3854 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3855 WREG32(CP_CE_UCODE_ADDR, 0);
3856
3857 /* ME */
3858 fw_data = (const __be32 *)rdev->me_fw->data;
3859 WREG32(CP_ME_RAM_WADDR, 0);
3860 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3861 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3862 WREG32(CP_ME_RAM_WADDR, 0);
3863
3864 WREG32(CP_PFP_UCODE_ADDR, 0);
3865 WREG32(CP_CE_UCODE_ADDR, 0);
3866 WREG32(CP_ME_RAM_WADDR, 0);
3867 WREG32(CP_ME_RAM_RADDR, 0);
3868 return 0;
3869}
3870
3871/**
3872 * cik_cp_gfx_start - start the gfx ring
3873 *
3874 * @rdev: radeon_device pointer
3875 *
3876 * Enables the ring and loads the clear state context and other
3877 * packets required to init the ring.
3878 * Returns 0 for success, error for failure.
3879 */
3880static int cik_cp_gfx_start(struct radeon_device *rdev)
3881{
3882 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3883 int r, i;
3884
3885 /* init the CP */
3886 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3887 WREG32(CP_ENDIAN_SWAP, 0);
3888 WREG32(CP_DEVICE_ID, 1);
3889
3890 cik_cp_gfx_enable(rdev, true);
3891
3892 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3893 if (r) {
3894 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3895 return r;
3896 }
3897
3898 /* init the CE partitions. CE only used for gfx on CIK */
3899 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3900 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3901 radeon_ring_write(ring, 0xc000);
3902 radeon_ring_write(ring, 0xc000);
3903
3904 /* setup clear context state */
3905 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3906 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3907
3908 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3909 radeon_ring_write(ring, 0x80000000);
3910 radeon_ring_write(ring, 0x80000000);
3911
3912 for (i = 0; i < cik_default_size; i++)
3913 radeon_ring_write(ring, cik_default_state[i]);
3914
3915 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3916 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3917
3918 /* set clear context state */
3919 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3920 radeon_ring_write(ring, 0);
3921
3922 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3923 radeon_ring_write(ring, 0x00000316);
3924 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3925 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3926
3927 radeon_ring_unlock_commit(rdev, ring);
3928
3929 return 0;
3930}
3931
3932/**
3933 * cik_cp_gfx_fini - stop the gfx ring
3934 *
3935 * @rdev: radeon_device pointer
3936 *
3937 * Stop the gfx ring and tear down the driver ring
3938 * info.
3939 */
3940static void cik_cp_gfx_fini(struct radeon_device *rdev)
3941{
3942 cik_cp_gfx_enable(rdev, false);
3943 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3944}
3945
3946/**
3947 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3948 *
3949 * @rdev: radeon_device pointer
3950 *
3951 * Program the location and size of the gfx ring buffer
3952 * and test it to make sure it's working.
3953 * Returns 0 for success, error for failure.
3954 */
3955static int cik_cp_gfx_resume(struct radeon_device *rdev)
3956{
3957 struct radeon_ring *ring;
3958 u32 tmp;
3959 u32 rb_bufsz;
3960 u64 rb_addr;
3961 int r;
3962
3963 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3964 if (rdev->family != CHIP_HAWAII)
3965 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3966
3967 /* Set the write pointer delay */
3968 WREG32(CP_RB_WPTR_DELAY, 0);
3969
3970 /* set the RB to use vmid 0 */
3971 WREG32(CP_RB_VMID, 0);
3972
3973 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3974
3975 /* ring 0 - compute and gfx */
3976 /* Set ring buffer size */
3977 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3978 rb_bufsz = order_base_2(ring->ring_size / 8);
3979 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3980#ifdef __BIG_ENDIAN
3981 tmp |= BUF_SWAP_32BIT;
3982#endif
3983 WREG32(CP_RB0_CNTL, tmp);
3984
3985 /* Initialize the ring buffer's read and write pointers */
3986 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3987 ring->wptr = 0;
3988 WREG32(CP_RB0_WPTR, ring->wptr);
3989
3990 /* set the wb address wether it's enabled or not */
3991 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3992 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3993
3994 /* scratch register shadowing is no longer supported */
3995 WREG32(SCRATCH_UMSK, 0);
3996
3997 if (!rdev->wb.enabled)
3998 tmp |= RB_NO_UPDATE;
3999
4000 mdelay(1);
4001 WREG32(CP_RB0_CNTL, tmp);
4002
4003 rb_addr = ring->gpu_addr >> 8;
4004 WREG32(CP_RB0_BASE, rb_addr);
4005 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4006
4007 ring->rptr = RREG32(CP_RB0_RPTR);
4008
4009 /* start the ring */
4010 cik_cp_gfx_start(rdev);
4011 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4012 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4013 if (r) {
4014 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4015 return r;
4016 }
4017 return 0;
4018}
4019
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4020u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4021 struct radeon_ring *ring)
4022{
4023 u32 rptr;
4024
4025
4026
4027 if (rdev->wb.enabled) {
4028 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4029 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4030 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4031 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4032 rptr = RREG32(CP_HQD_PQ_RPTR);
4033 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4034 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4035 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4036
4037 return rptr;
4038}
4039
4040u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4041 struct radeon_ring *ring)
4042{
4043 u32 wptr;
4044
4045 if (rdev->wb.enabled) {
4046 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4047 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4048 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4049 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4050 wptr = RREG32(CP_HQD_PQ_WPTR);
4051 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4052 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4053 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4054
4055 return wptr;
4056}
4057
4058void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4059 struct radeon_ring *ring)
4060{
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]4061 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4062 WDOORBELL32(ring->doorbell_index, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4063}
4064
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4065/**
4066 * cik_cp_compute_enable - enable/disable the compute CP MEs
4067 *
4068 * @rdev: radeon_device pointer
4069 * @enable: enable or disable the MEs
4070 *
4071 * Halts or unhalts the compute MEs.
4072 */
4073static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4074{
4075 if (enable)
4076 WREG32(CP_MEC_CNTL, 0);
4077 else
4078 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4079 udelay(50);
4080}
4081
4082/**
4083 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4084 *
4085 * @rdev: radeon_device pointer
4086 *
4087 * Loads the compute MEC1&2 ucode.
4088 * Returns 0 for success, -EINVAL if the ucode is not available.
4089 */
4090static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4091{
4092 const __be32 *fw_data;
4093 int i;
4094
4095 if (!rdev->mec_fw)
4096 return -EINVAL;
4097
4098 cik_cp_compute_enable(rdev, false);
4099
4100 /* MEC1 */
4101 fw_data = (const __be32 *)rdev->mec_fw->data;
4102 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4103 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4104 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4105 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4106
4107 if (rdev->family == CHIP_KAVERI) {
4108 /* MEC2 */
4109 fw_data = (const __be32 *)rdev->mec_fw->data;
4110 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4111 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4112 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4113 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4114 }
4115
4116 return 0;
4117}
4118
4119/**
4120 * cik_cp_compute_start - start the compute queues
4121 *
4122 * @rdev: radeon_device pointer
4123 *
4124 * Enable the compute queues.
4125 * Returns 0 for success, error for failure.
4126 */
4127static int cik_cp_compute_start(struct radeon_device *rdev)
4128{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4129 cik_cp_compute_enable(rdev, true);
4130
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4131 return 0;
4132}
4133
4134/**
4135 * cik_cp_compute_fini - stop the compute queues
4136 *
4137 * @rdev: radeon_device pointer
4138 *
4139 * Stop the compute queues and tear down the driver queue
4140 * info.
4141 */
4142static void cik_cp_compute_fini(struct radeon_device *rdev)
4143{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4144 int i, idx, r;
4145
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4146 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4147
4148 for (i = 0; i < 2; i++) {
4149 if (i == 0)
4150 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4151 else
4152 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4153
4154 if (rdev->ring[idx].mqd_obj) {
4155 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4156 if (unlikely(r != 0))
4157 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4158
4159 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4160 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4161
4162 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4163 rdev->ring[idx].mqd_obj = NULL;
4164 }
4165 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4166}
4167
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4168static void cik_mec_fini(struct radeon_device *rdev)
4169{
4170 int r;
4171
4172 if (rdev->mec.hpd_eop_obj) {
4173 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4174 if (unlikely(r != 0))
4175 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4176 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4177 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4178
4179 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4180 rdev->mec.hpd_eop_obj = NULL;
4181 }
4182}
4183
4184#define MEC_HPD_SIZE 2048
4185
4186static int cik_mec_init(struct radeon_device *rdev)
4187{
4188 int r;
4189 u32 *hpd;
4190
4191 /*
4192 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4193 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4194 */
4195 if (rdev->family == CHIP_KAVERI)
4196 rdev->mec.num_mec = 2;
4197 else
4198 rdev->mec.num_mec = 1;
4199 rdev->mec.num_pipe = 4;
4200 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4201
4202 if (rdev->mec.hpd_eop_obj == NULL) {
4203 r = radeon_bo_create(rdev,
4204 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4205 PAGE_SIZE, true,
4206 RADEON_GEM_DOMAIN_GTT, NULL,
4207 &rdev->mec.hpd_eop_obj);
4208 if (r) {
4209 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4210 return r;
4211 }
4212 }
4213
4214 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4215 if (unlikely(r != 0)) {
4216 cik_mec_fini(rdev);
4217 return r;
4218 }
4219 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4220 &rdev->mec.hpd_eop_gpu_addr);
4221 if (r) {
4222 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4223 cik_mec_fini(rdev);
4224 return r;
4225 }
4226 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4227 if (r) {
4228 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4229 cik_mec_fini(rdev);
4230 return r;
4231 }
4232
4233 /* clear memory. Not sure if this is required or not */
4234 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4235
4236 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4237 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4238
4239 return 0;
4240}
4241
4242struct hqd_registers
4243{
4244 u32 cp_mqd_base_addr;
4245 u32 cp_mqd_base_addr_hi;
4246 u32 cp_hqd_active;
4247 u32 cp_hqd_vmid;
4248 u32 cp_hqd_persistent_state;
4249 u32 cp_hqd_pipe_priority;
4250 u32 cp_hqd_queue_priority;
4251 u32 cp_hqd_quantum;
4252 u32 cp_hqd_pq_base;
4253 u32 cp_hqd_pq_base_hi;
4254 u32 cp_hqd_pq_rptr;
4255 u32 cp_hqd_pq_rptr_report_addr;
4256 u32 cp_hqd_pq_rptr_report_addr_hi;
4257 u32 cp_hqd_pq_wptr_poll_addr;
4258 u32 cp_hqd_pq_wptr_poll_addr_hi;
4259 u32 cp_hqd_pq_doorbell_control;
4260 u32 cp_hqd_pq_wptr;
4261 u32 cp_hqd_pq_control;
4262 u32 cp_hqd_ib_base_addr;
4263 u32 cp_hqd_ib_base_addr_hi;
4264 u32 cp_hqd_ib_rptr;
4265 u32 cp_hqd_ib_control;
4266 u32 cp_hqd_iq_timer;
4267 u32 cp_hqd_iq_rptr;
4268 u32 cp_hqd_dequeue_request;
4269 u32 cp_hqd_dma_offload;
4270 u32 cp_hqd_sema_cmd;
4271 u32 cp_hqd_msg_type;
4272 u32 cp_hqd_atomic0_preop_lo;
4273 u32 cp_hqd_atomic0_preop_hi;
4274 u32 cp_hqd_atomic1_preop_lo;
4275 u32 cp_hqd_atomic1_preop_hi;
4276 u32 cp_hqd_hq_scheduler0;
4277 u32 cp_hqd_hq_scheduler1;
4278 u32 cp_mqd_control;
4279};
4280
4281struct bonaire_mqd
4282{
4283 u32 header;
4284 u32 dispatch_initiator;
4285 u32 dimensions[3];
4286 u32 start_idx[3];
4287 u32 num_threads[3];
4288 u32 pipeline_stat_enable;
4289 u32 perf_counter_enable;
4290 u32 pgm[2];
4291 u32 tba[2];
4292 u32 tma[2];
4293 u32 pgm_rsrc[2];
4294 u32 vmid;
4295 u32 resource_limits;
4296 u32 static_thread_mgmt01[2];
4297 u32 tmp_ring_size;
4298 u32 static_thread_mgmt23[2];
4299 u32 restart[3];
4300 u32 thread_trace_enable;
4301 u32 reserved1;
4302 u32 user_data[16];
4303 u32 vgtcs_invoke_count[2];
4304 struct hqd_registers queue_state;
4305 u32 dequeue_cntr;
4306 u32 interrupt_queue[64];
4307};
4308
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4309/**
4310 * cik_cp_compute_resume - setup the compute queue registers
4311 *
4312 * @rdev: radeon_device pointer
4313 *
4314 * Program the compute queues and test them to make sure they
4315 * are working.
4316 * Returns 0 for success, error for failure.
4317 */
4318static int cik_cp_compute_resume(struct radeon_device *rdev)
4319{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4320 int r, i, idx;
4321 u32 tmp;
4322 bool use_doorbell = true;
4323 u64 hqd_gpu_addr;
4324 u64 mqd_gpu_addr;
4325 u64 eop_gpu_addr;
4326 u64 wb_gpu_addr;
4327 u32 *buf;
4328 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4329
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4330 r = cik_cp_compute_start(rdev);
4331 if (r)
4332 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4333
4334 /* fix up chicken bits */
4335 tmp = RREG32(CP_CPF_DEBUG);
4336 tmp |= (1 << 23);
4337 WREG32(CP_CPF_DEBUG, tmp);
4338
4339 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4340 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4341 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4342 int me = (i < 4) ? 1 : 2;
4343 int pipe = (i < 4) ? i : (i - 4);
4344
4345 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4346
4347 cik_srbm_select(rdev, me, pipe, 0, 0);
4348
4349 /* write the EOP addr */
4350 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4351 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4352
4353 /* set the VMID assigned */
4354 WREG32(CP_HPD_EOP_VMID, 0);
4355
4356 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4357 tmp = RREG32(CP_HPD_EOP_CONTROL);
4358 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4359 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4360 WREG32(CP_HPD_EOP_CONTROL, tmp);
4361 }
4362 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4363 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4364
4365 /* init the queues. Just two for now. */
4366 for (i = 0; i < 2; i++) {
4367 if (i == 0)
4368 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4369 else
4370 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4371
4372 if (rdev->ring[idx].mqd_obj == NULL) {
4373 r = radeon_bo_create(rdev,
4374 sizeof(struct bonaire_mqd),
4375 PAGE_SIZE, true,
4376 RADEON_GEM_DOMAIN_GTT, NULL,
4377 &rdev->ring[idx].mqd_obj);
4378 if (r) {
4379 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4380 return r;
4381 }
4382 }
4383
4384 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4385 if (unlikely(r != 0)) {
4386 cik_cp_compute_fini(rdev);
4387 return r;
4388 }
4389 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4390 &mqd_gpu_addr);
4391 if (r) {
4392 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4393 cik_cp_compute_fini(rdev);
4394 return r;
4395 }
4396 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4397 if (r) {
4398 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4399 cik_cp_compute_fini(rdev);
4400 return r;
4401 }
4402
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4403 /* init the mqd struct */
4404 memset(buf, 0, sizeof(struct bonaire_mqd));
4405
4406 mqd = (struct bonaire_mqd *)buf;
4407 mqd->header = 0xC0310800;
4408 mqd->static_thread_mgmt01[0] = 0xffffffff;
4409 mqd->static_thread_mgmt01[1] = 0xffffffff;
4410 mqd->static_thread_mgmt23[0] = 0xffffffff;
4411 mqd->static_thread_mgmt23[1] = 0xffffffff;
4412
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4413 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4414 cik_srbm_select(rdev, rdev->ring[idx].me,
4415 rdev->ring[idx].pipe,
4416 rdev->ring[idx].queue, 0);
4417
4418 /* disable wptr polling */
4419 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4420 tmp &= ~WPTR_POLL_EN;
4421 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4422
4423 /* enable doorbell? */
4424 mqd->queue_state.cp_hqd_pq_doorbell_control =
4425 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4426 if (use_doorbell)
4427 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4428 else
4429 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4430 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4431 mqd->queue_state.cp_hqd_pq_doorbell_control);
4432
4433 /* disable the queue if it's active */
4434 mqd->queue_state.cp_hqd_dequeue_request = 0;
4435 mqd->queue_state.cp_hqd_pq_rptr = 0;
4436 mqd->queue_state.cp_hqd_pq_wptr= 0;
4437 if (RREG32(CP_HQD_ACTIVE) & 1) {
4438 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4439 for (i = 0; i < rdev->usec_timeout; i++) {
4440 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4441 break;
4442 udelay(1);
4443 }
4444 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4445 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4446 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4447 }
4448
4449 /* set the pointer to the MQD */
4450 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4451 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4452 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4453 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4454 /* set MQD vmid to 0 */
4455 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4456 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4457 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4458
4459 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4460 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4461 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4462 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4463 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4464 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4465
4466 /* set up the HQD, this is similar to CP_RB0_CNTL */
4467 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4468 mqd->queue_state.cp_hqd_pq_control &=
4469 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4470
4471 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4472 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4473 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4474 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4475#ifdef __BIG_ENDIAN
4476 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4477#endif
4478 mqd->queue_state.cp_hqd_pq_control &=
4479 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4480 mqd->queue_state.cp_hqd_pq_control |=
4481 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4482 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4483
4484 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4485 if (i == 0)
4486 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4487 else
4488 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4489 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4490 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4491 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4492 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4493 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4494
4495 /* set the wb address wether it's enabled or not */
4496 if (i == 0)
4497 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4498 else
4499 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4500 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4501 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4502 upper_32_bits(wb_gpu_addr) & 0xffff;
4503 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4504 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4505 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4506 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4507
4508 /* enable the doorbell if requested */
4509 if (use_doorbell) {
4510 mqd->queue_state.cp_hqd_pq_doorbell_control =
4511 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4512 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4513 mqd->queue_state.cp_hqd_pq_doorbell_control |=
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]4514 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4515 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4516 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4517 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4518
4519 } else {
4520 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4521 }
4522 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4523 mqd->queue_state.cp_hqd_pq_doorbell_control);
4524
4525 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4526 rdev->ring[idx].wptr = 0;
4527 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4528 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4529 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4530 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4531
4532 /* set the vmid for the queue */
4533 mqd->queue_state.cp_hqd_vmid = 0;
4534 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4535
4536 /* activate the queue */
4537 mqd->queue_state.cp_hqd_active = 1;
4538 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4539
4540 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4541 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4542
4543 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4544 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4545
4546 rdev->ring[idx].ready = true;
4547 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4548 if (r)
4549 rdev->ring[idx].ready = false;
4550 }
4551
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4552 return 0;
4553}
4554
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4555static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4556{
4557 cik_cp_gfx_enable(rdev, enable);
4558 cik_cp_compute_enable(rdev, enable);
4559}
4560
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4561static int cik_cp_load_microcode(struct radeon_device *rdev)
4562{
4563 int r;
4564
4565 r = cik_cp_gfx_load_microcode(rdev);
4566 if (r)
4567 return r;
4568 r = cik_cp_compute_load_microcode(rdev);
4569 if (r)
4570 return r;
4571
4572 return 0;
4573}
4574
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4575static void cik_cp_fini(struct radeon_device *rdev)
4576{
4577 cik_cp_gfx_fini(rdev);
4578 cik_cp_compute_fini(rdev);
4579}
4580
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4581static int cik_cp_resume(struct radeon_device *rdev)
4582{
4583 int r;
4584
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4585 cik_enable_gui_idle_interrupt(rdev, false);
4586
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4587 r = cik_cp_load_microcode(rdev);
4588 if (r)
4589 return r;
4590
4591 r = cik_cp_gfx_resume(rdev);
4592 if (r)
4593 return r;
4594 r = cik_cp_compute_resume(rdev);
4595 if (r)
4596 return r;
4597
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4598 cik_enable_gui_idle_interrupt(rdev, true);
4599
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4600 return 0;
4601}
4602
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4603static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4604{
4605 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4606 RREG32(GRBM_STATUS));
4607 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4608 RREG32(GRBM_STATUS2));
4609 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4610 RREG32(GRBM_STATUS_SE0));
4611 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4612 RREG32(GRBM_STATUS_SE1));
4613 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4614 RREG32(GRBM_STATUS_SE2));
4615 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4616 RREG32(GRBM_STATUS_SE3));
4617 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4618 RREG32(SRBM_STATUS));
4619 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4620 RREG32(SRBM_STATUS2));
4621 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4622 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4623 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4624 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4625 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4626 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4627 RREG32(CP_STALLED_STAT1));
4628 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4629 RREG32(CP_STALLED_STAT2));
4630 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4631 RREG32(CP_STALLED_STAT3));
4632 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4633 RREG32(CP_CPF_BUSY_STAT));
4634 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4635 RREG32(CP_CPF_STALLED_STAT1));
4636 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4637 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4638 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4639 RREG32(CP_CPC_STALLED_STAT1));
4640 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4641}
4642
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4643/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4644 * cik_gpu_check_soft_reset - check which blocks are busy
4645 *
4646 * @rdev: radeon_device pointer
4647 *
4648 * Check which blocks are busy and return the relevant reset
4649 * mask to be used by cik_gpu_soft_reset().
4650 * Returns a mask of the blocks to be reset.
4651 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4652u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4653{
4654 u32 reset_mask = 0;
4655 u32 tmp;
4656
4657 /* GRBM_STATUS */
4658 tmp = RREG32(GRBM_STATUS);
4659 if (tmp & (PA_BUSY | SC_BUSY |
4660 BCI_BUSY | SX_BUSY |
4661 TA_BUSY | VGT_BUSY |
4662 DB_BUSY | CB_BUSY |
4663 GDS_BUSY | SPI_BUSY |
4664 IA_BUSY | IA_BUSY_NO_DMA))
4665 reset_mask |= RADEON_RESET_GFX;
4666
4667 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4668 reset_mask |= RADEON_RESET_CP;
4669
4670 /* GRBM_STATUS2 */
4671 tmp = RREG32(GRBM_STATUS2);
4672 if (tmp & RLC_BUSY)
4673 reset_mask |= RADEON_RESET_RLC;
4674
4675 /* SDMA0_STATUS_REG */
4676 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4677 if (!(tmp & SDMA_IDLE))
4678 reset_mask |= RADEON_RESET_DMA;
4679
4680 /* SDMA1_STATUS_REG */
4681 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4682 if (!(tmp & SDMA_IDLE))
4683 reset_mask |= RADEON_RESET_DMA1;
4684
4685 /* SRBM_STATUS2 */
4686 tmp = RREG32(SRBM_STATUS2);
4687 if (tmp & SDMA_BUSY)
4688 reset_mask |= RADEON_RESET_DMA;
4689
4690 if (tmp & SDMA1_BUSY)
4691 reset_mask |= RADEON_RESET_DMA1;
4692
4693 /* SRBM_STATUS */
4694 tmp = RREG32(SRBM_STATUS);
4695
4696 if (tmp & IH_BUSY)
4697 reset_mask |= RADEON_RESET_IH;
4698
4699 if (tmp & SEM_BUSY)
4700 reset_mask |= RADEON_RESET_SEM;
4701
4702 if (tmp & GRBM_RQ_PENDING)
4703 reset_mask |= RADEON_RESET_GRBM;
4704
4705 if (tmp & VMC_BUSY)
4706 reset_mask |= RADEON_RESET_VMC;
4707
4708 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4709 MCC_BUSY | MCD_BUSY))
4710 reset_mask |= RADEON_RESET_MC;
4711
4712 if (evergreen_is_display_hung(rdev))
4713 reset_mask |= RADEON_RESET_DISPLAY;
4714
4715 /* Skip MC reset as it's mostly likely not hung, just busy */
4716 if (reset_mask & RADEON_RESET_MC) {
4717 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4718 reset_mask &= ~RADEON_RESET_MC;
4719 }
4720
4721 return reset_mask;
4722}
4723
4724/**
4725 * cik_gpu_soft_reset - soft reset GPU
4726 *
4727 * @rdev: radeon_device pointer
4728 * @reset_mask: mask of which blocks to reset
4729 *
4730 * Soft reset the blocks specified in @reset_mask.
4731 */
4732static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4733{
4734 struct evergreen_mc_save save;
4735 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4736 u32 tmp;
4737
4738 if (reset_mask == 0)
4739 return;
4740
4741 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4742
4743 cik_print_gpu_status_regs(rdev);
4744 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4745 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4746 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4747 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4748
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4749 /* disable CG/PG */
4750 cik_fini_pg(rdev);
4751 cik_fini_cg(rdev);
4752
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4753 /* stop the rlc */
4754 cik_rlc_stop(rdev);
4755
4756 /* Disable GFX parsing/prefetching */
4757 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4758
4759 /* Disable MEC parsing/prefetching */
4760 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4761
4762 if (reset_mask & RADEON_RESET_DMA) {
4763 /* sdma0 */
4764 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4765 tmp |= SDMA_HALT;
4766 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4767 }
4768 if (reset_mask & RADEON_RESET_DMA1) {
4769 /* sdma1 */
4770 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4771 tmp |= SDMA_HALT;
4772 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4773 }
4774
4775 evergreen_mc_stop(rdev, &save);
4776 if (evergreen_mc_wait_for_idle(rdev)) {
4777 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4778 }
4779
4780 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4781 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4782
4783 if (reset_mask & RADEON_RESET_CP) {
4784 grbm_soft_reset |= SOFT_RESET_CP;
4785
4786 srbm_soft_reset |= SOFT_RESET_GRBM;
4787 }
4788
4789 if (reset_mask & RADEON_RESET_DMA)
4790 srbm_soft_reset |= SOFT_RESET_SDMA;
4791
4792 if (reset_mask & RADEON_RESET_DMA1)
4793 srbm_soft_reset |= SOFT_RESET_SDMA1;
4794
4795 if (reset_mask & RADEON_RESET_DISPLAY)
4796 srbm_soft_reset |= SOFT_RESET_DC;
4797
4798 if (reset_mask & RADEON_RESET_RLC)
4799 grbm_soft_reset |= SOFT_RESET_RLC;
4800
4801 if (reset_mask & RADEON_RESET_SEM)
4802 srbm_soft_reset |= SOFT_RESET_SEM;
4803
4804 if (reset_mask & RADEON_RESET_IH)
4805 srbm_soft_reset |= SOFT_RESET_IH;
4806
4807 if (reset_mask & RADEON_RESET_GRBM)
4808 srbm_soft_reset |= SOFT_RESET_GRBM;
4809
4810 if (reset_mask & RADEON_RESET_VMC)
4811 srbm_soft_reset |= SOFT_RESET_VMC;
4812
4813 if (!(rdev->flags & RADEON_IS_IGP)) {
4814 if (reset_mask & RADEON_RESET_MC)
4815 srbm_soft_reset |= SOFT_RESET_MC;
4816 }
4817
4818 if (grbm_soft_reset) {
4819 tmp = RREG32(GRBM_SOFT_RESET);
4820 tmp |= grbm_soft_reset;
4821 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4822 WREG32(GRBM_SOFT_RESET, tmp);
4823 tmp = RREG32(GRBM_SOFT_RESET);
4824
4825 udelay(50);
4826
4827 tmp &= ~grbm_soft_reset;
4828 WREG32(GRBM_SOFT_RESET, tmp);
4829 tmp = RREG32(GRBM_SOFT_RESET);
4830 }
4831
4832 if (srbm_soft_reset) {
4833 tmp = RREG32(SRBM_SOFT_RESET);
4834 tmp |= srbm_soft_reset;
4835 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4836 WREG32(SRBM_SOFT_RESET, tmp);
4837 tmp = RREG32(SRBM_SOFT_RESET);
4838
4839 udelay(50);
4840
4841 tmp &= ~srbm_soft_reset;
4842 WREG32(SRBM_SOFT_RESET, tmp);
4843 tmp = RREG32(SRBM_SOFT_RESET);
4844 }
4845
4846 /* Wait a little for things to settle down */
4847 udelay(50);
4848
4849 evergreen_mc_resume(rdev, &save);
4850 udelay(50);
4851
4852 cik_print_gpu_status_regs(rdev);
4853}
4854
4855/**
4856 * cik_asic_reset - soft reset GPU
4857 *
4858 * @rdev: radeon_device pointer
4859 *
4860 * Look up which blocks are hung and attempt
4861 * to reset them.
4862 * Returns 0 for success.
4863 */
4864int cik_asic_reset(struct radeon_device *rdev)
4865{
4866 u32 reset_mask;
4867
4868 reset_mask = cik_gpu_check_soft_reset(rdev);
4869
4870 if (reset_mask)
4871 r600_set_bios_scratch_engine_hung(rdev, true);
4872
4873 cik_gpu_soft_reset(rdev, reset_mask);
4874
4875 reset_mask = cik_gpu_check_soft_reset(rdev);
4876
4877 if (!reset_mask)
4878 r600_set_bios_scratch_engine_hung(rdev, false);
4879
4880 return 0;
4881}
4882
4883/**
4884 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4885 *
4886 * @rdev: radeon_device pointer
4887 * @ring: radeon_ring structure holding ring information
4888 *
4889 * Check if the 3D engine is locked up (CIK).
4890 * Returns true if the engine is locked, false if not.
4891 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4892bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4893{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4894 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4895
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4896 if (!(reset_mask & (RADEON_RESET_GFX |
4897 RADEON_RESET_COMPUTE |
4898 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4899 radeon_ring_lockup_update(ring);
4900 return false;
4901 }
4902 /* force CP activities */
4903 radeon_ring_force_activity(rdev, ring);
4904 return radeon_ring_test_lockup(rdev, ring);
4905}
4906
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4907/* MC */
4908/**
4909 * cik_mc_program - program the GPU memory controller
4910 *
4911 * @rdev: radeon_device pointer
4912 *
4913 * Set the location of vram, gart, and AGP in the GPU's
4914 * physical address space (CIK).
4915 */
4916static void cik_mc_program(struct radeon_device *rdev)
4917{
4918 struct evergreen_mc_save save;
4919 u32 tmp;
4920 int i, j;
4921
4922 /* Initialize HDP */
4923 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4924 WREG32((0x2c14 + j), 0x00000000);
4925 WREG32((0x2c18 + j), 0x00000000);
4926 WREG32((0x2c1c + j), 0x00000000);
4927 WREG32((0x2c20 + j), 0x00000000);
4928 WREG32((0x2c24 + j), 0x00000000);
4929 }
4930 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4931
4932 evergreen_mc_stop(rdev, &save);
4933 if (radeon_mc_wait_for_idle(rdev)) {
4934 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4935 }
4936 /* Lockout access through VGA aperture*/
4937 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4938 /* Update configuration */
4939 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4940 rdev->mc.vram_start >> 12);
4941 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4942 rdev->mc.vram_end >> 12);
4943 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4944 rdev->vram_scratch.gpu_addr >> 12);
4945 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4946 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4947 WREG32(MC_VM_FB_LOCATION, tmp);
4948 /* XXX double check these! */
4949 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4950 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4951 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4952 WREG32(MC_VM_AGP_BASE, 0);
4953 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4954 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4955 if (radeon_mc_wait_for_idle(rdev)) {
4956 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4957 }
4958 evergreen_mc_resume(rdev, &save);
4959 /* we need to own VRAM, so turn off the VGA renderer here
4960 * to stop it overwriting our objects */
4961 rv515_vga_render_disable(rdev);
4962}
4963
4964/**
4965 * cik_mc_init - initialize the memory controller driver params
4966 *
4967 * @rdev: radeon_device pointer
4968 *
4969 * Look up the amount of vram, vram width, and decide how to place
4970 * vram and gart within the GPU's physical address space (CIK).
4971 * Returns 0 for success.
4972 */
4973static int cik_mc_init(struct radeon_device *rdev)
4974{
4975 u32 tmp;
4976 int chansize, numchan;
4977
4978 /* Get VRAM informations */
4979 rdev->mc.vram_is_ddr = true;
4980 tmp = RREG32(MC_ARB_RAMCFG);
4981 if (tmp & CHANSIZE_MASK) {
4982 chansize = 64;
4983 } else {
4984 chansize = 32;
4985 }
4986 tmp = RREG32(MC_SHARED_CHMAP);
4987 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4988 case 0:
4989 default:
4990 numchan = 1;
4991 break;
4992 case 1:
4993 numchan = 2;
4994 break;
4995 case 2:
4996 numchan = 4;
4997 break;
4998 case 3:
4999 numchan = 8;
5000 break;
5001 case 4:
5002 numchan = 3;
5003 break;
5004 case 5:
5005 numchan = 6;
5006 break;
5007 case 6:
5008 numchan = 10;
5009 break;
5010 case 7:
5011 numchan = 12;
5012 break;
5013 case 8:
5014 numchan = 16;
5015 break;
5016 }
5017 rdev->mc.vram_width = numchan * chansize;
5018 /* Could aper size report 0 ? */
5019 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5020 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5021 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]5022 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5023 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5024 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5025 si_vram_gtt_location(rdev, &rdev->mc);
5026 radeon_update_bandwidth_info(rdev);
5027
5028 return 0;
5029}
5030
5031/*
5032 * GART
5033 * VMID 0 is the physical GPU addresses as used by the kernel.
5034 * VMIDs 1-15 are used for userspace clients and are handled
5035 * by the radeon vm/hsa code.
5036 */
5037/**
5038 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5039 *
5040 * @rdev: radeon_device pointer
5041 *
5042 * Flush the TLB for the VMID 0 page table (CIK).
5043 */
5044void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5045{
5046 /* flush hdp cache */
5047 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5048
5049 /* bits 0-15 are the VM contexts0-15 */
5050 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5051}
5052
5053/**
5054 * cik_pcie_gart_enable - gart enable
5055 *
5056 * @rdev: radeon_device pointer
5057 *
5058 * This sets up the TLBs, programs the page tables for VMID0,
5059 * sets up the hw for VMIDs 1-15 which are allocated on
5060 * demand, and sets up the global locations for the LDS, GDS,
5061 * and GPUVM for FSA64 clients (CIK).
5062 * Returns 0 for success, errors for failure.
5063 */
5064static int cik_pcie_gart_enable(struct radeon_device *rdev)
5065{
5066 int r, i;
5067
5068 if (rdev->gart.robj == NULL) {
5069 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5070 return -EINVAL;
5071 }
5072 r = radeon_gart_table_vram_pin(rdev);
5073 if (r)
5074 return r;
5075 radeon_gart_restore(rdev);
5076 /* Setup TLB control */
5077 WREG32(MC_VM_MX_L1_TLB_CNTL,
5078 (0xA << 7) |
5079 ENABLE_L1_TLB |
5080 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5081 ENABLE_ADVANCED_DRIVER_MODEL |
5082 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5083 /* Setup L2 cache */
5084 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5085 ENABLE_L2_FRAGMENT_PROCESSING |
5086 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5087 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5088 EFFECTIVE_L2_QUEUE_SIZE(7) |
5089 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5090 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5091 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5092 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5093 /* setup context0 */
5094 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5095 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5096 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5097 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5098 (u32)(rdev->dummy_page.addr >> 12));
5099 WREG32(VM_CONTEXT0_CNTL2, 0);
5100 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5101 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5102
5103 WREG32(0x15D4, 0);
5104 WREG32(0x15D8, 0);
5105 WREG32(0x15DC, 0);
5106
5107 /* empty context1-15 */
5108 /* FIXME start with 4G, once using 2 level pt switch to full
5109 * vm size space
5110 */
5111 /* set vm size, must be a multiple of 4 */
5112 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5113 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5114 for (i = 1; i < 16; i++) {
5115 if (i < 8)
5116 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5117 rdev->gart.table_addr >> 12);
5118 else
5119 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5120 rdev->gart.table_addr >> 12);
5121 }
5122
5123 /* enable context1-15 */
5124 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5125 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5126 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5127 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5128 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5129 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5130 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5131 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5132 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5133 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5134 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5135 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5136 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5137 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5138 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5139 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5140
5141 /* TC cache setup ??? */
5142 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5143 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5144 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5145
5146 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5147 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5148 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5149 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5150 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5151
5152 WREG32(TC_CFG_L1_VOLATILE, 0);
5153 WREG32(TC_CFG_L2_VOLATILE, 0);
5154
5155 if (rdev->family == CHIP_KAVERI) {
5156 u32 tmp = RREG32(CHUB_CONTROL);
5157 tmp &= ~BYPASS_VM;
5158 WREG32(CHUB_CONTROL, tmp);
5159 }
5160
5161 /* XXX SH_MEM regs */
5162 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5163 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5164 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5165 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5166 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5167 WREG32(SH_MEM_CONFIG, 0);
5168 WREG32(SH_MEM_APE1_BASE, 1);
5169 WREG32(SH_MEM_APE1_LIMIT, 0);
5170 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5171 /* SDMA GFX */
5172 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5173 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5174 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5175 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5176 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5177 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5178 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5179 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5180
5181 cik_pcie_gart_tlb_flush(rdev);
5182 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5183 (unsigned)(rdev->mc.gtt_size >> 20),
5184 (unsigned long long)rdev->gart.table_addr);
5185 rdev->gart.ready = true;
5186 return 0;
5187}
5188
5189/**
5190 * cik_pcie_gart_disable - gart disable
5191 *
5192 * @rdev: radeon_device pointer
5193 *
5194 * This disables all VM page table (CIK).
5195 */
5196static void cik_pcie_gart_disable(struct radeon_device *rdev)
5197{
5198 /* Disable all tables */
5199 WREG32(VM_CONTEXT0_CNTL, 0);
5200 WREG32(VM_CONTEXT1_CNTL, 0);
5201 /* Setup TLB control */
5202 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5203 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5204 /* Setup L2 cache */
5205 WREG32(VM_L2_CNTL,
5206 ENABLE_L2_FRAGMENT_PROCESSING |
5207 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5208 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5209 EFFECTIVE_L2_QUEUE_SIZE(7) |
5210 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5211 WREG32(VM_L2_CNTL2, 0);
5212 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5213 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5214 radeon_gart_table_vram_unpin(rdev);
5215}
5216
5217/**
5218 * cik_pcie_gart_fini - vm fini callback
5219 *
5220 * @rdev: radeon_device pointer
5221 *
5222 * Tears down the driver GART/VM setup (CIK).
5223 */
5224static void cik_pcie_gart_fini(struct radeon_device *rdev)
5225{
5226 cik_pcie_gart_disable(rdev);
5227 radeon_gart_table_vram_free(rdev);
5228 radeon_gart_fini(rdev);
5229}
5230
5231/* vm parser */
5232/**
5233 * cik_ib_parse - vm ib_parse callback
5234 *
5235 * @rdev: radeon_device pointer
5236 * @ib: indirect buffer pointer
5237 *
5238 * CIK uses hw IB checking so this is a nop (CIK).
5239 */
5240int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5241{
5242 return 0;
5243}
5244
5245/*
5246 * vm
5247 * VMID 0 is the physical GPU addresses as used by the kernel.
5248 * VMIDs 1-15 are used for userspace clients and are handled
5249 * by the radeon vm/hsa code.
5250 */
5251/**
5252 * cik_vm_init - cik vm init callback
5253 *
5254 * @rdev: radeon_device pointer
5255 *
5256 * Inits cik specific vm parameters (number of VMs, base of vram for
5257 * VMIDs 1-15) (CIK).
5258 * Returns 0 for success.
5259 */
5260int cik_vm_init(struct radeon_device *rdev)
5261{
5262 /* number of VMs */
5263 rdev->vm_manager.nvm = 16;
5264 /* base offset of vram pages */
5265 if (rdev->flags & RADEON_IS_IGP) {
5266 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5267 tmp <<= 22;
5268 rdev->vm_manager.vram_base_offset = tmp;
5269 } else
5270 rdev->vm_manager.vram_base_offset = 0;
5271
5272 return 0;
5273}
5274
5275/**
5276 * cik_vm_fini - cik vm fini callback
5277 *
5278 * @rdev: radeon_device pointer
5279 *
5280 * Tear down any asic specific VM setup (CIK).
5281 */
5282void cik_vm_fini(struct radeon_device *rdev)
5283{
5284}
5285
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5286/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5287 * cik_vm_decode_fault - print human readable fault info
5288 *
5289 * @rdev: radeon_device pointer
5290 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5291 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5292 *
5293 * Print human readable fault information (CIK).
5294 */
5295static void cik_vm_decode_fault(struct radeon_device *rdev,
5296 u32 status, u32 addr, u32 mc_client)
5297{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5298 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5299 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5300 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5301 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5302 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5303
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5304 if (rdev->family == CHIP_HAWAII)
5305 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5306 else
5307 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5308
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5309 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5310 protections, vmid, addr,
5311 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5312 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5313}
5314
5315/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5316 * cik_vm_flush - cik vm flush using the CP
5317 *
5318 * @rdev: radeon_device pointer
5319 *
5320 * Update the page table base and flush the VM TLB
5321 * using the CP (CIK).
5322 */
5323void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5324{
5325 struct radeon_ring *ring = &rdev->ring[ridx];
5326
5327 if (vm == NULL)
5328 return;
5329
5330 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5331 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5332 WRITE_DATA_DST_SEL(0)));
5333 if (vm->id < 8) {
5334 radeon_ring_write(ring,
5335 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5336 } else {
5337 radeon_ring_write(ring,
5338 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5339 }
5340 radeon_ring_write(ring, 0);
5341 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5342
5343 /* update SH_MEM_* regs */
5344 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5345 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5346 WRITE_DATA_DST_SEL(0)));
5347 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5348 radeon_ring_write(ring, 0);
5349 radeon_ring_write(ring, VMID(vm->id));
5350
5351 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5352 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5353 WRITE_DATA_DST_SEL(0)));
5354 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5355 radeon_ring_write(ring, 0);
5356
5357 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5358 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5359 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5360 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5361
5362 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5363 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5364 WRITE_DATA_DST_SEL(0)));
5365 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5366 radeon_ring_write(ring, 0);
5367 radeon_ring_write(ring, VMID(0));
5368
5369 /* HDP flush */
5370 /* We should be using the WAIT_REG_MEM packet here like in
5371 * cik_fence_ring_emit(), but it causes the CP to hang in this
5372 * context...
5373 */
5374 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5375 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5376 WRITE_DATA_DST_SEL(0)));
5377 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5378 radeon_ring_write(ring, 0);
5379 radeon_ring_write(ring, 0);
5380
5381 /* bits 0-15 are the VM contexts0-15 */
5382 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5383 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5384 WRITE_DATA_DST_SEL(0)));
5385 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5386 radeon_ring_write(ring, 0);
5387 radeon_ring_write(ring, 1 << vm->id);
5388
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5389 /* compute doesn't have PFP */
5390 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5391 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5392 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5393 radeon_ring_write(ring, 0x0);
5394 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5395}
5396
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5397/*
5398 * RLC
5399 * The RLC is a multi-purpose microengine that handles a
5400 * variety of functions, the most important of which is
5401 * the interrupt controller.
5402 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5403static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5404 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5405{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5406 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5407
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5408 if (enable)
5409 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5410 else
5411 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5412 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5413}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5414
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5415static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5416{
5417 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5418
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5419 tmp = RREG32(RLC_LB_CNTL);
5420 if (enable)
5421 tmp |= LOAD_BALANCE_ENABLE;
5422 else
5423 tmp &= ~LOAD_BALANCE_ENABLE;
5424 WREG32(RLC_LB_CNTL, tmp);
5425}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5426
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5427static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5428{
5429 u32 i, j, k;
5430 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5431
5432 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5433 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5434 cik_select_se_sh(rdev, i, j);
5435 for (k = 0; k < rdev->usec_timeout; k++) {
5436 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5437 break;
5438 udelay(1);
5439 }
5440 }
5441 }
5442 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5443
5444 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5445 for (k = 0; k < rdev->usec_timeout; k++) {
5446 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5447 break;
5448 udelay(1);
5449 }
5450}
5451
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5452static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5453{
5454 u32 tmp;
5455
5456 tmp = RREG32(RLC_CNTL);
5457 if (tmp != rlc)
5458 WREG32(RLC_CNTL, rlc);
5459}
5460
5461static u32 cik_halt_rlc(struct radeon_device *rdev)
5462{
5463 u32 data, orig;
5464
5465 orig = data = RREG32(RLC_CNTL);
5466
5467 if (data & RLC_ENABLE) {
5468 u32 i;
5469
5470 data &= ~RLC_ENABLE;
5471 WREG32(RLC_CNTL, data);
5472
5473 for (i = 0; i < rdev->usec_timeout; i++) {
5474 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5475 break;
5476 udelay(1);
5477 }
5478
5479 cik_wait_for_rlc_serdes(rdev);
5480 }
5481
5482 return orig;
5483}
5484
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5485void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5486{
5487 u32 tmp, i, mask;
5488
5489 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5490 WREG32(RLC_GPR_REG2, tmp);
5491
5492 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5493 for (i = 0; i < rdev->usec_timeout; i++) {
5494 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5495 break;
5496 udelay(1);
5497 }
5498
5499 for (i = 0; i < rdev->usec_timeout; i++) {
5500 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5501 break;
5502 udelay(1);
5503 }
5504}
5505
5506void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5507{
5508 u32 tmp;
5509
5510 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5511 WREG32(RLC_GPR_REG2, tmp);
5512}
5513
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5514/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5515 * cik_rlc_stop - stop the RLC ME
5516 *
5517 * @rdev: radeon_device pointer
5518 *
5519 * Halt the RLC ME (MicroEngine) (CIK).
5520 */
5521static void cik_rlc_stop(struct radeon_device *rdev)
5522{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5523 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5524
5525 cik_enable_gui_idle_interrupt(rdev, false);
5526
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5527 cik_wait_for_rlc_serdes(rdev);
5528}
5529
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5530/**
5531 * cik_rlc_start - start the RLC ME
5532 *
5533 * @rdev: radeon_device pointer
5534 *
5535 * Unhalt the RLC ME (MicroEngine) (CIK).
5536 */
5537static void cik_rlc_start(struct radeon_device *rdev)
5538{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5539 WREG32(RLC_CNTL, RLC_ENABLE);
5540
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5541 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5542
5543 udelay(50);
5544}
5545
5546/**
5547 * cik_rlc_resume - setup the RLC hw
5548 *
5549 * @rdev: radeon_device pointer
5550 *
5551 * Initialize the RLC registers, load the ucode,
5552 * and start the RLC (CIK).
5553 * Returns 0 for success, -EINVAL if the ucode is not available.
5554 */
5555static int cik_rlc_resume(struct radeon_device *rdev)
5556{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5557 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5558 const __be32 *fw_data;
5559
5560 if (!rdev->rlc_fw)
5561 return -EINVAL;
5562
5563 switch (rdev->family) {
5564 case CHIP_BONAIRE:
Alex Deucherd4775652013-08-08 16:06:35 -0400[diff] [blame]5565 case CHIP_HAWAII:
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5566 default:
5567 size = BONAIRE_RLC_UCODE_SIZE;
5568 break;
5569 case CHIP_KAVERI:
5570 size = KV_RLC_UCODE_SIZE;
5571 break;
5572 case CHIP_KABINI:
5573 size = KB_RLC_UCODE_SIZE;
5574 break;
5575 }
5576
5577 cik_rlc_stop(rdev);
5578
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5579 /* disable CG */
5580 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5581 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5582
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5583 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5584
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5585 cik_init_pg(rdev);
5586
5587 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5588
5589 WREG32(RLC_LB_CNTR_INIT, 0);
5590 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5591
5592 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5593 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5594 WREG32(RLC_LB_PARAMS, 0x00600408);
5595 WREG32(RLC_LB_CNTL, 0x80000004);
5596
5597 WREG32(RLC_MC_CNTL, 0);
5598 WREG32(RLC_UCODE_CNTL, 0);
5599
5600 fw_data = (const __be32 *)rdev->rlc_fw->data;
5601 WREG32(RLC_GPM_UCODE_ADDR, 0);
5602 for (i = 0; i < size; i++)
5603 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5604 WREG32(RLC_GPM_UCODE_ADDR, 0);
5605
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5606 /* XXX - find out what chips support lbpw */
5607 cik_enable_lbpw(rdev, false);
5608
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5609 if (rdev->family == CHIP_BONAIRE)
5610 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5611
5612 cik_rlc_start(rdev);
5613
5614 return 0;
5615}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5616
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5617static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5618{
5619 u32 data, orig, tmp, tmp2;
5620
5621 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5622
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5623 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5624 cik_enable_gui_idle_interrupt(rdev, true);
5625
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5626 tmp = cik_halt_rlc(rdev);
5627
5628 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5629 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5630 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5631 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5632 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5633
5634 cik_update_rlc(rdev, tmp);
5635
5636 data |= CGCG_EN | CGLS_EN;
5637 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5638 cik_enable_gui_idle_interrupt(rdev, false);
5639
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5640 RREG32(CB_CGTT_SCLK_CTRL);
5641 RREG32(CB_CGTT_SCLK_CTRL);
5642 RREG32(CB_CGTT_SCLK_CTRL);
5643 RREG32(CB_CGTT_SCLK_CTRL);
5644
5645 data &= ~(CGCG_EN | CGLS_EN);
5646 }
5647
5648 if (orig != data)
5649 WREG32(RLC_CGCG_CGLS_CTRL, data);
5650
5651}
5652
5653static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5654{
5655 u32 data, orig, tmp = 0;
5656
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5657 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5658 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5659 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5660 orig = data = RREG32(CP_MEM_SLP_CNTL);
5661 data |= CP_MEM_LS_EN;
5662 if (orig != data)
5663 WREG32(CP_MEM_SLP_CNTL, data);
5664 }
5665 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5666
5667 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5668 data &= 0xfffffffd;
5669 if (orig != data)
5670 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5671
5672 tmp = cik_halt_rlc(rdev);
5673
5674 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5675 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5676 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5677 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5678 WREG32(RLC_SERDES_WR_CTRL, data);
5679
5680 cik_update_rlc(rdev, tmp);
5681
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5682 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5683 orig = data = RREG32(CGTS_SM_CTRL_REG);
5684 data &= ~SM_MODE_MASK;
5685 data |= SM_MODE(0x2);
5686 data |= SM_MODE_ENABLE;
5687 data &= ~CGTS_OVERRIDE;
5688 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5689 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5690 data &= ~CGTS_LS_OVERRIDE;
5691 data &= ~ON_MONITOR_ADD_MASK;
5692 data |= ON_MONITOR_ADD_EN;
5693 data |= ON_MONITOR_ADD(0x96);
5694 if (orig != data)
5695 WREG32(CGTS_SM_CTRL_REG, data);
5696 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5697 } else {
5698 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5699 data |= 0x00000002;
5700 if (orig != data)
5701 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5702
5703 data = RREG32(RLC_MEM_SLP_CNTL);
5704 if (data & RLC_MEM_LS_EN) {
5705 data &= ~RLC_MEM_LS_EN;
5706 WREG32(RLC_MEM_SLP_CNTL, data);
5707 }
5708
5709 data = RREG32(CP_MEM_SLP_CNTL);
5710 if (data & CP_MEM_LS_EN) {
5711 data &= ~CP_MEM_LS_EN;
5712 WREG32(CP_MEM_SLP_CNTL, data);
5713 }
5714
5715 orig = data = RREG32(CGTS_SM_CTRL_REG);
5716 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5717 if (orig != data)
5718 WREG32(CGTS_SM_CTRL_REG, data);
5719
5720 tmp = cik_halt_rlc(rdev);
5721
5722 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5723 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5724 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5725 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5726 WREG32(RLC_SERDES_WR_CTRL, data);
5727
5728 cik_update_rlc(rdev, tmp);
5729 }
5730}
5731
5732static const u32 mc_cg_registers[] =
5733{
5734 MC_HUB_MISC_HUB_CG,
5735 MC_HUB_MISC_SIP_CG,
5736 MC_HUB_MISC_VM_CG,
5737 MC_XPB_CLK_GAT,
5738 ATC_MISC_CG,
5739 MC_CITF_MISC_WR_CG,
5740 MC_CITF_MISC_RD_CG,
5741 MC_CITF_MISC_VM_CG,
5742 VM_L2_CG,
5743};
5744
5745static void cik_enable_mc_ls(struct radeon_device *rdev,
5746 bool enable)
5747{
5748 int i;
5749 u32 orig, data;
5750
5751 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5752 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5753 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5754 data |= MC_LS_ENABLE;
5755 else
5756 data &= ~MC_LS_ENABLE;
5757 if (data != orig)
5758 WREG32(mc_cg_registers[i], data);
5759 }
5760}
5761
5762static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5763 bool enable)
5764{
5765 int i;
5766 u32 orig, data;
5767
5768 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5769 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5770 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5771 data |= MC_CG_ENABLE;
5772 else
5773 data &= ~MC_CG_ENABLE;
5774 if (data != orig)
5775 WREG32(mc_cg_registers[i], data);
5776 }
5777}
5778
5779static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5780 bool enable)
5781{
5782 u32 orig, data;
5783
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5784 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5785 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5786 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5787 } else {
5788 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5789 data |= 0xff000000;
5790 if (data != orig)
5791 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5792
5793 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5794 data |= 0xff000000;
5795 if (data != orig)
5796 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5797 }
5798}
5799
5800static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5801 bool enable)
5802{
5803 u32 orig, data;
5804
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5805 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5806 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5807 data |= 0x100;
5808 if (orig != data)
5809 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5810
5811 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5812 data |= 0x100;
5813 if (orig != data)
5814 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5815 } else {
5816 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5817 data &= ~0x100;
5818 if (orig != data)
5819 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5820
5821 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5822 data &= ~0x100;
5823 if (orig != data)
5824 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5825 }
5826}
5827
5828static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5829 bool enable)
5830{
5831 u32 orig, data;
5832
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5833 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5834 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5835 data = 0xfff;
5836 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5837
5838 orig = data = RREG32(UVD_CGC_CTRL);
5839 data |= DCM;
5840 if (orig != data)
5841 WREG32(UVD_CGC_CTRL, data);
5842 } else {
5843 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5844 data &= ~0xfff;
5845 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5846
5847 orig = data = RREG32(UVD_CGC_CTRL);
5848 data &= ~DCM;
5849 if (orig != data)
5850 WREG32(UVD_CGC_CTRL, data);
5851 }
5852}
5853
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5854static void cik_enable_bif_mgls(struct radeon_device *rdev,
5855 bool enable)
5856{
5857 u32 orig, data;
5858
5859 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5860
5861 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5862 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5863 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5864 else
5865 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5866 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5867
5868 if (orig != data)
5869 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5870}
5871
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5872static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5873 bool enable)
5874{
5875 u32 orig, data;
5876
5877 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5878
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5879 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5880 data &= ~CLOCK_GATING_DIS;
5881 else
5882 data |= CLOCK_GATING_DIS;
5883
5884 if (orig != data)
5885 WREG32(HDP_HOST_PATH_CNTL, data);
5886}
5887
5888static void cik_enable_hdp_ls(struct radeon_device *rdev,
5889 bool enable)
5890{
5891 u32 orig, data;
5892
5893 orig = data = RREG32(HDP_MEM_POWER_LS);
5894
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5895 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5896 data |= HDP_LS_ENABLE;
5897 else
5898 data &= ~HDP_LS_ENABLE;
5899
5900 if (orig != data)
5901 WREG32(HDP_MEM_POWER_LS, data);
5902}
5903
5904void cik_update_cg(struct radeon_device *rdev,
5905 u32 block, bool enable)
5906{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5907
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5908 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5909 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5910 /* order matters! */
5911 if (enable) {
5912 cik_enable_mgcg(rdev, true);
5913 cik_enable_cgcg(rdev, true);
5914 } else {
5915 cik_enable_cgcg(rdev, false);
5916 cik_enable_mgcg(rdev, false);
5917 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5918 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5919 }
5920
5921 if (block & RADEON_CG_BLOCK_MC) {
5922 if (!(rdev->flags & RADEON_IS_IGP)) {
5923 cik_enable_mc_mgcg(rdev, enable);
5924 cik_enable_mc_ls(rdev, enable);
5925 }
5926 }
5927
5928 if (block & RADEON_CG_BLOCK_SDMA) {
5929 cik_enable_sdma_mgcg(rdev, enable);
5930 cik_enable_sdma_mgls(rdev, enable);
5931 }
5932
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5933 if (block & RADEON_CG_BLOCK_BIF) {
5934 cik_enable_bif_mgls(rdev, enable);
5935 }
5936
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5937 if (block & RADEON_CG_BLOCK_UVD) {
5938 if (rdev->has_uvd)
5939 cik_enable_uvd_mgcg(rdev, enable);
5940 }
5941
5942 if (block & RADEON_CG_BLOCK_HDP) {
5943 cik_enable_hdp_mgcg(rdev, enable);
5944 cik_enable_hdp_ls(rdev, enable);
5945 }
5946}
5947
5948static void cik_init_cg(struct radeon_device *rdev)
5949{
5950
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5951 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5952
5953 if (rdev->has_uvd)
5954 si_init_uvd_internal_cg(rdev);
5955
5956 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5957 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5958 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5959 RADEON_CG_BLOCK_UVD |
5960 RADEON_CG_BLOCK_HDP), true);
5961}
5962
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5963static void cik_fini_cg(struct radeon_device *rdev)
5964{
5965 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5966 RADEON_CG_BLOCK_SDMA |
5967 RADEON_CG_BLOCK_BIF |
5968 RADEON_CG_BLOCK_UVD |
5969 RADEON_CG_BLOCK_HDP), false);
5970
5971 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5972}
5973
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5974static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5975 bool enable)
5976{
5977 u32 data, orig;
5978
5979 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5980 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5981 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5982 else
5983 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5984 if (orig != data)
5985 WREG32(RLC_PG_CNTL, data);
5986}
5987
5988static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5989 bool enable)
5990{
5991 u32 data, orig;
5992
5993 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5994 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5995 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5996 else
5997 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5998 if (orig != data)
5999 WREG32(RLC_PG_CNTL, data);
6000}
6001
6002static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6003{
6004 u32 data, orig;
6005
6006 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6007 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6008 data &= ~DISABLE_CP_PG;
6009 else
6010 data |= DISABLE_CP_PG;
6011 if (orig != data)
6012 WREG32(RLC_PG_CNTL, data);
6013}
6014
6015static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6016{
6017 u32 data, orig;
6018
6019 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6020 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6021 data &= ~DISABLE_GDS_PG;
6022 else
6023 data |= DISABLE_GDS_PG;
6024 if (orig != data)
6025 WREG32(RLC_PG_CNTL, data);
6026}
6027
6028#define CP_ME_TABLE_SIZE 96
6029#define CP_ME_TABLE_OFFSET 2048
6030#define CP_MEC_TABLE_OFFSET 4096
6031
6032void cik_init_cp_pg_table(struct radeon_device *rdev)
6033{
6034 const __be32 *fw_data;
6035 volatile u32 *dst_ptr;
6036 int me, i, max_me = 4;
6037 u32 bo_offset = 0;
6038 u32 table_offset;
6039
6040 if (rdev->family == CHIP_KAVERI)
6041 max_me = 5;
6042
6043 if (rdev->rlc.cp_table_ptr == NULL)
6044 return;
6045
6046 /* write the cp table buffer */
6047 dst_ptr = rdev->rlc.cp_table_ptr;
6048 for (me = 0; me < max_me; me++) {
6049 if (me == 0) {
6050 fw_data = (const __be32 *)rdev->ce_fw->data;
6051 table_offset = CP_ME_TABLE_OFFSET;
6052 } else if (me == 1) {
6053 fw_data = (const __be32 *)rdev->pfp_fw->data;
6054 table_offset = CP_ME_TABLE_OFFSET;
6055 } else if (me == 2) {
6056 fw_data = (const __be32 *)rdev->me_fw->data;
6057 table_offset = CP_ME_TABLE_OFFSET;
6058 } else {
6059 fw_data = (const __be32 *)rdev->mec_fw->data;
6060 table_offset = CP_MEC_TABLE_OFFSET;
6061 }
6062
6063 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6064 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6065 }
6066 bo_offset += CP_ME_TABLE_SIZE;
6067 }
6068}
6069
6070static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6071 bool enable)
6072{
6073 u32 data, orig;
6074
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6075 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6076 orig = data = RREG32(RLC_PG_CNTL);
6077 data |= GFX_PG_ENABLE;
6078 if (orig != data)
6079 WREG32(RLC_PG_CNTL, data);
6080
6081 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6082 data |= AUTO_PG_EN;
6083 if (orig != data)
6084 WREG32(RLC_AUTO_PG_CTRL, data);
6085 } else {
6086 orig = data = RREG32(RLC_PG_CNTL);
6087 data &= ~GFX_PG_ENABLE;
6088 if (orig != data)
6089 WREG32(RLC_PG_CNTL, data);
6090
6091 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6092 data &= ~AUTO_PG_EN;
6093 if (orig != data)
6094 WREG32(RLC_AUTO_PG_CTRL, data);
6095
6096 data = RREG32(DB_RENDER_CONTROL);
6097 }
6098}
6099
6100static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6101{
6102 u32 mask = 0, tmp, tmp1;
6103 int i;
6104
6105 cik_select_se_sh(rdev, se, sh);
6106 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6107 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6109
6110 tmp &= 0xffff0000;
6111
6112 tmp |= tmp1;
6113 tmp >>= 16;
6114
6115 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6116 mask <<= 1;
6117 mask |= 1;
6118 }
6119
6120 return (~tmp) & mask;
6121}
6122
6123static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6124{
6125 u32 i, j, k, active_cu_number = 0;
6126 u32 mask, counter, cu_bitmap;
6127 u32 tmp = 0;
6128
6129 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6130 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6131 mask = 1;
6132 cu_bitmap = 0;
6133 counter = 0;
6134 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6135 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6136 if (counter < 2)
6137 cu_bitmap |= mask;
6138 counter ++;
6139 }
6140 mask <<= 1;
6141 }
6142
6143 active_cu_number += counter;
6144 tmp |= (cu_bitmap << (i * 16 + j * 8));
6145 }
6146 }
6147
6148 WREG32(RLC_PG_AO_CU_MASK, tmp);
6149
6150 tmp = RREG32(RLC_MAX_PG_CU);
6151 tmp &= ~MAX_PU_CU_MASK;
6152 tmp |= MAX_PU_CU(active_cu_number);
6153 WREG32(RLC_MAX_PG_CU, tmp);
6154}
6155
6156static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6157 bool enable)
6158{
6159 u32 data, orig;
6160
6161 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6162 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6163 data |= STATIC_PER_CU_PG_ENABLE;
6164 else
6165 data &= ~STATIC_PER_CU_PG_ENABLE;
6166 if (orig != data)
6167 WREG32(RLC_PG_CNTL, data);
6168}
6169
6170static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6171 bool enable)
6172{
6173 u32 data, orig;
6174
6175 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6176 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6177 data |= DYN_PER_CU_PG_ENABLE;
6178 else
6179 data &= ~DYN_PER_CU_PG_ENABLE;
6180 if (orig != data)
6181 WREG32(RLC_PG_CNTL, data);
6182}
6183
6184#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6185#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6186
6187static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6188{
6189 u32 data, orig;
6190 u32 i;
6191
6192 if (rdev->rlc.cs_data) {
6193 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6194 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6195 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6196 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6197 } else {
6198 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6199 for (i = 0; i < 3; i++)
6200 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6201 }
6202 if (rdev->rlc.reg_list) {
6203 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6204 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6205 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6206 }
6207
6208 orig = data = RREG32(RLC_PG_CNTL);
6209 data |= GFX_PG_SRC;
6210 if (orig != data)
6211 WREG32(RLC_PG_CNTL, data);
6212
6213 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6214 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6215
6216 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6217 data &= ~IDLE_POLL_COUNT_MASK;
6218 data |= IDLE_POLL_COUNT(0x60);
6219 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6220
6221 data = 0x10101010;
6222 WREG32(RLC_PG_DELAY, data);
6223
6224 data = RREG32(RLC_PG_DELAY_2);
6225 data &= ~0xff;
6226 data |= 0x3;
6227 WREG32(RLC_PG_DELAY_2, data);
6228
6229 data = RREG32(RLC_AUTO_PG_CTRL);
6230 data &= ~GRBM_REG_SGIT_MASK;
6231 data |= GRBM_REG_SGIT(0x700);
6232 WREG32(RLC_AUTO_PG_CTRL, data);
6233
6234}
6235
6236static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6237{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6238 cik_enable_gfx_cgpg(rdev, enable);
6239 cik_enable_gfx_static_mgpg(rdev, enable);
6240 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6241}
6242
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6243u32 cik_get_csb_size(struct radeon_device *rdev)
6244{
6245 u32 count = 0;
6246 const struct cs_section_def *sect = NULL;
6247 const struct cs_extent_def *ext = NULL;
6248
6249 if (rdev->rlc.cs_data == NULL)
6250 return 0;
6251
6252 /* begin clear state */
6253 count += 2;
6254 /* context control state */
6255 count += 3;
6256
6257 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6258 for (ext = sect->section; ext->extent != NULL; ++ext) {
6259 if (sect->id == SECT_CONTEXT)
6260 count += 2 + ext->reg_count;
6261 else
6262 return 0;
6263 }
6264 }
6265 /* pa_sc_raster_config/pa_sc_raster_config1 */
6266 count += 4;
6267 /* end clear state */
6268 count += 2;
6269 /* clear state */
6270 count += 2;
6271
6272 return count;
6273}
6274
6275void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6276{
6277 u32 count = 0, i;
6278 const struct cs_section_def *sect = NULL;
6279 const struct cs_extent_def *ext = NULL;
6280
6281 if (rdev->rlc.cs_data == NULL)
6282 return;
6283 if (buffer == NULL)
6284 return;
6285
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6286 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6287 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6288
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6289 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6290 buffer[count++] = cpu_to_le32(0x80000000);
6291 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6292
6293 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6294 for (ext = sect->section; ext->extent != NULL; ++ext) {
6295 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6296 buffer[count++] =
6297 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6298 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6299 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6300 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6301 } else {
6302 return;
6303 }
6304 }
6305 }
6306
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6307 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6308 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6309 switch (rdev->family) {
6310 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6311 buffer[count++] = cpu_to_le32(0x16000012);
6312 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6313 break;
6314 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6315 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6316 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6317 break;
6318 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6319 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6320 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6321 break;
Alex Deucherbbfe90b2013-08-13 22:59:41 -0400[diff] [blame]6322 case CHIP_HAWAII:
6323 buffer[count++] = 0x3a00161a;
6324 buffer[count++] = 0x0000002e;
6325 break;
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6326 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6327 buffer[count++] = cpu_to_le32(0x00000000);
6328 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6329 break;
6330 }
6331
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6332 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6333 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6334
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6335 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6336 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6337}
6338
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6339static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6340{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6341 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6342 cik_enable_sck_slowdown_on_pu(rdev, true);
6343 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6344 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6345 cik_init_gfx_cgpg(rdev);
6346 cik_enable_cp_pg(rdev, true);
6347 cik_enable_gds_pg(rdev, true);
6348 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6349 cik_init_ao_cu_mask(rdev);
6350 cik_update_gfx_pg(rdev, true);
6351 }
6352}
6353
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6354static void cik_fini_pg(struct radeon_device *rdev)
6355{
6356 if (rdev->pg_flags) {
6357 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6358 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6359 cik_enable_cp_pg(rdev, false);
6360 cik_enable_gds_pg(rdev, false);
6361 }
6362 }
6363}
6364
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6365/*
6366 * Interrupts
6367 * Starting with r6xx, interrupts are handled via a ring buffer.
6368 * Ring buffers are areas of GPU accessible memory that the GPU
6369 * writes interrupt vectors into and the host reads vectors out of.
6370 * There is a rptr (read pointer) that determines where the
6371 * host is currently reading, and a wptr (write pointer)
6372 * which determines where the GPU has written. When the
6373 * pointers are equal, the ring is idle. When the GPU
6374 * writes vectors to the ring buffer, it increments the
6375 * wptr. When there is an interrupt, the host then starts
6376 * fetching commands and processing them until the pointers are
6377 * equal again at which point it updates the rptr.
6378 */
6379
6380/**
6381 * cik_enable_interrupts - Enable the interrupt ring buffer
6382 *
6383 * @rdev: radeon_device pointer
6384 *
6385 * Enable the interrupt ring buffer (CIK).
6386 */
6387static void cik_enable_interrupts(struct radeon_device *rdev)
6388{
6389 u32 ih_cntl = RREG32(IH_CNTL);
6390 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6391
6392 ih_cntl |= ENABLE_INTR;
6393 ih_rb_cntl |= IH_RB_ENABLE;
6394 WREG32(IH_CNTL, ih_cntl);
6395 WREG32(IH_RB_CNTL, ih_rb_cntl);
6396 rdev->ih.enabled = true;
6397}
6398
6399/**
6400 * cik_disable_interrupts - Disable the interrupt ring buffer
6401 *
6402 * @rdev: radeon_device pointer
6403 *
6404 * Disable the interrupt ring buffer (CIK).
6405 */
6406static void cik_disable_interrupts(struct radeon_device *rdev)
6407{
6408 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6409 u32 ih_cntl = RREG32(IH_CNTL);
6410
6411 ih_rb_cntl &= ~IH_RB_ENABLE;
6412 ih_cntl &= ~ENABLE_INTR;
6413 WREG32(IH_RB_CNTL, ih_rb_cntl);
6414 WREG32(IH_CNTL, ih_cntl);
6415 /* set rptr, wptr to 0 */
6416 WREG32(IH_RB_RPTR, 0);
6417 WREG32(IH_RB_WPTR, 0);
6418 rdev->ih.enabled = false;
6419 rdev->ih.rptr = 0;
6420}
6421
6422/**
6423 * cik_disable_interrupt_state - Disable all interrupt sources
6424 *
6425 * @rdev: radeon_device pointer
6426 *
6427 * Clear all interrupt enable bits used by the driver (CIK).
6428 */
6429static void cik_disable_interrupt_state(struct radeon_device *rdev)
6430{
6431 u32 tmp;
6432
6433 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6434 tmp = RREG32(CP_INT_CNTL_RING0) &
6435 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6436 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6437 /* sdma */
6438 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6439 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6440 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6441 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6442 /* compute queues */
6443 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6444 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6445 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6446 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6447 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6448 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6449 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6450 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6451 /* grbm */
6452 WREG32(GRBM_INT_CNTL, 0);
6453 /* vline/vblank, etc. */
6454 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6455 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6456 if (rdev->num_crtc >= 4) {
6457 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6458 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6459 }
6460 if (rdev->num_crtc >= 6) {
6461 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6462 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6463 }
6464
6465 /* dac hotplug */
6466 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6467
6468 /* digital hotplug */
6469 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6470 WREG32(DC_HPD1_INT_CONTROL, tmp);
6471 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6472 WREG32(DC_HPD2_INT_CONTROL, tmp);
6473 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6474 WREG32(DC_HPD3_INT_CONTROL, tmp);
6475 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6476 WREG32(DC_HPD4_INT_CONTROL, tmp);
6477 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6478 WREG32(DC_HPD5_INT_CONTROL, tmp);
6479 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6480 WREG32(DC_HPD6_INT_CONTROL, tmp);
6481
6482}
6483
6484/**
6485 * cik_irq_init - init and enable the interrupt ring
6486 *
6487 * @rdev: radeon_device pointer
6488 *
6489 * Allocate a ring buffer for the interrupt controller,
6490 * enable the RLC, disable interrupts, enable the IH
6491 * ring buffer and enable it (CIK).
6492 * Called at device load and reume.
6493 * Returns 0 for success, errors for failure.
6494 */
6495static int cik_irq_init(struct radeon_device *rdev)
6496{
6497 int ret = 0;
6498 int rb_bufsz;
6499 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6500
6501 /* allocate ring */
6502 ret = r600_ih_ring_alloc(rdev);
6503 if (ret)
6504 return ret;
6505
6506 /* disable irqs */
6507 cik_disable_interrupts(rdev);
6508
6509 /* init rlc */
6510 ret = cik_rlc_resume(rdev);
6511 if (ret) {
6512 r600_ih_ring_fini(rdev);
6513 return ret;
6514 }
6515
6516 /* setup interrupt control */
6517 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6518 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6519 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6520 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6521 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6522 */
6523 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6524 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6525 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6526 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6527
6528 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6529 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6530
6531 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6532 IH_WPTR_OVERFLOW_CLEAR |
6533 (rb_bufsz << 1));
6534
6535 if (rdev->wb.enabled)
6536 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6537
6538 /* set the writeback address whether it's enabled or not */
6539 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6540 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6541
6542 WREG32(IH_RB_CNTL, ih_rb_cntl);
6543
6544 /* set rptr, wptr to 0 */
6545 WREG32(IH_RB_RPTR, 0);
6546 WREG32(IH_RB_WPTR, 0);
6547
6548 /* Default settings for IH_CNTL (disabled at first) */
6549 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6550 /* RPTR_REARM only works if msi's are enabled */
6551 if (rdev->msi_enabled)
6552 ih_cntl |= RPTR_REARM;
6553 WREG32(IH_CNTL, ih_cntl);
6554
6555 /* force the active interrupt state to all disabled */
6556 cik_disable_interrupt_state(rdev);
6557
6558 pci_set_master(rdev->pdev);
6559
6560 /* enable irqs */
6561 cik_enable_interrupts(rdev);
6562
6563 return ret;
6564}
6565
6566/**
6567 * cik_irq_set - enable/disable interrupt sources
6568 *
6569 * @rdev: radeon_device pointer
6570 *
6571 * Enable interrupt sources on the GPU (vblanks, hpd,
6572 * etc.) (CIK).
6573 * Returns 0 for success, errors for failure.
6574 */
6575int cik_irq_set(struct radeon_device *rdev)
6576{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6577 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6578 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6579 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6580 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6581 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6582 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6583 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6584 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6585
6586 if (!rdev->irq.installed) {
6587 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6588 return -EINVAL;
6589 }
6590 /* don't enable anything if the ih is disabled */
6591 if (!rdev->ih.enabled) {
6592 cik_disable_interrupts(rdev);
6593 /* force the active interrupt state to all disabled */
6594 cik_disable_interrupt_state(rdev);
6595 return 0;
6596 }
6597
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6598 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6599 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6600 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6601
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6602 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6604 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6605 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6606 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6607 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6608
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6609 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6610 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6611
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6612 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6616 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6617 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6618 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6619 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6620
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6621 if (rdev->flags & RADEON_IS_IGP)
6622 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6623 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6624 else
6625 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6626 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6627
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6628 /* enable CP interrupts on all rings */
6629 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6630 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6631 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6632 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6633 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6634 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6635 DRM_DEBUG("si_irq_set: sw int cp1\n");
6636 if (ring->me == 1) {
6637 switch (ring->pipe) {
6638 case 0:
6639 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6640 break;
6641 case 1:
6642 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6643 break;
6644 case 2:
6645 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6646 break;
6647 case 3:
6648 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6649 break;
6650 default:
6651 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6652 break;
6653 }
6654 } else if (ring->me == 2) {
6655 switch (ring->pipe) {
6656 case 0:
6657 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6658 break;
6659 case 1:
6660 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6661 break;
6662 case 2:
6663 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6664 break;
6665 case 3:
6666 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6667 break;
6668 default:
6669 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6670 break;
6671 }
6672 } else {
6673 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6674 }
6675 }
6676 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6677 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6678 DRM_DEBUG("si_irq_set: sw int cp2\n");
6679 if (ring->me == 1) {
6680 switch (ring->pipe) {
6681 case 0:
6682 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6683 break;
6684 case 1:
6685 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6686 break;
6687 case 2:
6688 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6689 break;
6690 case 3:
6691 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6692 break;
6693 default:
6694 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6695 break;
6696 }
6697 } else if (ring->me == 2) {
6698 switch (ring->pipe) {
6699 case 0:
6700 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6701 break;
6702 case 1:
6703 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6704 break;
6705 case 2:
6706 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6707 break;
6708 case 3:
6709 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6710 break;
6711 default:
6712 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6713 break;
6714 }
6715 } else {
6716 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6717 }
6718 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6719
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6720 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6721 DRM_DEBUG("cik_irq_set: sw int dma\n");
6722 dma_cntl |= TRAP_ENABLE;
6723 }
6724
6725 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6726 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6727 dma_cntl1 |= TRAP_ENABLE;
6728 }
6729
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6730 if (rdev->irq.crtc_vblank_int[0] ||
6731 atomic_read(&rdev->irq.pflip[0])) {
6732 DRM_DEBUG("cik_irq_set: vblank 0\n");
6733 crtc1 |= VBLANK_INTERRUPT_MASK;
6734 }
6735 if (rdev->irq.crtc_vblank_int[1] ||
6736 atomic_read(&rdev->irq.pflip[1])) {
6737 DRM_DEBUG("cik_irq_set: vblank 1\n");
6738 crtc2 |= VBLANK_INTERRUPT_MASK;
6739 }
6740 if (rdev->irq.crtc_vblank_int[2] ||
6741 atomic_read(&rdev->irq.pflip[2])) {
6742 DRM_DEBUG("cik_irq_set: vblank 2\n");
6743 crtc3 |= VBLANK_INTERRUPT_MASK;
6744 }
6745 if (rdev->irq.crtc_vblank_int[3] ||
6746 atomic_read(&rdev->irq.pflip[3])) {
6747 DRM_DEBUG("cik_irq_set: vblank 3\n");
6748 crtc4 |= VBLANK_INTERRUPT_MASK;
6749 }
6750 if (rdev->irq.crtc_vblank_int[4] ||
6751 atomic_read(&rdev->irq.pflip[4])) {
6752 DRM_DEBUG("cik_irq_set: vblank 4\n");
6753 crtc5 |= VBLANK_INTERRUPT_MASK;
6754 }
6755 if (rdev->irq.crtc_vblank_int[5] ||
6756 atomic_read(&rdev->irq.pflip[5])) {
6757 DRM_DEBUG("cik_irq_set: vblank 5\n");
6758 crtc6 |= VBLANK_INTERRUPT_MASK;
6759 }
6760 if (rdev->irq.hpd[0]) {
6761 DRM_DEBUG("cik_irq_set: hpd 1\n");
6762 hpd1 |= DC_HPDx_INT_EN;
6763 }
6764 if (rdev->irq.hpd[1]) {
6765 DRM_DEBUG("cik_irq_set: hpd 2\n");
6766 hpd2 |= DC_HPDx_INT_EN;
6767 }
6768 if (rdev->irq.hpd[2]) {
6769 DRM_DEBUG("cik_irq_set: hpd 3\n");
6770 hpd3 |= DC_HPDx_INT_EN;
6771 }
6772 if (rdev->irq.hpd[3]) {
6773 DRM_DEBUG("cik_irq_set: hpd 4\n");
6774 hpd4 |= DC_HPDx_INT_EN;
6775 }
6776 if (rdev->irq.hpd[4]) {
6777 DRM_DEBUG("cik_irq_set: hpd 5\n");
6778 hpd5 |= DC_HPDx_INT_EN;
6779 }
6780 if (rdev->irq.hpd[5]) {
6781 DRM_DEBUG("cik_irq_set: hpd 6\n");
6782 hpd6 |= DC_HPDx_INT_EN;
6783 }
6784
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6785 if (rdev->irq.dpm_thermal) {
6786 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6787 if (rdev->flags & RADEON_IS_IGP)
6788 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6789 else
6790 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6791 }
6792
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6793 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6794
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6795 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6796 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6797
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6798 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6799 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6800 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6801 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6802 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6803 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6804 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6805 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6806
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6807 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6808
6809 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6810 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6811 if (rdev->num_crtc >= 4) {
6812 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6813 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6814 }
6815 if (rdev->num_crtc >= 6) {
6816 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6817 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6818 }
6819
6820 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6821 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6822 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6823 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6824 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6825 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6826
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6827 if (rdev->flags & RADEON_IS_IGP)
6828 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6829 else
6830 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6831
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6832 return 0;
6833}
6834
6835/**
6836 * cik_irq_ack - ack interrupt sources
6837 *
6838 * @rdev: radeon_device pointer
6839 *
6840 * Ack interrupt sources on the GPU (vblanks, hpd,
6841 * etc.) (CIK). Certain interrupts sources are sw
6842 * generated and do not require an explicit ack.
6843 */
6844static inline void cik_irq_ack(struct radeon_device *rdev)
6845{
6846 u32 tmp;
6847
6848 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6849 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6850 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6851 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6852 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6853 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6854 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6855
6856 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6857 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6858 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6859 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6860 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6861 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6862 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6863 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6864
6865 if (rdev->num_crtc >= 4) {
6866 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6867 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6868 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6869 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6870 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6871 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6872 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6873 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6874 }
6875
6876 if (rdev->num_crtc >= 6) {
6877 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6878 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6879 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6880 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6881 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6882 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6883 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6884 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6885 }
6886
6887 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6888 tmp = RREG32(DC_HPD1_INT_CONTROL);
6889 tmp |= DC_HPDx_INT_ACK;
6890 WREG32(DC_HPD1_INT_CONTROL, tmp);
6891 }
6892 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6893 tmp = RREG32(DC_HPD2_INT_CONTROL);
6894 tmp |= DC_HPDx_INT_ACK;
6895 WREG32(DC_HPD2_INT_CONTROL, tmp);
6896 }
6897 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6898 tmp = RREG32(DC_HPD3_INT_CONTROL);
6899 tmp |= DC_HPDx_INT_ACK;
6900 WREG32(DC_HPD3_INT_CONTROL, tmp);
6901 }
6902 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6903 tmp = RREG32(DC_HPD4_INT_CONTROL);
6904 tmp |= DC_HPDx_INT_ACK;
6905 WREG32(DC_HPD4_INT_CONTROL, tmp);
6906 }
6907 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6908 tmp = RREG32(DC_HPD5_INT_CONTROL);
6909 tmp |= DC_HPDx_INT_ACK;
6910 WREG32(DC_HPD5_INT_CONTROL, tmp);
6911 }
6912 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6913 tmp = RREG32(DC_HPD5_INT_CONTROL);
6914 tmp |= DC_HPDx_INT_ACK;
6915 WREG32(DC_HPD6_INT_CONTROL, tmp);
6916 }
6917}
6918
6919/**
6920 * cik_irq_disable - disable interrupts
6921 *
6922 * @rdev: radeon_device pointer
6923 *
6924 * Disable interrupts on the hw (CIK).
6925 */
6926static void cik_irq_disable(struct radeon_device *rdev)
6927{
6928 cik_disable_interrupts(rdev);
6929 /* Wait and acknowledge irq */
6930 mdelay(1);
6931 cik_irq_ack(rdev);
6932 cik_disable_interrupt_state(rdev);
6933}
6934
6935/**
6936 * cik_irq_disable - disable interrupts for suspend
6937 *
6938 * @rdev: radeon_device pointer
6939 *
6940 * Disable interrupts and stop the RLC (CIK).
6941 * Used for suspend.
6942 */
6943static void cik_irq_suspend(struct radeon_device *rdev)
6944{
6945 cik_irq_disable(rdev);
6946 cik_rlc_stop(rdev);
6947}
6948
6949/**
6950 * cik_irq_fini - tear down interrupt support
6951 *
6952 * @rdev: radeon_device pointer
6953 *
6954 * Disable interrupts on the hw and free the IH ring
6955 * buffer (CIK).
6956 * Used for driver unload.
6957 */
6958static void cik_irq_fini(struct radeon_device *rdev)
6959{
6960 cik_irq_suspend(rdev);
6961 r600_ih_ring_fini(rdev);
6962}
6963
6964/**
6965 * cik_get_ih_wptr - get the IH ring buffer wptr
6966 *
6967 * @rdev: radeon_device pointer
6968 *
6969 * Get the IH ring buffer wptr from either the register
6970 * or the writeback memory buffer (CIK). Also check for
6971 * ring buffer overflow and deal with it.
6972 * Used by cik_irq_process().
6973 * Returns the value of the wptr.
6974 */
6975static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6976{
6977 u32 wptr, tmp;
6978
6979 if (rdev->wb.enabled)
6980 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6981 else
6982 wptr = RREG32(IH_RB_WPTR);
6983
6984 if (wptr & RB_OVERFLOW) {
6985 /* When a ring buffer overflow happen start parsing interrupt
6986 * from the last not overwritten vector (wptr + 16). Hopefully
6987 * this should allow us to catchup.
6988 */
6989 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6990 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6991 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6992 tmp = RREG32(IH_RB_CNTL);
6993 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6994 WREG32(IH_RB_CNTL, tmp);
6995 }
6996 return (wptr & rdev->ih.ptr_mask);
6997}
6998
6999/* CIK IV Ring
7000 * Each IV ring entry is 128 bits:
7001 * [7:0] - interrupt source id
7002 * [31:8] - reserved
7003 * [59:32] - interrupt source data
7004 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7005 * [71:64] - RINGID
7006 * CP:
7007 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7008 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7009 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7010 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7011 * PIPE_ID - ME0 0=3D
7012 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7013 * SDMA:
7014 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7015 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7016 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7017 * [79:72] - VMID
7018 * [95:80] - PASID
7019 * [127:96] - reserved
7020 */
7021/**
7022 * cik_irq_process - interrupt handler
7023 *
7024 * @rdev: radeon_device pointer
7025 *
7026 * Interrupt hander (CIK). Walk the IH ring,
7027 * ack interrupts and schedule work to handle
7028 * interrupt events.
7029 * Returns irq process return code.
7030 */
7031int cik_irq_process(struct radeon_device *rdev)
7032{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7033 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7034 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7035 u32 wptr;
7036 u32 rptr;
7037 u32 src_id, src_data, ring_id;
7038 u8 me_id, pipe_id, queue_id;
7039 u32 ring_index;
7040 bool queue_hotplug = false;
7041 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7042 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7043 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7044
7045 if (!rdev->ih.enabled || rdev->shutdown)
7046 return IRQ_NONE;
7047
7048 wptr = cik_get_ih_wptr(rdev);
7049
7050restart_ih:
7051 /* is somebody else already processing irqs? */
7052 if (atomic_xchg(&rdev->ih.lock, 1))
7053 return IRQ_NONE;
7054
7055 rptr = rdev->ih.rptr;
7056 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7057
7058 /* Order reading of wptr vs. reading of IH ring data */
7059 rmb();
7060
7061 /* display interrupts */
7062 cik_irq_ack(rdev);
7063
7064 while (rptr != wptr) {
7065 /* wptr/rptr are in bytes! */
7066 ring_index = rptr / 4;
7067 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7068 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7069 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7070
7071 switch (src_id) {
7072 case 1: /* D1 vblank/vline */
7073 switch (src_data) {
7074 case 0: /* D1 vblank */
7075 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7076 if (rdev->irq.crtc_vblank_int[0]) {
7077 drm_handle_vblank(rdev->ddev, 0);
7078 rdev->pm.vblank_sync = true;
7079 wake_up(&rdev->irq.vblank_queue);
7080 }
7081 if (atomic_read(&rdev->irq.pflip[0]))
7082 radeon_crtc_handle_flip(rdev, 0);
7083 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7084 DRM_DEBUG("IH: D1 vblank\n");
7085 }
7086 break;
7087 case 1: /* D1 vline */
7088 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7089 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7090 DRM_DEBUG("IH: D1 vline\n");
7091 }
7092 break;
7093 default:
7094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7095 break;
7096 }
7097 break;
7098 case 2: /* D2 vblank/vline */
7099 switch (src_data) {
7100 case 0: /* D2 vblank */
7101 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7102 if (rdev->irq.crtc_vblank_int[1]) {
7103 drm_handle_vblank(rdev->ddev, 1);
7104 rdev->pm.vblank_sync = true;
7105 wake_up(&rdev->irq.vblank_queue);
7106 }
7107 if (atomic_read(&rdev->irq.pflip[1]))
7108 radeon_crtc_handle_flip(rdev, 1);
7109 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7110 DRM_DEBUG("IH: D2 vblank\n");
7111 }
7112 break;
7113 case 1: /* D2 vline */
7114 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7115 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7116 DRM_DEBUG("IH: D2 vline\n");
7117 }
7118 break;
7119 default:
7120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7121 break;
7122 }
7123 break;
7124 case 3: /* D3 vblank/vline */
7125 switch (src_data) {
7126 case 0: /* D3 vblank */
7127 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7128 if (rdev->irq.crtc_vblank_int[2]) {
7129 drm_handle_vblank(rdev->ddev, 2);
7130 rdev->pm.vblank_sync = true;
7131 wake_up(&rdev->irq.vblank_queue);
7132 }
7133 if (atomic_read(&rdev->irq.pflip[2]))
7134 radeon_crtc_handle_flip(rdev, 2);
7135 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7136 DRM_DEBUG("IH: D3 vblank\n");
7137 }
7138 break;
7139 case 1: /* D3 vline */
7140 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7141 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7142 DRM_DEBUG("IH: D3 vline\n");
7143 }
7144 break;
7145 default:
7146 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7147 break;
7148 }
7149 break;
7150 case 4: /* D4 vblank/vline */
7151 switch (src_data) {
7152 case 0: /* D4 vblank */
7153 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7154 if (rdev->irq.crtc_vblank_int[3]) {
7155 drm_handle_vblank(rdev->ddev, 3);
7156 rdev->pm.vblank_sync = true;
7157 wake_up(&rdev->irq.vblank_queue);
7158 }
7159 if (atomic_read(&rdev->irq.pflip[3]))
7160 radeon_crtc_handle_flip(rdev, 3);
7161 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7162 DRM_DEBUG("IH: D4 vblank\n");
7163 }
7164 break;
7165 case 1: /* D4 vline */
7166 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7167 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7168 DRM_DEBUG("IH: D4 vline\n");
7169 }
7170 break;
7171 default:
7172 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7173 break;
7174 }
7175 break;
7176 case 5: /* D5 vblank/vline */
7177 switch (src_data) {
7178 case 0: /* D5 vblank */
7179 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7180 if (rdev->irq.crtc_vblank_int[4]) {
7181 drm_handle_vblank(rdev->ddev, 4);
7182 rdev->pm.vblank_sync = true;
7183 wake_up(&rdev->irq.vblank_queue);
7184 }
7185 if (atomic_read(&rdev->irq.pflip[4]))
7186 radeon_crtc_handle_flip(rdev, 4);
7187 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7188 DRM_DEBUG("IH: D5 vblank\n");
7189 }
7190 break;
7191 case 1: /* D5 vline */
7192 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7193 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7194 DRM_DEBUG("IH: D5 vline\n");
7195 }
7196 break;
7197 default:
7198 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7199 break;
7200 }
7201 break;
7202 case 6: /* D6 vblank/vline */
7203 switch (src_data) {
7204 case 0: /* D6 vblank */
7205 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7206 if (rdev->irq.crtc_vblank_int[5]) {
7207 drm_handle_vblank(rdev->ddev, 5);
7208 rdev->pm.vblank_sync = true;
7209 wake_up(&rdev->irq.vblank_queue);
7210 }
7211 if (atomic_read(&rdev->irq.pflip[5]))
7212 radeon_crtc_handle_flip(rdev, 5);
7213 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7214 DRM_DEBUG("IH: D6 vblank\n");
7215 }
7216 break;
7217 case 1: /* D6 vline */
7218 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7219 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7220 DRM_DEBUG("IH: D6 vline\n");
7221 }
7222 break;
7223 default:
7224 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7225 break;
7226 }
7227 break;
7228 case 42: /* HPD hotplug */
7229 switch (src_data) {
7230 case 0:
7231 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7232 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7233 queue_hotplug = true;
7234 DRM_DEBUG("IH: HPD1\n");
7235 }
7236 break;
7237 case 1:
7238 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7239 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7240 queue_hotplug = true;
7241 DRM_DEBUG("IH: HPD2\n");
7242 }
7243 break;
7244 case 2:
7245 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7246 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7247 queue_hotplug = true;
7248 DRM_DEBUG("IH: HPD3\n");
7249 }
7250 break;
7251 case 3:
7252 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7253 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7254 queue_hotplug = true;
7255 DRM_DEBUG("IH: HPD4\n");
7256 }
7257 break;
7258 case 4:
7259 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7260 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7261 queue_hotplug = true;
7262 DRM_DEBUG("IH: HPD5\n");
7263 }
7264 break;
7265 case 5:
7266 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7267 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7268 queue_hotplug = true;
7269 DRM_DEBUG("IH: HPD6\n");
7270 }
7271 break;
7272 default:
7273 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7274 break;
7275 }
7276 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7277 case 124: /* UVD */
7278 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7279 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7280 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7281 case 146:
7282 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7283 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7284 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7285 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7286 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7287 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7288 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7289 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7290 status);
7291 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7292 /* reset addr and status */
7293 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7294 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7295 case 176: /* GFX RB CP_INT */
7296 case 177: /* GFX IB CP_INT */
7297 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7298 break;
7299 case 181: /* CP EOP event */
7300 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7301 /* XXX check the bitfield order! */
7302 me_id = (ring_id & 0x60) >> 5;
7303 pipe_id = (ring_id & 0x18) >> 3;
7304 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7305 switch (me_id) {
7306 case 0:
7307 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7308 break;
7309 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7310 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7311 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7312 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7313 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7314 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7315 break;
7316 }
7317 break;
7318 case 184: /* CP Privileged reg access */
7319 DRM_ERROR("Illegal register access in command stream\n");
7320 /* XXX check the bitfield order! */
7321 me_id = (ring_id & 0x60) >> 5;
7322 pipe_id = (ring_id & 0x18) >> 3;
7323 queue_id = (ring_id & 0x7) >> 0;
7324 switch (me_id) {
7325 case 0:
7326 /* This results in a full GPU reset, but all we need to do is soft
7327 * reset the CP for gfx
7328 */
7329 queue_reset = true;
7330 break;
7331 case 1:
7332 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7333 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7334 break;
7335 case 2:
7336 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7337 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7338 break;
7339 }
7340 break;
7341 case 185: /* CP Privileged inst */
7342 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7343 /* XXX check the bitfield order! */
7344 me_id = (ring_id & 0x60) >> 5;
7345 pipe_id = (ring_id & 0x18) >> 3;
7346 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7347 switch (me_id) {
7348 case 0:
7349 /* This results in a full GPU reset, but all we need to do is soft
7350 * reset the CP for gfx
7351 */
7352 queue_reset = true;
7353 break;
7354 case 1:
7355 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7356 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7357 break;
7358 case 2:
7359 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7360 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7361 break;
7362 }
7363 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7364 case 224: /* SDMA trap event */
7365 /* XXX check the bitfield order! */
7366 me_id = (ring_id & 0x3) >> 0;
7367 queue_id = (ring_id & 0xc) >> 2;
7368 DRM_DEBUG("IH: SDMA trap\n");
7369 switch (me_id) {
7370 case 0:
7371 switch (queue_id) {
7372 case 0:
7373 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7374 break;
7375 case 1:
7376 /* XXX compute */
7377 break;
7378 case 2:
7379 /* XXX compute */
7380 break;
7381 }
7382 break;
7383 case 1:
7384 switch (queue_id) {
7385 case 0:
7386 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7387 break;
7388 case 1:
7389 /* XXX compute */
7390 break;
7391 case 2:
7392 /* XXX compute */
7393 break;
7394 }
7395 break;
7396 }
7397 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7398 case 230: /* thermal low to high */
7399 DRM_DEBUG("IH: thermal low to high\n");
7400 rdev->pm.dpm.thermal.high_to_low = false;
7401 queue_thermal = true;
7402 break;
7403 case 231: /* thermal high to low */
7404 DRM_DEBUG("IH: thermal high to low\n");
7405 rdev->pm.dpm.thermal.high_to_low = true;
7406 queue_thermal = true;
7407 break;
7408 case 233: /* GUI IDLE */
7409 DRM_DEBUG("IH: GUI idle\n");
7410 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7411 case 241: /* SDMA Privileged inst */
7412 case 247: /* SDMA Privileged inst */
7413 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7414 /* XXX check the bitfield order! */
7415 me_id = (ring_id & 0x3) >> 0;
7416 queue_id = (ring_id & 0xc) >> 2;
7417 switch (me_id) {
7418 case 0:
7419 switch (queue_id) {
7420 case 0:
7421 queue_reset = true;
7422 break;
7423 case 1:
7424 /* XXX compute */
7425 queue_reset = true;
7426 break;
7427 case 2:
7428 /* XXX compute */
7429 queue_reset = true;
7430 break;
7431 }
7432 break;
7433 case 1:
7434 switch (queue_id) {
7435 case 0:
7436 queue_reset = true;
7437 break;
7438 case 1:
7439 /* XXX compute */
7440 queue_reset = true;
7441 break;
7442 case 2:
7443 /* XXX compute */
7444 queue_reset = true;
7445 break;
7446 }
7447 break;
7448 }
7449 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7450 default:
7451 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7452 break;
7453 }
7454
7455 /* wptr/rptr are in bytes! */
7456 rptr += 16;
7457 rptr &= rdev->ih.ptr_mask;
7458 }
7459 if (queue_hotplug)
7460 schedule_work(&rdev->hotplug_work);
7461 if (queue_reset)
7462 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7463 if (queue_thermal)
7464 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7465 rdev->ih.rptr = rptr;
7466 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7467 atomic_set(&rdev->ih.lock, 0);
7468
7469 /* make sure wptr hasn't changed while processing */
7470 wptr = cik_get_ih_wptr(rdev);
7471 if (wptr != rptr)
7472 goto restart_ih;
7473
7474 return IRQ_HANDLED;
7475}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7476
7477/*
7478 * startup/shutdown callbacks
7479 */
7480/**
7481 * cik_startup - program the asic to a functional state
7482 *
7483 * @rdev: radeon_device pointer
7484 *
7485 * Programs the asic to a functional state (CIK).
7486 * Called by cik_init() and cik_resume().
7487 * Returns 0 for success, error for failure.
7488 */
7489static int cik_startup(struct radeon_device *rdev)
7490{
7491 struct radeon_ring *ring;
7492 int r;
7493
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7494 /* enable pcie gen2/3 link */
7495 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7496 /* enable aspm */
7497 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7498
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7499 /* scratch needs to be initialized before MC */
7500 r = r600_vram_scratch_init(rdev);
7501 if (r)
7502 return r;
7503
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7504 cik_mc_program(rdev);
7505
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7506 if (rdev->flags & RADEON_IS_IGP) {
7507 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7508 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7509 r = cik_init_microcode(rdev);
7510 if (r) {
7511 DRM_ERROR("Failed to load firmware!\n");
7512 return r;
7513 }
7514 }
7515 } else {
7516 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7517 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7518 !rdev->mc_fw) {
7519 r = cik_init_microcode(rdev);
7520 if (r) {
7521 DRM_ERROR("Failed to load firmware!\n");
7522 return r;
7523 }
7524 }
7525
7526 r = ci_mc_load_microcode(rdev);
7527 if (r) {
7528 DRM_ERROR("Failed to load MC firmware!\n");
7529 return r;
7530 }
7531 }
7532
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7533 r = cik_pcie_gart_enable(rdev);
7534 if (r)
7535 return r;
7536 cik_gpu_init(rdev);
7537
7538 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7539 if (rdev->flags & RADEON_IS_IGP) {
7540 if (rdev->family == CHIP_KAVERI) {
7541 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7542 rdev->rlc.reg_list_size =
7543 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7544 } else {
7545 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7546 rdev->rlc.reg_list_size =
7547 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7548 }
7549 }
7550 rdev->rlc.cs_data = ci_cs_data;
7551 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7552 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7553 if (r) {
7554 DRM_ERROR("Failed to init rlc BOs!\n");
7555 return r;
7556 }
7557
7558 /* allocate wb buffer */
7559 r = radeon_wb_init(rdev);
7560 if (r)
7561 return r;
7562
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7563 /* allocate mec buffers */
7564 r = cik_mec_init(rdev);
7565 if (r) {
7566 DRM_ERROR("Failed to init MEC BOs!\n");
7567 return r;
7568 }
7569
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7570 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7571 if (r) {
7572 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7573 return r;
7574 }
7575
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7576 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7577 if (r) {
7578 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7579 return r;
7580 }
7581
7582 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7583 if (r) {
7584 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7585 return r;
7586 }
7587
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7588 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7589 if (r) {
7590 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7591 return r;
7592 }
7593
7594 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7595 if (r) {
7596 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7597 return r;
7598 }
7599
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7600 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7601 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7602 r = uvd_v4_2_resume(rdev);
7603 if (!r) {
7604 r = radeon_fence_driver_start_ring(rdev,
7605 R600_RING_TYPE_UVD_INDEX);
7606 if (r)
7607 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7608 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7609 }
7610 if (r)
7611 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7612
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7613 /* Enable IRQ */
7614 if (!rdev->irq.installed) {
7615 r = radeon_irq_kms_init(rdev);
7616 if (r)
7617 return r;
7618 }
7619
7620 r = cik_irq_init(rdev);
7621 if (r) {
7622 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7623 radeon_irq_kms_fini(rdev);
7624 return r;
7625 }
7626 cik_irq_set(rdev);
7627
7628 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7629 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7630 CP_RB0_RPTR, CP_RB0_WPTR,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7631 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7632 if (r)
7633 return r;
7634
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7635 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7636 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7637 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7638 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7639 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7640 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7641 if (r)
7642 return r;
7643 ring->me = 1; /* first MEC */
7644 ring->pipe = 0; /* first pipe */
7645 ring->queue = 0; /* first queue */
7646 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7647
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7648 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7649 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7650 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7651 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7652 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7653 if (r)
7654 return r;
7655 /* dGPU only have 1 MEC */
7656 ring->me = 1; /* first MEC */
7657 ring->pipe = 0; /* first pipe */
7658 ring->queue = 1; /* second queue */
7659 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7660
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7661 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7662 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7663 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7664 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7665 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7666 if (r)
7667 return r;
7668
7669 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7670 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7671 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7672 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7673 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7674 if (r)
7675 return r;
7676
7677 r = cik_cp_resume(rdev);
7678 if (r)
7679 return r;
7680
7681 r = cik_sdma_resume(rdev);
7682 if (r)
7683 return r;
7684
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7685 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7686 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7687 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7688 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7689 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7690 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7691 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7692 if (r)
7693 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7694 }
7695
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7696 r = radeon_ib_pool_init(rdev);
7697 if (r) {
7698 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7699 return r;
7700 }
7701
7702 r = radeon_vm_manager_init(rdev);
7703 if (r) {
7704 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7705 return r;
7706 }
7707
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7708 r = dce6_audio_init(rdev);
7709 if (r)
7710 return r;
7711
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7712 return 0;
7713}
7714
7715/**
7716 * cik_resume - resume the asic to a functional state
7717 *
7718 * @rdev: radeon_device pointer
7719 *
7720 * Programs the asic to a functional state (CIK).
7721 * Called at resume.
7722 * Returns 0 for success, error for failure.
7723 */
7724int cik_resume(struct radeon_device *rdev)
7725{
7726 int r;
7727
7728 /* post card */
7729 atom_asic_init(rdev->mode_info.atom_context);
7730
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7731 /* init golden registers */
7732 cik_init_golden_registers(rdev);
7733
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7734 rdev->accel_working = true;
7735 r = cik_startup(rdev);
7736 if (r) {
7737 DRM_ERROR("cik startup failed on resume\n");
7738 rdev->accel_working = false;
7739 return r;
7740 }
7741
7742 return r;
7743
7744}
7745
7746/**
7747 * cik_suspend - suspend the asic
7748 *
7749 * @rdev: radeon_device pointer
7750 *
7751 * Bring the chip into a state suitable for suspend (CIK).
7752 * Called at suspend.
7753 * Returns 0 for success.
7754 */
7755int cik_suspend(struct radeon_device *rdev)
7756{
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7757 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7758 radeon_vm_manager_fini(rdev);
7759 cik_cp_enable(rdev, false);
7760 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7761 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7762 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7763 cik_fini_pg(rdev);
7764 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7765 cik_irq_suspend(rdev);
7766 radeon_wb_disable(rdev);
7767 cik_pcie_gart_disable(rdev);
7768 return 0;
7769}
7770
7771/* Plan is to move initialization in that function and use
7772 * helper function so that radeon_device_init pretty much
7773 * do nothing more than calling asic specific function. This
7774 * should also allow to remove a bunch of callback function
7775 * like vram_info.
7776 */
7777/**
7778 * cik_init - asic specific driver and hw init
7779 *
7780 * @rdev: radeon_device pointer
7781 *
7782 * Setup asic specific driver variables and program the hw
7783 * to a functional state (CIK).
7784 * Called at driver startup.
7785 * Returns 0 for success, errors for failure.
7786 */
7787int cik_init(struct radeon_device *rdev)
7788{
7789 struct radeon_ring *ring;
7790 int r;
7791
7792 /* Read BIOS */
7793 if (!radeon_get_bios(rdev)) {
7794 if (ASIC_IS_AVIVO(rdev))
7795 return -EINVAL;
7796 }
7797 /* Must be an ATOMBIOS */
7798 if (!rdev->is_atom_bios) {
7799 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7800 return -EINVAL;
7801 }
7802 r = radeon_atombios_init(rdev);
7803 if (r)
7804 return r;
7805
7806 /* Post card if necessary */
7807 if (!radeon_card_posted(rdev)) {
7808 if (!rdev->bios) {
7809 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7810 return -EINVAL;
7811 }
7812 DRM_INFO("GPU not posted. posting now...\n");
7813 atom_asic_init(rdev->mode_info.atom_context);
7814 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7815 /* init golden registers */
7816 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7817 /* Initialize scratch registers */
7818 cik_scratch_init(rdev);
7819 /* Initialize surface registers */
7820 radeon_surface_init(rdev);
7821 /* Initialize clocks */
7822 radeon_get_clock_info(rdev->ddev);
7823
7824 /* Fence driver */
7825 r = radeon_fence_driver_init(rdev);
7826 if (r)
7827 return r;
7828
7829 /* initialize memory controller */
7830 r = cik_mc_init(rdev);
7831 if (r)
7832 return r;
7833 /* Memory manager */
7834 r = radeon_bo_init(rdev);
7835 if (r)
7836 return r;
7837
7838 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7839 ring->ring_obj = NULL;
7840 r600_ring_init(rdev, ring, 1024 * 1024);
7841
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7842 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7843 ring->ring_obj = NULL;
7844 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]7845 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7846 if (r)
7847 return r;
7848
7849 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7850 ring->ring_obj = NULL;
7851 r600_ring_init(rdev, ring, 1024 * 1024);
Andrew Lewyckyd5754ab2013-11-13 15:54:17 -0500[diff] [blame]7852 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7853 if (r)
7854 return r;
7855
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7856 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7857 ring->ring_obj = NULL;
7858 r600_ring_init(rdev, ring, 256 * 1024);
7859
7860 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7861 ring->ring_obj = NULL;
7862 r600_ring_init(rdev, ring, 256 * 1024);
7863
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7864 r = radeon_uvd_init(rdev);
7865 if (!r) {
7866 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7867 ring->ring_obj = NULL;
7868 r600_ring_init(rdev, ring, 4096);
7869 }
7870
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7871 rdev->ih.ring_obj = NULL;
7872 r600_ih_ring_init(rdev, 64 * 1024);
7873
7874 r = r600_pcie_gart_init(rdev);
7875 if (r)
7876 return r;
7877
7878 rdev->accel_working = true;
7879 r = cik_startup(rdev);
7880 if (r) {
7881 dev_err(rdev->dev, "disabling GPU acceleration\n");
7882 cik_cp_fini(rdev);
7883 cik_sdma_fini(rdev);
7884 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7885 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7886 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7887 radeon_wb_fini(rdev);
7888 radeon_ib_pool_fini(rdev);
7889 radeon_vm_manager_fini(rdev);
7890 radeon_irq_kms_fini(rdev);
7891 cik_pcie_gart_fini(rdev);
7892 rdev->accel_working = false;
7893 }
7894
7895 /* Don't start up if the MC ucode is missing.
7896 * The default clocks and voltages before the MC ucode
7897 * is loaded are not suffient for advanced operations.
7898 */
7899 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7900 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7901 return -EINVAL;
7902 }
7903
7904 return 0;
7905}
7906
7907/**
7908 * cik_fini - asic specific driver and hw fini
7909 *
7910 * @rdev: radeon_device pointer
7911 *
7912 * Tear down the asic specific driver variables and program the hw
7913 * to an idle state (CIK).
7914 * Called at driver unload.
7915 */
7916void cik_fini(struct radeon_device *rdev)
7917{
7918 cik_cp_fini(rdev);
7919 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7920 cik_fini_pg(rdev);
7921 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7922 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7923 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7924 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7925 radeon_wb_fini(rdev);
7926 radeon_vm_manager_fini(rdev);
7927 radeon_ib_pool_fini(rdev);
7928 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7929 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7930 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7931 cik_pcie_gart_fini(rdev);
7932 r600_vram_scratch_fini(rdev);
7933 radeon_gem_fini(rdev);
7934 radeon_fence_driver_fini(rdev);
7935 radeon_bo_fini(rdev);
7936 radeon_atombios_fini(rdev);
7937 kfree(rdev->bios);
7938 rdev->bios = NULL;
7939}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7940
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7941void dce8_program_fmt(struct drm_encoder *encoder)
7942{
7943 struct drm_device *dev = encoder->dev;
7944 struct radeon_device *rdev = dev->dev_private;
7945 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7946 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7947 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7948 int bpc = 0;
7949 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7950 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7951
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7952 if (connector) {
7953 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7954 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7955 dither = radeon_connector->dither;
7956 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7957
7958 /* LVDS/eDP FMT is set up by atom */
7959 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7960 return;
7961
7962 /* not needed for analog */
7963 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7964 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7965 return;
7966
7967 if (bpc == 0)
7968 return;
7969
7970 switch (bpc) {
7971 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7972 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7973 /* XXX sort out optimal dither settings */
7974 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7975 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7976 else
7977 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7978 break;
7979 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7980 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7981 /* XXX sort out optimal dither settings */
7982 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7983 FMT_RGB_RANDOM_ENABLE |
7984 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7985 else
7986 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7987 break;
7988 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7989 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7990 /* XXX sort out optimal dither settings */
7991 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7992 FMT_RGB_RANDOM_ENABLE |
7993 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7994 else
7995 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7996 break;
7997 default:
7998 /* not needed */
7999 break;
8000 }
8001
8002 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8003}
8004
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8005/* display watermark setup */
8006/**
8007 * dce8_line_buffer_adjust - Set up the line buffer
8008 *
8009 * @rdev: radeon_device pointer
8010 * @radeon_crtc: the selected display controller
8011 * @mode: the current display mode on the selected display
8012 * controller
8013 *
8014 * Setup up the line buffer allocation for
8015 * the selected display controller (CIK).
8016 * Returns the line buffer size in pixels.
8017 */
8018static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8019 struct radeon_crtc *radeon_crtc,
8020 struct drm_display_mode *mode)
8021{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8022 u32 tmp, buffer_alloc, i;
8023 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8024 /*
8025 * Line Buffer Setup
8026 * There are 6 line buffers, one for each display controllers.
8027 * There are 3 partitions per LB. Select the number of partitions
8028 * to enable based on the display width. For display widths larger
8029 * than 4096, you need use to use 2 display controllers and combine
8030 * them using the stereo blender.
8031 */
8032 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8033 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8034 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8035 buffer_alloc = 2;
8036 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8037 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8038 buffer_alloc = 2;
8039 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8040 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8041 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8042 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8043 DRM_DEBUG_KMS("Mode too big for LB!\n");
8044 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8045 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8046 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8047 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8048 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8049 buffer_alloc = 0;
8050 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8051
8052 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8053 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8054
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]8055 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8056 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8057 for (i = 0; i < rdev->usec_timeout; i++) {
8058 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8059 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8060 break;
8061 udelay(1);
8062 }
8063
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8064 if (radeon_crtc->base.enabled && mode) {
8065 switch (tmp) {
8066 case 0:
8067 default:
8068 return 4096 * 2;
8069 case 1:
8070 return 1920 * 2;
8071 case 2:
8072 return 2560 * 2;
8073 }
8074 }
8075
8076 /* controller not enabled, so no lb used */
8077 return 0;
8078}
8079
8080/**
8081 * cik_get_number_of_dram_channels - get the number of dram channels
8082 *
8083 * @rdev: radeon_device pointer
8084 *
8085 * Look up the number of video ram channels (CIK).
8086 * Used for display watermark bandwidth calculations
8087 * Returns the number of dram channels
8088 */
8089static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8090{
8091 u32 tmp = RREG32(MC_SHARED_CHMAP);
8092
8093 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8094 case 0:
8095 default:
8096 return 1;
8097 case 1:
8098 return 2;
8099 case 2:
8100 return 4;
8101 case 3:
8102 return 8;
8103 case 4:
8104 return 3;
8105 case 5:
8106 return 6;
8107 case 6:
8108 return 10;
8109 case 7:
8110 return 12;
8111 case 8:
8112 return 16;
8113 }
8114}
8115
8116struct dce8_wm_params {
8117 u32 dram_channels; /* number of dram channels */
8118 u32 yclk; /* bandwidth per dram data pin in kHz */
8119 u32 sclk; /* engine clock in kHz */
8120 u32 disp_clk; /* display clock in kHz */
8121 u32 src_width; /* viewport width */
8122 u32 active_time; /* active display time in ns */
8123 u32 blank_time; /* blank time in ns */
8124 bool interlaced; /* mode is interlaced */
8125 fixed20_12 vsc; /* vertical scale ratio */
8126 u32 num_heads; /* number of active crtcs */
8127 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8128 u32 lb_size; /* line buffer allocated to pipe */
8129 u32 vtaps; /* vertical scaler taps */
8130};
8131
8132/**
8133 * dce8_dram_bandwidth - get the dram bandwidth
8134 *
8135 * @wm: watermark calculation data
8136 *
8137 * Calculate the raw dram bandwidth (CIK).
8138 * Used for display watermark bandwidth calculations
8139 * Returns the dram bandwidth in MBytes/s
8140 */
8141static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8142{
8143 /* Calculate raw DRAM Bandwidth */
8144 fixed20_12 dram_efficiency; /* 0.7 */
8145 fixed20_12 yclk, dram_channels, bandwidth;
8146 fixed20_12 a;
8147
8148 a.full = dfixed_const(1000);
8149 yclk.full = dfixed_const(wm->yclk);
8150 yclk.full = dfixed_div(yclk, a);
8151 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8152 a.full = dfixed_const(10);
8153 dram_efficiency.full = dfixed_const(7);
8154 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8155 bandwidth.full = dfixed_mul(dram_channels, yclk);
8156 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8157
8158 return dfixed_trunc(bandwidth);
8159}
8160
8161/**
8162 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8163 *
8164 * @wm: watermark calculation data
8165 *
8166 * Calculate the dram bandwidth used for display (CIK).
8167 * Used for display watermark bandwidth calculations
8168 * Returns the dram bandwidth for display in MBytes/s
8169 */
8170static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8171{
8172 /* Calculate DRAM Bandwidth and the part allocated to display. */
8173 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8174 fixed20_12 yclk, dram_channels, bandwidth;
8175 fixed20_12 a;
8176
8177 a.full = dfixed_const(1000);
8178 yclk.full = dfixed_const(wm->yclk);
8179 yclk.full = dfixed_div(yclk, a);
8180 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8181 a.full = dfixed_const(10);
8182 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8183 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8184 bandwidth.full = dfixed_mul(dram_channels, yclk);
8185 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8186
8187 return dfixed_trunc(bandwidth);
8188}
8189
8190/**
8191 * dce8_data_return_bandwidth - get the data return bandwidth
8192 *
8193 * @wm: watermark calculation data
8194 *
8195 * Calculate the data return bandwidth used for display (CIK).
8196 * Used for display watermark bandwidth calculations
8197 * Returns the data return bandwidth in MBytes/s
8198 */
8199static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8200{
8201 /* Calculate the display Data return Bandwidth */
8202 fixed20_12 return_efficiency; /* 0.8 */
8203 fixed20_12 sclk, bandwidth;
8204 fixed20_12 a;
8205
8206 a.full = dfixed_const(1000);
8207 sclk.full = dfixed_const(wm->sclk);
8208 sclk.full = dfixed_div(sclk, a);
8209 a.full = dfixed_const(10);
8210 return_efficiency.full = dfixed_const(8);
8211 return_efficiency.full = dfixed_div(return_efficiency, a);
8212 a.full = dfixed_const(32);
8213 bandwidth.full = dfixed_mul(a, sclk);
8214 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8215
8216 return dfixed_trunc(bandwidth);
8217}
8218
8219/**
8220 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8221 *
8222 * @wm: watermark calculation data
8223 *
8224 * Calculate the dmif bandwidth used for display (CIK).
8225 * Used for display watermark bandwidth calculations
8226 * Returns the dmif bandwidth in MBytes/s
8227 */
8228static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8229{
8230 /* Calculate the DMIF Request Bandwidth */
8231 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8232 fixed20_12 disp_clk, bandwidth;
8233 fixed20_12 a, b;
8234
8235 a.full = dfixed_const(1000);
8236 disp_clk.full = dfixed_const(wm->disp_clk);
8237 disp_clk.full = dfixed_div(disp_clk, a);
8238 a.full = dfixed_const(32);
8239 b.full = dfixed_mul(a, disp_clk);
8240
8241 a.full = dfixed_const(10);
8242 disp_clk_request_efficiency.full = dfixed_const(8);
8243 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8244
8245 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8246
8247 return dfixed_trunc(bandwidth);
8248}
8249
8250/**
8251 * dce8_available_bandwidth - get the min available bandwidth
8252 *
8253 * @wm: watermark calculation data
8254 *
8255 * Calculate the min available bandwidth used for display (CIK).
8256 * Used for display watermark bandwidth calculations
8257 * Returns the min available bandwidth in MBytes/s
8258 */
8259static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8260{
8261 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8262 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8263 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8264 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8265
8266 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8267}
8268
8269/**
8270 * dce8_average_bandwidth - get the average available bandwidth
8271 *
8272 * @wm: watermark calculation data
8273 *
8274 * Calculate the average available bandwidth used for display (CIK).
8275 * Used for display watermark bandwidth calculations
8276 * Returns the average available bandwidth in MBytes/s
8277 */
8278static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8279{
8280 /* Calculate the display mode Average Bandwidth
8281 * DisplayMode should contain the source and destination dimensions,
8282 * timing, etc.
8283 */
8284 fixed20_12 bpp;
8285 fixed20_12 line_time;
8286 fixed20_12 src_width;
8287 fixed20_12 bandwidth;
8288 fixed20_12 a;
8289
8290 a.full = dfixed_const(1000);
8291 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8292 line_time.full = dfixed_div(line_time, a);
8293 bpp.full = dfixed_const(wm->bytes_per_pixel);
8294 src_width.full = dfixed_const(wm->src_width);
8295 bandwidth.full = dfixed_mul(src_width, bpp);
8296 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8297 bandwidth.full = dfixed_div(bandwidth, line_time);
8298
8299 return dfixed_trunc(bandwidth);
8300}
8301
8302/**
8303 * dce8_latency_watermark - get the latency watermark
8304 *
8305 * @wm: watermark calculation data
8306 *
8307 * Calculate the latency watermark (CIK).
8308 * Used for display watermark bandwidth calculations
8309 * Returns the latency watermark in ns
8310 */
8311static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8312{
8313 /* First calculate the latency in ns */
8314 u32 mc_latency = 2000; /* 2000 ns. */
8315 u32 available_bandwidth = dce8_available_bandwidth(wm);
8316 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8317 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8318 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8319 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8320 (wm->num_heads * cursor_line_pair_return_time);
8321 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8322 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8323 u32 tmp, dmif_size = 12288;
8324 fixed20_12 a, b, c;
8325
8326 if (wm->num_heads == 0)
8327 return 0;
8328
8329 a.full = dfixed_const(2);
8330 b.full = dfixed_const(1);
8331 if ((wm->vsc.full > a.full) ||
8332 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8333 (wm->vtaps >= 5) ||
8334 ((wm->vsc.full >= a.full) && wm->interlaced))
8335 max_src_lines_per_dst_line = 4;
8336 else
8337 max_src_lines_per_dst_line = 2;
8338
8339 a.full = dfixed_const(available_bandwidth);
8340 b.full = dfixed_const(wm->num_heads);
8341 a.full = dfixed_div(a, b);
8342
8343 b.full = dfixed_const(mc_latency + 512);
8344 c.full = dfixed_const(wm->disp_clk);
8345 b.full = dfixed_div(b, c);
8346
8347 c.full = dfixed_const(dmif_size);
8348 b.full = dfixed_div(c, b);
8349
8350 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8351
8352 b.full = dfixed_const(1000);
8353 c.full = dfixed_const(wm->disp_clk);
8354 b.full = dfixed_div(c, b);
8355 c.full = dfixed_const(wm->bytes_per_pixel);
8356 b.full = dfixed_mul(b, c);
8357
8358 lb_fill_bw = min(tmp, dfixed_trunc(b));
8359
8360 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8361 b.full = dfixed_const(1000);
8362 c.full = dfixed_const(lb_fill_bw);
8363 b.full = dfixed_div(c, b);
8364 a.full = dfixed_div(a, b);
8365 line_fill_time = dfixed_trunc(a);
8366
8367 if (line_fill_time < wm->active_time)
8368 return latency;
8369 else
8370 return latency + (line_fill_time - wm->active_time);
8371
8372}
8373
8374/**
8375 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8376 * average and available dram bandwidth
8377 *
8378 * @wm: watermark calculation data
8379 *
8380 * Check if the display average bandwidth fits in the display
8381 * dram bandwidth (CIK).
8382 * Used for display watermark bandwidth calculations
8383 * Returns true if the display fits, false if not.
8384 */
8385static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8386{
8387 if (dce8_average_bandwidth(wm) <=
8388 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8389 return true;
8390 else
8391 return false;
8392}
8393
8394/**
8395 * dce8_average_bandwidth_vs_available_bandwidth - check
8396 * average and available bandwidth
8397 *
8398 * @wm: watermark calculation data
8399 *
8400 * Check if the display average bandwidth fits in the display
8401 * available bandwidth (CIK).
8402 * Used for display watermark bandwidth calculations
8403 * Returns true if the display fits, false if not.
8404 */
8405static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8406{
8407 if (dce8_average_bandwidth(wm) <=
8408 (dce8_available_bandwidth(wm) / wm->num_heads))
8409 return true;
8410 else
8411 return false;
8412}
8413
8414/**
8415 * dce8_check_latency_hiding - check latency hiding
8416 *
8417 * @wm: watermark calculation data
8418 *
8419 * Check latency hiding (CIK).
8420 * Used for display watermark bandwidth calculations
8421 * Returns true if the display fits, false if not.
8422 */
8423static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8424{
8425 u32 lb_partitions = wm->lb_size / wm->src_width;
8426 u32 line_time = wm->active_time + wm->blank_time;
8427 u32 latency_tolerant_lines;
8428 u32 latency_hiding;
8429 fixed20_12 a;
8430
8431 a.full = dfixed_const(1);
8432 if (wm->vsc.full > a.full)
8433 latency_tolerant_lines = 1;
8434 else {
8435 if (lb_partitions <= (wm->vtaps + 1))
8436 latency_tolerant_lines = 1;
8437 else
8438 latency_tolerant_lines = 2;
8439 }
8440
8441 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8442
8443 if (dce8_latency_watermark(wm) <= latency_hiding)
8444 return true;
8445 else
8446 return false;
8447}
8448
8449/**
8450 * dce8_program_watermarks - program display watermarks
8451 *
8452 * @rdev: radeon_device pointer
8453 * @radeon_crtc: the selected display controller
8454 * @lb_size: line buffer size
8455 * @num_heads: number of display controllers in use
8456 *
8457 * Calculate and program the display watermarks for the
8458 * selected display controller (CIK).
8459 */
8460static void dce8_program_watermarks(struct radeon_device *rdev,
8461 struct radeon_crtc *radeon_crtc,
8462 u32 lb_size, u32 num_heads)
8463{
8464 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8465 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8466 u32 pixel_period;
8467 u32 line_time = 0;
8468 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8469 u32 tmp, wm_mask;
8470
8471 if (radeon_crtc->base.enabled && num_heads && mode) {
8472 pixel_period = 1000000 / (u32)mode->clock;
8473 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8474
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8475 /* watermark for high clocks */
8476 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8477 rdev->pm.dpm_enabled) {
8478 wm_high.yclk =
8479 radeon_dpm_get_mclk(rdev, false) * 10;
8480 wm_high.sclk =
8481 radeon_dpm_get_sclk(rdev, false) * 10;
8482 } else {
8483 wm_high.yclk = rdev->pm.current_mclk * 10;
8484 wm_high.sclk = rdev->pm.current_sclk * 10;
8485 }
8486
8487 wm_high.disp_clk = mode->clock;
8488 wm_high.src_width = mode->crtc_hdisplay;
8489 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8490 wm_high.blank_time = line_time - wm_high.active_time;
8491 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8492 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8493 wm_high.interlaced = true;
8494 wm_high.vsc = radeon_crtc->vsc;
8495 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8496 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8497 wm_high.vtaps = 2;
8498 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8499 wm_high.lb_size = lb_size;
8500 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8501 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8502
8503 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8504 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8505
8506 /* possibly force display priority to high */
8507 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8508 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8509 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8510 !dce8_check_latency_hiding(&wm_high) ||
8511 (rdev->disp_priority == 2)) {
8512 DRM_DEBUG_KMS("force priority to high\n");
8513 }
8514
8515 /* watermark for low clocks */
8516 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8517 rdev->pm.dpm_enabled) {
8518 wm_low.yclk =
8519 radeon_dpm_get_mclk(rdev, true) * 10;
8520 wm_low.sclk =
8521 radeon_dpm_get_sclk(rdev, true) * 10;
8522 } else {
8523 wm_low.yclk = rdev->pm.current_mclk * 10;
8524 wm_low.sclk = rdev->pm.current_sclk * 10;
8525 }
8526
8527 wm_low.disp_clk = mode->clock;
8528 wm_low.src_width = mode->crtc_hdisplay;
8529 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8530 wm_low.blank_time = line_time - wm_low.active_time;
8531 wm_low.interlaced = false;
8532 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8533 wm_low.interlaced = true;
8534 wm_low.vsc = radeon_crtc->vsc;
8535 wm_low.vtaps = 1;
8536 if (radeon_crtc->rmx_type != RMX_OFF)
8537 wm_low.vtaps = 2;
8538 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8539 wm_low.lb_size = lb_size;
8540 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8541 wm_low.num_heads = num_heads;
8542
8543 /* set for low clocks */
8544 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8545
8546 /* possibly force display priority to high */
8547 /* should really do this at mode validation time... */
8548 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8549 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8550 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8551 (rdev->disp_priority == 2)) {
8552 DRM_DEBUG_KMS("force priority to high\n");
8553 }
8554 }
8555
8556 /* select wm A */
8557 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8558 tmp = wm_mask;
8559 tmp &= ~LATENCY_WATERMARK_MASK(3);
8560 tmp |= LATENCY_WATERMARK_MASK(1);
8561 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8562 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8563 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8564 LATENCY_HIGH_WATERMARK(line_time)));
8565 /* select wm B */
8566 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8567 tmp &= ~LATENCY_WATERMARK_MASK(3);
8568 tmp |= LATENCY_WATERMARK_MASK(2);
8569 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8570 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8571 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8572 LATENCY_HIGH_WATERMARK(line_time)));
8573 /* restore original selection */
8574 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8575
8576 /* save values for DPM */
8577 radeon_crtc->line_time = line_time;
8578 radeon_crtc->wm_high = latency_watermark_a;
8579 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8580}
8581
8582/**
8583 * dce8_bandwidth_update - program display watermarks
8584 *
8585 * @rdev: radeon_device pointer
8586 *
8587 * Calculate and program the display watermarks and line
8588 * buffer allocation (CIK).
8589 */
8590void dce8_bandwidth_update(struct radeon_device *rdev)
8591{
8592 struct drm_display_mode *mode = NULL;
8593 u32 num_heads = 0, lb_size;
8594 int i;
8595
8596 radeon_update_display_priority(rdev);
8597
8598 for (i = 0; i < rdev->num_crtc; i++) {
8599 if (rdev->mode_info.crtcs[i]->base.enabled)
8600 num_heads++;
8601 }
8602 for (i = 0; i < rdev->num_crtc; i++) {
8603 mode = &rdev->mode_info.crtcs[i]->base.mode;
8604 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8605 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8606 }
8607}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8608
8609/**
8610 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8611 *
8612 * @rdev: radeon_device pointer
8613 *
8614 * Fetches a GPU clock counter snapshot (SI).
8615 * Returns the 64 bit clock counter snapshot.
8616 */
8617uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8618{
8619 uint64_t clock;
8620
8621 mutex_lock(&rdev->gpu_clock_mutex);
8622 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8623 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8624 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8625 mutex_unlock(&rdev->gpu_clock_mutex);
8626 return clock;
8627}
8628
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8629static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8630 u32 cntl_reg, u32 status_reg)
8631{
8632 int r, i;
8633 struct atom_clock_dividers dividers;
8634 uint32_t tmp;
8635
8636 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8637 clock, false, &dividers);
8638 if (r)
8639 return r;
8640
8641 tmp = RREG32_SMC(cntl_reg);
8642 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8643 tmp |= dividers.post_divider;
8644 WREG32_SMC(cntl_reg, tmp);
8645
8646 for (i = 0; i < 100; i++) {
8647 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8648 break;
8649 mdelay(10);
8650 }
8651 if (i == 100)
8652 return -ETIMEDOUT;
8653
8654 return 0;
8655}
8656
8657int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8658{
8659 int r = 0;
8660
8661 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8662 if (r)
8663 return r;
8664
8665 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8666 return r;
8667}
8668
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8669static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8670{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8671 struct pci_dev *root = rdev->pdev->bus->self;
8672 int bridge_pos, gpu_pos;
8673 u32 speed_cntl, mask, current_data_rate;
8674 int ret, i;
8675 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8676
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8677 if (radeon_pcie_gen2 == 0)
8678 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8679
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8680 if (rdev->flags & RADEON_IS_IGP)
8681 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8682
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8683 if (!(rdev->flags & RADEON_IS_PCIE))
8684 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8685
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8686 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8687 if (ret != 0)
8688 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8689
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8690 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8691 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8692
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8693 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8694 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8695 LC_CURRENT_DATA_RATE_SHIFT;
8696 if (mask & DRM_PCIE_SPEED_80) {
8697 if (current_data_rate == 2) {
8698 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8699 return;
8700 }
8701 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8702 } else if (mask & DRM_PCIE_SPEED_50) {
8703 if (current_data_rate == 1) {
8704 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8705 return;
8706 }
8707 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8708 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8709
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8710 bridge_pos = pci_pcie_cap(root);
8711 if (!bridge_pos)
8712 return;
8713
8714 gpu_pos = pci_pcie_cap(rdev->pdev);
8715 if (!gpu_pos)
8716 return;
8717
8718 if (mask & DRM_PCIE_SPEED_80) {
8719 /* re-try equalization if gen3 is not already enabled */
8720 if (current_data_rate != 2) {
8721 u16 bridge_cfg, gpu_cfg;
8722 u16 bridge_cfg2, gpu_cfg2;
8723 u32 max_lw, current_lw, tmp;
8724
8725 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8726 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8727
8728 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8729 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8730
8731 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8732 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8733
8734 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8735 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8736 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8737
8738 if (current_lw < max_lw) {
8739 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8740 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8741 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8742 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8743 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8744 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8745 }
8746 }
8747
8748 for (i = 0; i < 10; i++) {
8749 /* check status */
8750 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8751 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8752 break;
8753
8754 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8755 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8756
8757 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8758 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8759
8760 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8761 tmp |= LC_SET_QUIESCE;
8762 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8763
8764 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8765 tmp |= LC_REDO_EQ;
8766 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8767
8768 mdelay(100);
8769
8770 /* linkctl */
8771 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8772 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8773 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8774 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8775
8776 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8777 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8778 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8779 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8780
8781 /* linkctl2 */
8782 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8783 tmp16 &= ~((1 << 4) | (7 << 9));
8784 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8785 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8786
8787 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8788 tmp16 &= ~((1 << 4) | (7 << 9));
8789 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8790 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8791
8792 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8793 tmp &= ~LC_SET_QUIESCE;
8794 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8795 }
8796 }
8797 }
8798
8799 /* set the link speed */
8800 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8801 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8802 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8803
8804 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8805 tmp16 &= ~0xf;
8806 if (mask & DRM_PCIE_SPEED_80)
8807 tmp16 |= 3; /* gen3 */
8808 else if (mask & DRM_PCIE_SPEED_50)
8809 tmp16 |= 2; /* gen2 */
8810 else
8811 tmp16 |= 1; /* gen1 */
8812 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8813
8814 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8815 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8816 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8817
8818 for (i = 0; i < rdev->usec_timeout; i++) {
8819 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8820 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8821 break;
8822 udelay(1);
8823 }
8824}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]8825
8826static void cik_program_aspm(struct radeon_device *rdev)
8827{
8828 u32 data, orig;
8829 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8830 bool disable_clkreq = false;
8831
8832 if (radeon_aspm == 0)
8833 return;
8834
8835 /* XXX double check IGPs */
8836 if (rdev->flags & RADEON_IS_IGP)
8837 return;
8838
8839 if (!(rdev->flags & RADEON_IS_PCIE))
8840 return;
8841
8842 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8843 data &= ~LC_XMIT_N_FTS_MASK;
8844 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8845 if (orig != data)
8846 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8847
8848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8849 data |= LC_GO_TO_RECOVERY;
8850 if (orig != data)
8851 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8852
8853 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8854 data |= P_IGNORE_EDB_ERR;
8855 if (orig != data)
8856 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8857
8858 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8859 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8860 data |= LC_PMI_TO_L1_DIS;
8861 if (!disable_l0s)
8862 data |= LC_L0S_INACTIVITY(7);
8863
8864 if (!disable_l1) {
8865 data |= LC_L1_INACTIVITY(7);
8866 data &= ~LC_PMI_TO_L1_DIS;
8867 if (orig != data)
8868 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8869
8870 if (!disable_plloff_in_l1) {
8871 bool clk_req_support;
8872
8873 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8874 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8875 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8876 if (orig != data)
8877 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8878
8879 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8880 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8881 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8882 if (orig != data)
8883 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8884
8885 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8886 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8887 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8888 if (orig != data)
8889 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8890
8891 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8892 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8893 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8894 if (orig != data)
8895 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8896
8897 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8898 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8899 data |= LC_DYN_LANES_PWR_STATE(3);
8900 if (orig != data)
8901 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8902
8903 if (!disable_clkreq) {
8904 struct pci_dev *root = rdev->pdev->bus->self;
8905 u32 lnkcap;
8906
8907 clk_req_support = false;
8908 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8909 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8910 clk_req_support = true;
8911 } else {
8912 clk_req_support = false;
8913 }
8914
8915 if (clk_req_support) {
8916 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8917 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8918 if (orig != data)
8919 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8920
8921 orig = data = RREG32_SMC(THM_CLK_CNTL);
8922 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8923 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8924 if (orig != data)
8925 WREG32_SMC(THM_CLK_CNTL, data);
8926
8927 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8928 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8929 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8930 if (orig != data)
8931 WREG32_SMC(MISC_CLK_CTRL, data);
8932
8933 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8934 data &= ~BCLK_AS_XCLK;
8935 if (orig != data)
8936 WREG32_SMC(CG_CLKPIN_CNTL, data);
8937
8938 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8939 data &= ~FORCE_BIF_REFCLK_EN;
8940 if (orig != data)
8941 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8942
8943 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8944 data &= ~MPLL_CLKOUT_SEL_MASK;
8945 data |= MPLL_CLKOUT_SEL(4);
8946 if (orig != data)
8947 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8948 }
8949 }
8950 } else {
8951 if (orig != data)
8952 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8953 }
8954
8955 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8956 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8957 if (orig != data)
8958 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8959
8960 if (!disable_l0s) {
8961 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8962 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8963 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8964 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8965 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8966 data &= ~LC_L0S_INACTIVITY_MASK;
8967 if (orig != data)
8968 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8969 }
8970 }
8971 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8972}