Gitiles
Code Review Sign In
gerrit-public.fairphone.software / kernel / msm-4.9 / fc821b70b0e58908e94464d0479c9898dd1074f9 / . / drivers / gpu / drm / radeon / cik.c
blob: 24d96cbb1f0370203b657a7c79a9f1678cc51ba8 [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]44MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]49MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]50MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51MODULE_FIRMWARE("radeon/KABINI_me.bin");
52MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]55MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]56
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]57extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]59extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]61extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]62extern void sumo_rlc_fini(struct radeon_device *rdev);
63extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]64extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]65extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]66extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]67extern int cik_sdma_resume(struct radeon_device *rdev);
68extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]70static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]71static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]72static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]73static void cik_init_pg(struct radeon_device *rdev);
74static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]75static void cik_fini_pg(struct radeon_device *rdev);
76static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]77static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
78 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]79
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]80/* get temperature in millidegrees */
81int ci_get_temp(struct radeon_device *rdev)
82{
83 u32 temp;
84 int actual_temp = 0;
85
86 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
87 CTF_TEMP_SHIFT;
88
89 if (temp & 0x200)
90 actual_temp = 255;
91 else
92 actual_temp = temp & 0x1ff;
93
94 actual_temp = actual_temp * 1000;
95
96 return actual_temp;
97}
98
99/* get temperature in millidegrees */
100int kv_get_temp(struct radeon_device *rdev)
101{
102 u32 temp;
103 int actual_temp = 0;
104
105 temp = RREG32_SMC(0xC0300E0C);
106
107 if (temp)
108 actual_temp = (temp / 8) - 49;
109 else
110 actual_temp = 0;
111
112 actual_temp = actual_temp * 1000;
113
114 return actual_temp;
115}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]116
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]117/*
118 * Indirect registers accessor
119 */
120u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
121{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]122 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]123 u32 r;
124
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]125 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]126 WREG32(PCIE_INDEX, reg);
127 (void)RREG32(PCIE_INDEX);
128 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]129 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]130 return r;
131}
132
133void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
134{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]135 unsigned long flags;
136
137 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 WREG32(PCIE_INDEX, reg);
139 (void)RREG32(PCIE_INDEX);
140 WREG32(PCIE_DATA, v);
141 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]142 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]143}
144
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]145static const u32 spectre_rlc_save_restore_register_list[] =
146{
147 (0x0e00 << 16) | (0xc12c >> 2),
148 0x00000000,
149 (0x0e00 << 16) | (0xc140 >> 2),
150 0x00000000,
151 (0x0e00 << 16) | (0xc150 >> 2),
152 0x00000000,
153 (0x0e00 << 16) | (0xc15c >> 2),
154 0x00000000,
155 (0x0e00 << 16) | (0xc168 >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc170 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc178 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc204 >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc2b4 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc2b8 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc2bc >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc2c0 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0x8228 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0x829c >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0x869c >> 2),
176 0x00000000,
177 (0x0600 << 16) | (0x98f4 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x98f8 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x9900 >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0xc260 >> 2),
184 0x00000000,
185 (0x0e00 << 16) | (0x90e8 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x3c000 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x3c00c >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0x8c1c >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x9700 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0xcd20 >> 2),
196 0x00000000,
197 (0x4e00 << 16) | (0xcd20 >> 2),
198 0x00000000,
199 (0x5e00 << 16) | (0xcd20 >> 2),
200 0x00000000,
201 (0x6e00 << 16) | (0xcd20 >> 2),
202 0x00000000,
203 (0x7e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x8e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x9e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0xae00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0xbe00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0x89bc >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0x8900 >> 2),
216 0x00000000,
217 0x3,
218 (0x0e00 << 16) | (0xc130 >> 2),
219 0x00000000,
220 (0x0e00 << 16) | (0xc134 >> 2),
221 0x00000000,
222 (0x0e00 << 16) | (0xc1fc >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0xc208 >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0xc264 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc268 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc26c >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc270 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc274 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc278 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc27c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc280 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc284 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc288 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc28c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc290 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc294 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc298 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc29c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc2a0 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc2a4 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc2a8 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc2ac >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2b0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0x301d0 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0x30238 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0x30250 >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0x30254 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x30258 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x3025c >> 2),
277 0x00000000,
278 (0x4e00 << 16) | (0xc900 >> 2),
279 0x00000000,
280 (0x5e00 << 16) | (0xc900 >> 2),
281 0x00000000,
282 (0x6e00 << 16) | (0xc900 >> 2),
283 0x00000000,
284 (0x7e00 << 16) | (0xc900 >> 2),
285 0x00000000,
286 (0x8e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x9e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0xae00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0xbe00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x4e00 << 16) | (0xc904 >> 2),
295 0x00000000,
296 (0x5e00 << 16) | (0xc904 >> 2),
297 0x00000000,
298 (0x6e00 << 16) | (0xc904 >> 2),
299 0x00000000,
300 (0x7e00 << 16) | (0xc904 >> 2),
301 0x00000000,
302 (0x8e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x9e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0xae00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0xbe00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x4e00 << 16) | (0xc908 >> 2),
311 0x00000000,
312 (0x5e00 << 16) | (0xc908 >> 2),
313 0x00000000,
314 (0x6e00 << 16) | (0xc908 >> 2),
315 0x00000000,
316 (0x7e00 << 16) | (0xc908 >> 2),
317 0x00000000,
318 (0x8e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x9e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0xae00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0xbe00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x4e00 << 16) | (0xc90c >> 2),
327 0x00000000,
328 (0x5e00 << 16) | (0xc90c >> 2),
329 0x00000000,
330 (0x6e00 << 16) | (0xc90c >> 2),
331 0x00000000,
332 (0x7e00 << 16) | (0xc90c >> 2),
333 0x00000000,
334 (0x8e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x9e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0xae00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0xbe00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x4e00 << 16) | (0xc910 >> 2),
343 0x00000000,
344 (0x5e00 << 16) | (0xc910 >> 2),
345 0x00000000,
346 (0x6e00 << 16) | (0xc910 >> 2),
347 0x00000000,
348 (0x7e00 << 16) | (0xc910 >> 2),
349 0x00000000,
350 (0x8e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x9e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0xae00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0xbe00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc99c >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0x9834 >> 2),
361 0x00000000,
362 (0x0000 << 16) | (0x30f00 >> 2),
363 0x00000000,
364 (0x0001 << 16) | (0x30f00 >> 2),
365 0x00000000,
366 (0x0000 << 16) | (0x30f04 >> 2),
367 0x00000000,
368 (0x0001 << 16) | (0x30f04 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f08 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f08 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f0c >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f0c >> 2),
377 0x00000000,
378 (0x0600 << 16) | (0x9b7c >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0x8a14 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0x8a18 >> 2),
383 0x00000000,
384 (0x0600 << 16) | (0x30a00 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0x8bf0 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8bcc >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8b24 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30a04 >> 2),
393 0x00000000,
394 (0x0600 << 16) | (0x30a10 >> 2),
395 0x00000000,
396 (0x0600 << 16) | (0x30a14 >> 2),
397 0x00000000,
398 (0x0600 << 16) | (0x30a18 >> 2),
399 0x00000000,
400 (0x0600 << 16) | (0x30a2c >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0xc700 >> 2),
403 0x00000000,
404 (0x0e00 << 16) | (0xc704 >> 2),
405 0x00000000,
406 (0x0e00 << 16) | (0xc708 >> 2),
407 0x00000000,
408 (0x0e00 << 16) | (0xc768 >> 2),
409 0x00000000,
410 (0x0400 << 16) | (0xc770 >> 2),
411 0x00000000,
412 (0x0400 << 16) | (0xc774 >> 2),
413 0x00000000,
414 (0x0400 << 16) | (0xc778 >> 2),
415 0x00000000,
416 (0x0400 << 16) | (0xc77c >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc780 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc784 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc788 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc78c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc798 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc79c >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc7a0 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc7a4 >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc7a8 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc7ac >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7b0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7b4 >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x9100 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x3c010 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x92a8 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x92ac >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x92b4 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x92b8 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92bc >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92c0 >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92c4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92c8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92cc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92d0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x8c00 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x8c04 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x8c20 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x8c38 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c3c >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0xae00 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x9604 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xac08 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xac0c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xac10 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0xac14 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac58 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac68 >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac6c >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac70 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac74 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac78 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac7c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac80 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac84 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac88 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac8c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x970c >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x9714 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x9718 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x971c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x31068 >> 2),
519 0x00000000,
520 (0x4e00 << 16) | (0x31068 >> 2),
521 0x00000000,
522 (0x5e00 << 16) | (0x31068 >> 2),
523 0x00000000,
524 (0x6e00 << 16) | (0x31068 >> 2),
525 0x00000000,
526 (0x7e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x8e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x9e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0xae00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0xbe00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0xcd10 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xcd14 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x88b0 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0x88b4 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0x88b8 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0x88bc >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0x89c0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88c4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88c8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88d0 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0x88d4 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88d8 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x8980 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x30938 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x3093c >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x30940 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x89a0 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30900 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x30904 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x89b4 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x3c210 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x3c214 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x3c218 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x8904 >> 2),
583 0x00000000,
584 0x5,
585 (0x0e00 << 16) | (0x8c28 >> 2),
586 (0x0e00 << 16) | (0x8c2c >> 2),
587 (0x0e00 << 16) | (0x8c30 >> 2),
588 (0x0e00 << 16) | (0x8c34 >> 2),
589 (0x0e00 << 16) | (0x9600 >> 2),
590};
591
592static const u32 kalindi_rlc_save_restore_register_list[] =
593{
594 (0x0e00 << 16) | (0xc12c >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0xc140 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xc150 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xc15c >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xc168 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc170 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc204 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc2b4 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc2b8 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc2bc >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc2c0 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x8228 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x829c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x869c >> 2),
621 0x00000000,
622 (0x0600 << 16) | (0x98f4 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x98f8 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x9900 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xc260 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x90e8 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x3c000 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x3c00c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x8c1c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9700 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0xcd20 >> 2),
641 0x00000000,
642 (0x4e00 << 16) | (0xcd20 >> 2),
643 0x00000000,
644 (0x5e00 << 16) | (0xcd20 >> 2),
645 0x00000000,
646 (0x6e00 << 16) | (0xcd20 >> 2),
647 0x00000000,
648 (0x7e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x0e00 << 16) | (0x89bc >> 2),
651 0x00000000,
652 (0x0e00 << 16) | (0x8900 >> 2),
653 0x00000000,
654 0x3,
655 (0x0e00 << 16) | (0xc130 >> 2),
656 0x00000000,
657 (0x0e00 << 16) | (0xc134 >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0xc1fc >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0xc208 >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0xc264 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc268 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc26c >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc270 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc274 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc28c >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc290 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc294 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc298 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc2a0 >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc2a4 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc2a8 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc2ac >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0x301d0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0x30238 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0x30250 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0x30254 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x30258 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x3025c >> 2),
700 0x00000000,
701 (0x4e00 << 16) | (0xc900 >> 2),
702 0x00000000,
703 (0x5e00 << 16) | (0xc900 >> 2),
704 0x00000000,
705 (0x6e00 << 16) | (0xc900 >> 2),
706 0x00000000,
707 (0x7e00 << 16) | (0xc900 >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc904 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc904 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc904 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc904 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc908 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc908 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc908 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc908 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc90c >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc90c >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc90c >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc90c >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc910 >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc910 >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc910 >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc910 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc99c >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0x9834 >> 2),
744 0x00000000,
745 (0x0000 << 16) | (0x30f00 >> 2),
746 0x00000000,
747 (0x0000 << 16) | (0x30f04 >> 2),
748 0x00000000,
749 (0x0000 << 16) | (0x30f08 >> 2),
750 0x00000000,
751 (0x0000 << 16) | (0x30f0c >> 2),
752 0x00000000,
753 (0x0600 << 16) | (0x9b7c >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x8a14 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x8a18 >> 2),
758 0x00000000,
759 (0x0600 << 16) | (0x30a00 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x8bf0 >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8bcc >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8b24 >> 2),
766 0x00000000,
767 (0x0e00 << 16) | (0x30a04 >> 2),
768 0x00000000,
769 (0x0600 << 16) | (0x30a10 >> 2),
770 0x00000000,
771 (0x0600 << 16) | (0x30a14 >> 2),
772 0x00000000,
773 (0x0600 << 16) | (0x30a18 >> 2),
774 0x00000000,
775 (0x0600 << 16) | (0x30a2c >> 2),
776 0x00000000,
777 (0x0e00 << 16) | (0xc700 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0xc704 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc708 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc768 >> 2),
784 0x00000000,
785 (0x0400 << 16) | (0xc770 >> 2),
786 0x00000000,
787 (0x0400 << 16) | (0xc774 >> 2),
788 0x00000000,
789 (0x0400 << 16) | (0xc798 >> 2),
790 0x00000000,
791 (0x0400 << 16) | (0xc79c >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0x9100 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0x3c010 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0x8c00 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0x8c04 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x8c20 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x8c38 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c3c >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xae00 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x9604 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xac08 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xac0c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xac10 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0xac14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac58 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac68 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac6c >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac70 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac74 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac78 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac7c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac80 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac84 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac88 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac8c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0x970c >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0x9714 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0x9718 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0x971c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x31068 >> 2),
850 0x00000000,
851 (0x4e00 << 16) | (0x31068 >> 2),
852 0x00000000,
853 (0x5e00 << 16) | (0x31068 >> 2),
854 0x00000000,
855 (0x6e00 << 16) | (0x31068 >> 2),
856 0x00000000,
857 (0x7e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0xcd10 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0xcd14 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x88b0 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x88b4 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x88b8 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x88bc >> 2),
870 0x00000000,
871 (0x0400 << 16) | (0x89c0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88c4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88c8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88d0 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x88d4 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88d8 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8980 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x30938 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x3093c >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x30940 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x89a0 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30900 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x30904 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x89b4 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x3e1fc >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x3c210 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x3c214 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x3c218 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x8904 >> 2),
908 0x00000000,
909 0x5,
910 (0x0e00 << 16) | (0x8c28 >> 2),
911 (0x0e00 << 16) | (0x8c2c >> 2),
912 (0x0e00 << 16) | (0x8c30 >> 2),
913 (0x0e00 << 16) | (0x8c34 >> 2),
914 (0x0e00 << 16) | (0x9600 >> 2),
915};
916
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]917static const u32 bonaire_golden_spm_registers[] =
918{
919 0x30800, 0xe0ffffff, 0xe0000000
920};
921
922static const u32 bonaire_golden_common_registers[] =
923{
924 0xc770, 0xffffffff, 0x00000800,
925 0xc774, 0xffffffff, 0x00000800,
926 0xc798, 0xffffffff, 0x00007fbf,
927 0xc79c, 0xffffffff, 0x00007faf
928};
929
930static const u32 bonaire_golden_registers[] =
931{
932 0x3354, 0x00000333, 0x00000333,
933 0x3350, 0x000c0fc0, 0x00040200,
934 0x9a10, 0x00010000, 0x00058208,
935 0x3c000, 0xffff1fff, 0x00140000,
936 0x3c200, 0xfdfc0fff, 0x00000100,
937 0x3c234, 0x40000000, 0x40000200,
938 0x9830, 0xffffffff, 0x00000000,
939 0x9834, 0xf00fffff, 0x00000400,
940 0x9838, 0x0002021c, 0x00020200,
941 0xc78, 0x00000080, 0x00000000,
942 0x5bb0, 0x000000f0, 0x00000070,
943 0x5bc0, 0xf0311fff, 0x80300000,
944 0x98f8, 0x73773777, 0x12010001,
945 0x350c, 0x00810000, 0x408af000,
946 0x7030, 0x31000111, 0x00000011,
947 0x2f48, 0x73773777, 0x12010001,
948 0x220c, 0x00007fb6, 0x0021a1b1,
949 0x2210, 0x00007fb6, 0x002021b1,
950 0x2180, 0x00007fb6, 0x00002191,
951 0x2218, 0x00007fb6, 0x002121b1,
952 0x221c, 0x00007fb6, 0x002021b1,
953 0x21dc, 0x00007fb6, 0x00002191,
954 0x21e0, 0x00007fb6, 0x00002191,
955 0x3628, 0x0000003f, 0x0000000a,
956 0x362c, 0x0000003f, 0x0000000a,
957 0x2ae4, 0x00073ffe, 0x000022a2,
958 0x240c, 0x000007ff, 0x00000000,
959 0x8a14, 0xf000003f, 0x00000007,
960 0x8bf0, 0x00002001, 0x00000001,
961 0x8b24, 0xffffffff, 0x00ffffff,
962 0x30a04, 0x0000ff0f, 0x00000000,
963 0x28a4c, 0x07ffffff, 0x06000000,
964 0x4d8, 0x00000fff, 0x00000100,
965 0x3e78, 0x00000001, 0x00000002,
966 0x9100, 0x03000000, 0x0362c688,
967 0x8c00, 0x000000ff, 0x00000001,
968 0xe40, 0x00001fff, 0x00001fff,
969 0x9060, 0x0000007f, 0x00000020,
970 0x9508, 0x00010000, 0x00010000,
971 0xac14, 0x000003ff, 0x000000f3,
972 0xac0c, 0xffffffff, 0x00001032
973};
974
975static const u32 bonaire_mgcg_cgcg_init[] =
976{
977 0xc420, 0xffffffff, 0xfffffffc,
978 0x30800, 0xffffffff, 0xe0000000,
979 0x3c2a0, 0xffffffff, 0x00000100,
980 0x3c208, 0xffffffff, 0x00000100,
981 0x3c2c0, 0xffffffff, 0xc0000100,
982 0x3c2c8, 0xffffffff, 0xc0000100,
983 0x3c2c4, 0xffffffff, 0xc0000100,
984 0x55e4, 0xffffffff, 0x00600100,
985 0x3c280, 0xffffffff, 0x00000100,
986 0x3c214, 0xffffffff, 0x06000100,
987 0x3c220, 0xffffffff, 0x00000100,
988 0x3c218, 0xffffffff, 0x06000100,
989 0x3c204, 0xffffffff, 0x00000100,
990 0x3c2e0, 0xffffffff, 0x00000100,
991 0x3c224, 0xffffffff, 0x00000100,
992 0x3c200, 0xffffffff, 0x00000100,
993 0x3c230, 0xffffffff, 0x00000100,
994 0x3c234, 0xffffffff, 0x00000100,
995 0x3c250, 0xffffffff, 0x00000100,
996 0x3c254, 0xffffffff, 0x00000100,
997 0x3c258, 0xffffffff, 0x00000100,
998 0x3c25c, 0xffffffff, 0x00000100,
999 0x3c260, 0xffffffff, 0x00000100,
1000 0x3c27c, 0xffffffff, 0x00000100,
1001 0x3c278, 0xffffffff, 0x00000100,
1002 0x3c210, 0xffffffff, 0x06000100,
1003 0x3c290, 0xffffffff, 0x00000100,
1004 0x3c274, 0xffffffff, 0x00000100,
1005 0x3c2b4, 0xffffffff, 0x00000100,
1006 0x3c2b0, 0xffffffff, 0x00000100,
1007 0x3c270, 0xffffffff, 0x00000100,
1008 0x30800, 0xffffffff, 0xe0000000,
1009 0x3c020, 0xffffffff, 0x00010000,
1010 0x3c024, 0xffffffff, 0x00030002,
1011 0x3c028, 0xffffffff, 0x00040007,
1012 0x3c02c, 0xffffffff, 0x00060005,
1013 0x3c030, 0xffffffff, 0x00090008,
1014 0x3c034, 0xffffffff, 0x00010000,
1015 0x3c038, 0xffffffff, 0x00030002,
1016 0x3c03c, 0xffffffff, 0x00040007,
1017 0x3c040, 0xffffffff, 0x00060005,
1018 0x3c044, 0xffffffff, 0x00090008,
1019 0x3c048, 0xffffffff, 0x00010000,
1020 0x3c04c, 0xffffffff, 0x00030002,
1021 0x3c050, 0xffffffff, 0x00040007,
1022 0x3c054, 0xffffffff, 0x00060005,
1023 0x3c058, 0xffffffff, 0x00090008,
1024 0x3c05c, 0xffffffff, 0x00010000,
1025 0x3c060, 0xffffffff, 0x00030002,
1026 0x3c064, 0xffffffff, 0x00040007,
1027 0x3c068, 0xffffffff, 0x00060005,
1028 0x3c06c, 0xffffffff, 0x00090008,
1029 0x3c070, 0xffffffff, 0x00010000,
1030 0x3c074, 0xffffffff, 0x00030002,
1031 0x3c078, 0xffffffff, 0x00040007,
1032 0x3c07c, 0xffffffff, 0x00060005,
1033 0x3c080, 0xffffffff, 0x00090008,
1034 0x3c084, 0xffffffff, 0x00010000,
1035 0x3c088, 0xffffffff, 0x00030002,
1036 0x3c08c, 0xffffffff, 0x00040007,
1037 0x3c090, 0xffffffff, 0x00060005,
1038 0x3c094, 0xffffffff, 0x00090008,
1039 0x3c098, 0xffffffff, 0x00010000,
1040 0x3c09c, 0xffffffff, 0x00030002,
1041 0x3c0a0, 0xffffffff, 0x00040007,
1042 0x3c0a4, 0xffffffff, 0x00060005,
1043 0x3c0a8, 0xffffffff, 0x00090008,
1044 0x3c000, 0xffffffff, 0x96e00200,
1045 0x8708, 0xffffffff, 0x00900100,
1046 0xc424, 0xffffffff, 0x0020003f,
1047 0x38, 0xffffffff, 0x0140001c,
1048 0x3c, 0x000f0000, 0x000f0000,
1049 0x220, 0xffffffff, 0xC060000C,
1050 0x224, 0xc0000fff, 0x00000100,
1051 0xf90, 0xffffffff, 0x00000100,
1052 0xf98, 0x00000101, 0x00000000,
1053 0x20a8, 0xffffffff, 0x00000104,
1054 0x55e4, 0xff000fff, 0x00000100,
1055 0x30cc, 0xc0000fff, 0x00000104,
1056 0xc1e4, 0x00000001, 0x00000001,
1057 0xd00c, 0xff000ff0, 0x00000100,
1058 0xd80c, 0xff000ff0, 0x00000100
1059};
1060
1061static const u32 spectre_golden_spm_registers[] =
1062{
1063 0x30800, 0xe0ffffff, 0xe0000000
1064};
1065
1066static const u32 spectre_golden_common_registers[] =
1067{
1068 0xc770, 0xffffffff, 0x00000800,
1069 0xc774, 0xffffffff, 0x00000800,
1070 0xc798, 0xffffffff, 0x00007fbf,
1071 0xc79c, 0xffffffff, 0x00007faf
1072};
1073
1074static const u32 spectre_golden_registers[] =
1075{
1076 0x3c000, 0xffff1fff, 0x96940200,
1077 0x3c00c, 0xffff0001, 0xff000000,
1078 0x3c200, 0xfffc0fff, 0x00000100,
1079 0x6ed8, 0x00010101, 0x00010000,
1080 0x9834, 0xf00fffff, 0x00000400,
1081 0x9838, 0xfffffffc, 0x00020200,
1082 0x5bb0, 0x000000f0, 0x00000070,
1083 0x5bc0, 0xf0311fff, 0x80300000,
1084 0x98f8, 0x73773777, 0x12010001,
1085 0x9b7c, 0x00ff0000, 0x00fc0000,
1086 0x2f48, 0x73773777, 0x12010001,
1087 0x8a14, 0xf000003f, 0x00000007,
1088 0x8b24, 0xffffffff, 0x00ffffff,
1089 0x28350, 0x3f3f3fff, 0x00000082,
1090 0x28355, 0x0000003f, 0x00000000,
1091 0x3e78, 0x00000001, 0x00000002,
1092 0x913c, 0xffff03df, 0x00000004,
1093 0xc768, 0x00000008, 0x00000008,
1094 0x8c00, 0x000008ff, 0x00000800,
1095 0x9508, 0x00010000, 0x00010000,
1096 0xac0c, 0xffffffff, 0x54763210,
1097 0x214f8, 0x01ff01ff, 0x00000002,
1098 0x21498, 0x007ff800, 0x00200000,
1099 0x2015c, 0xffffffff, 0x00000f40,
1100 0x30934, 0xffffffff, 0x00000001
1101};
1102
1103static const u32 spectre_mgcg_cgcg_init[] =
1104{
1105 0xc420, 0xffffffff, 0xfffffffc,
1106 0x30800, 0xffffffff, 0xe0000000,
1107 0x3c2a0, 0xffffffff, 0x00000100,
1108 0x3c208, 0xffffffff, 0x00000100,
1109 0x3c2c0, 0xffffffff, 0x00000100,
1110 0x3c2c8, 0xffffffff, 0x00000100,
1111 0x3c2c4, 0xffffffff, 0x00000100,
1112 0x55e4, 0xffffffff, 0x00600100,
1113 0x3c280, 0xffffffff, 0x00000100,
1114 0x3c214, 0xffffffff, 0x06000100,
1115 0x3c220, 0xffffffff, 0x00000100,
1116 0x3c218, 0xffffffff, 0x06000100,
1117 0x3c204, 0xffffffff, 0x00000100,
1118 0x3c2e0, 0xffffffff, 0x00000100,
1119 0x3c224, 0xffffffff, 0x00000100,
1120 0x3c200, 0xffffffff, 0x00000100,
1121 0x3c230, 0xffffffff, 0x00000100,
1122 0x3c234, 0xffffffff, 0x00000100,
1123 0x3c250, 0xffffffff, 0x00000100,
1124 0x3c254, 0xffffffff, 0x00000100,
1125 0x3c258, 0xffffffff, 0x00000100,
1126 0x3c25c, 0xffffffff, 0x00000100,
1127 0x3c260, 0xffffffff, 0x00000100,
1128 0x3c27c, 0xffffffff, 0x00000100,
1129 0x3c278, 0xffffffff, 0x00000100,
1130 0x3c210, 0xffffffff, 0x06000100,
1131 0x3c290, 0xffffffff, 0x00000100,
1132 0x3c274, 0xffffffff, 0x00000100,
1133 0x3c2b4, 0xffffffff, 0x00000100,
1134 0x3c2b0, 0xffffffff, 0x00000100,
1135 0x3c270, 0xffffffff, 0x00000100,
1136 0x30800, 0xffffffff, 0xe0000000,
1137 0x3c020, 0xffffffff, 0x00010000,
1138 0x3c024, 0xffffffff, 0x00030002,
1139 0x3c028, 0xffffffff, 0x00040007,
1140 0x3c02c, 0xffffffff, 0x00060005,
1141 0x3c030, 0xffffffff, 0x00090008,
1142 0x3c034, 0xffffffff, 0x00010000,
1143 0x3c038, 0xffffffff, 0x00030002,
1144 0x3c03c, 0xffffffff, 0x00040007,
1145 0x3c040, 0xffffffff, 0x00060005,
1146 0x3c044, 0xffffffff, 0x00090008,
1147 0x3c048, 0xffffffff, 0x00010000,
1148 0x3c04c, 0xffffffff, 0x00030002,
1149 0x3c050, 0xffffffff, 0x00040007,
1150 0x3c054, 0xffffffff, 0x00060005,
1151 0x3c058, 0xffffffff, 0x00090008,
1152 0x3c05c, 0xffffffff, 0x00010000,
1153 0x3c060, 0xffffffff, 0x00030002,
1154 0x3c064, 0xffffffff, 0x00040007,
1155 0x3c068, 0xffffffff, 0x00060005,
1156 0x3c06c, 0xffffffff, 0x00090008,
1157 0x3c070, 0xffffffff, 0x00010000,
1158 0x3c074, 0xffffffff, 0x00030002,
1159 0x3c078, 0xffffffff, 0x00040007,
1160 0x3c07c, 0xffffffff, 0x00060005,
1161 0x3c080, 0xffffffff, 0x00090008,
1162 0x3c084, 0xffffffff, 0x00010000,
1163 0x3c088, 0xffffffff, 0x00030002,
1164 0x3c08c, 0xffffffff, 0x00040007,
1165 0x3c090, 0xffffffff, 0x00060005,
1166 0x3c094, 0xffffffff, 0x00090008,
1167 0x3c098, 0xffffffff, 0x00010000,
1168 0x3c09c, 0xffffffff, 0x00030002,
1169 0x3c0a0, 0xffffffff, 0x00040007,
1170 0x3c0a4, 0xffffffff, 0x00060005,
1171 0x3c0a8, 0xffffffff, 0x00090008,
1172 0x3c0ac, 0xffffffff, 0x00010000,
1173 0x3c0b0, 0xffffffff, 0x00030002,
1174 0x3c0b4, 0xffffffff, 0x00040007,
1175 0x3c0b8, 0xffffffff, 0x00060005,
1176 0x3c0bc, 0xffffffff, 0x00090008,
1177 0x3c000, 0xffffffff, 0x96e00200,
1178 0x8708, 0xffffffff, 0x00900100,
1179 0xc424, 0xffffffff, 0x0020003f,
1180 0x38, 0xffffffff, 0x0140001c,
1181 0x3c, 0x000f0000, 0x000f0000,
1182 0x220, 0xffffffff, 0xC060000C,
1183 0x224, 0xc0000fff, 0x00000100,
1184 0xf90, 0xffffffff, 0x00000100,
1185 0xf98, 0x00000101, 0x00000000,
1186 0x20a8, 0xffffffff, 0x00000104,
1187 0x55e4, 0xff000fff, 0x00000100,
1188 0x30cc, 0xc0000fff, 0x00000104,
1189 0xc1e4, 0x00000001, 0x00000001,
1190 0xd00c, 0xff000ff0, 0x00000100,
1191 0xd80c, 0xff000ff0, 0x00000100
1192};
1193
1194static const u32 kalindi_golden_spm_registers[] =
1195{
1196 0x30800, 0xe0ffffff, 0xe0000000
1197};
1198
1199static const u32 kalindi_golden_common_registers[] =
1200{
1201 0xc770, 0xffffffff, 0x00000800,
1202 0xc774, 0xffffffff, 0x00000800,
1203 0xc798, 0xffffffff, 0x00007fbf,
1204 0xc79c, 0xffffffff, 0x00007faf
1205};
1206
1207static const u32 kalindi_golden_registers[] =
1208{
1209 0x3c000, 0xffffdfff, 0x6e944040,
1210 0x55e4, 0xff607fff, 0xfc000100,
1211 0x3c220, 0xff000fff, 0x00000100,
1212 0x3c224, 0xff000fff, 0x00000100,
1213 0x3c200, 0xfffc0fff, 0x00000100,
1214 0x6ed8, 0x00010101, 0x00010000,
1215 0x9830, 0xffffffff, 0x00000000,
1216 0x9834, 0xf00fffff, 0x00000400,
1217 0x5bb0, 0x000000f0, 0x00000070,
1218 0x5bc0, 0xf0311fff, 0x80300000,
1219 0x98f8, 0x73773777, 0x12010001,
1220 0x98fc, 0xffffffff, 0x00000010,
1221 0x9b7c, 0x00ff0000, 0x00fc0000,
1222 0x8030, 0x00001f0f, 0x0000100a,
1223 0x2f48, 0x73773777, 0x12010001,
1224 0x2408, 0x000fffff, 0x000c007f,
1225 0x8a14, 0xf000003f, 0x00000007,
1226 0x8b24, 0x3fff3fff, 0x00ffcfff,
1227 0x30a04, 0x0000ff0f, 0x00000000,
1228 0x28a4c, 0x07ffffff, 0x06000000,
1229 0x4d8, 0x00000fff, 0x00000100,
1230 0x3e78, 0x00000001, 0x00000002,
1231 0xc768, 0x00000008, 0x00000008,
1232 0x8c00, 0x000000ff, 0x00000003,
1233 0x214f8, 0x01ff01ff, 0x00000002,
1234 0x21498, 0x007ff800, 0x00200000,
1235 0x2015c, 0xffffffff, 0x00000f40,
1236 0x88c4, 0x001f3ae3, 0x00000082,
1237 0x88d4, 0x0000001f, 0x00000010,
1238 0x30934, 0xffffffff, 0x00000000
1239};
1240
1241static const u32 kalindi_mgcg_cgcg_init[] =
1242{
1243 0xc420, 0xffffffff, 0xfffffffc,
1244 0x30800, 0xffffffff, 0xe0000000,
1245 0x3c2a0, 0xffffffff, 0x00000100,
1246 0x3c208, 0xffffffff, 0x00000100,
1247 0x3c2c0, 0xffffffff, 0x00000100,
1248 0x3c2c8, 0xffffffff, 0x00000100,
1249 0x3c2c4, 0xffffffff, 0x00000100,
1250 0x55e4, 0xffffffff, 0x00600100,
1251 0x3c280, 0xffffffff, 0x00000100,
1252 0x3c214, 0xffffffff, 0x06000100,
1253 0x3c220, 0xffffffff, 0x00000100,
1254 0x3c218, 0xffffffff, 0x06000100,
1255 0x3c204, 0xffffffff, 0x00000100,
1256 0x3c2e0, 0xffffffff, 0x00000100,
1257 0x3c224, 0xffffffff, 0x00000100,
1258 0x3c200, 0xffffffff, 0x00000100,
1259 0x3c230, 0xffffffff, 0x00000100,
1260 0x3c234, 0xffffffff, 0x00000100,
1261 0x3c250, 0xffffffff, 0x00000100,
1262 0x3c254, 0xffffffff, 0x00000100,
1263 0x3c258, 0xffffffff, 0x00000100,
1264 0x3c25c, 0xffffffff, 0x00000100,
1265 0x3c260, 0xffffffff, 0x00000100,
1266 0x3c27c, 0xffffffff, 0x00000100,
1267 0x3c278, 0xffffffff, 0x00000100,
1268 0x3c210, 0xffffffff, 0x06000100,
1269 0x3c290, 0xffffffff, 0x00000100,
1270 0x3c274, 0xffffffff, 0x00000100,
1271 0x3c2b4, 0xffffffff, 0x00000100,
1272 0x3c2b0, 0xffffffff, 0x00000100,
1273 0x3c270, 0xffffffff, 0x00000100,
1274 0x30800, 0xffffffff, 0xe0000000,
1275 0x3c020, 0xffffffff, 0x00010000,
1276 0x3c024, 0xffffffff, 0x00030002,
1277 0x3c028, 0xffffffff, 0x00040007,
1278 0x3c02c, 0xffffffff, 0x00060005,
1279 0x3c030, 0xffffffff, 0x00090008,
1280 0x3c034, 0xffffffff, 0x00010000,
1281 0x3c038, 0xffffffff, 0x00030002,
1282 0x3c03c, 0xffffffff, 0x00040007,
1283 0x3c040, 0xffffffff, 0x00060005,
1284 0x3c044, 0xffffffff, 0x00090008,
1285 0x3c000, 0xffffffff, 0x96e00200,
1286 0x8708, 0xffffffff, 0x00900100,
1287 0xc424, 0xffffffff, 0x0020003f,
1288 0x38, 0xffffffff, 0x0140001c,
1289 0x3c, 0x000f0000, 0x000f0000,
1290 0x220, 0xffffffff, 0xC060000C,
1291 0x224, 0xc0000fff, 0x00000100,
1292 0x20a8, 0xffffffff, 0x00000104,
1293 0x55e4, 0xff000fff, 0x00000100,
1294 0x30cc, 0xc0000fff, 0x00000104,
1295 0xc1e4, 0x00000001, 0x00000001,
1296 0xd00c, 0xff000ff0, 0x00000100,
1297 0xd80c, 0xff000ff0, 0x00000100
1298};
1299
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1300static const u32 hawaii_golden_spm_registers[] =
1301{
1302 0x30800, 0xe0ffffff, 0xe0000000
1303};
1304
1305static const u32 hawaii_golden_common_registers[] =
1306{
1307 0x30800, 0xffffffff, 0xe0000000,
1308 0x28350, 0xffffffff, 0x3a00161a,
1309 0x28354, 0xffffffff, 0x0000002e,
1310 0x9a10, 0xffffffff, 0x00018208,
1311 0x98f8, 0xffffffff, 0x12011003
1312};
1313
1314static const u32 hawaii_golden_registers[] =
1315{
1316 0x3354, 0x00000333, 0x00000333,
1317 0x9a10, 0x00010000, 0x00058208,
1318 0x9830, 0xffffffff, 0x00000000,
1319 0x9834, 0xf00fffff, 0x00000400,
1320 0x9838, 0x0002021c, 0x00020200,
1321 0xc78, 0x00000080, 0x00000000,
1322 0x5bb0, 0x000000f0, 0x00000070,
1323 0x5bc0, 0xf0311fff, 0x80300000,
1324 0x350c, 0x00810000, 0x408af000,
1325 0x7030, 0x31000111, 0x00000011,
1326 0x2f48, 0x73773777, 0x12010001,
1327 0x2120, 0x0000007f, 0x0000001b,
1328 0x21dc, 0x00007fb6, 0x00002191,
1329 0x3628, 0x0000003f, 0x0000000a,
1330 0x362c, 0x0000003f, 0x0000000a,
1331 0x2ae4, 0x00073ffe, 0x000022a2,
1332 0x240c, 0x000007ff, 0x00000000,
1333 0x8bf0, 0x00002001, 0x00000001,
1334 0x8b24, 0xffffffff, 0x00ffffff,
1335 0x30a04, 0x0000ff0f, 0x00000000,
1336 0x28a4c, 0x07ffffff, 0x06000000,
1337 0x3e78, 0x00000001, 0x00000002,
1338 0xc768, 0x00000008, 0x00000008,
1339 0xc770, 0x00000f00, 0x00000800,
1340 0xc774, 0x00000f00, 0x00000800,
1341 0xc798, 0x00ffffff, 0x00ff7fbf,
1342 0xc79c, 0x00ffffff, 0x00ff7faf,
1343 0x8c00, 0x000000ff, 0x00000800,
1344 0xe40, 0x00001fff, 0x00001fff,
1345 0x9060, 0x0000007f, 0x00000020,
1346 0x9508, 0x00010000, 0x00010000,
1347 0xae00, 0x00100000, 0x000ff07c,
1348 0xac14, 0x000003ff, 0x0000000f,
1349 0xac10, 0xffffffff, 0x7564fdec,
1350 0xac0c, 0xffffffff, 0x3120b9a8,
1351 0xac08, 0x20000000, 0x0f9c0000
1352};
1353
1354static const u32 hawaii_mgcg_cgcg_init[] =
1355{
1356 0xc420, 0xffffffff, 0xfffffffd,
1357 0x30800, 0xffffffff, 0xe0000000,
1358 0x3c2a0, 0xffffffff, 0x00000100,
1359 0x3c208, 0xffffffff, 0x00000100,
1360 0x3c2c0, 0xffffffff, 0x00000100,
1361 0x3c2c8, 0xffffffff, 0x00000100,
1362 0x3c2c4, 0xffffffff, 0x00000100,
1363 0x55e4, 0xffffffff, 0x00200100,
1364 0x3c280, 0xffffffff, 0x00000100,
1365 0x3c214, 0xffffffff, 0x06000100,
1366 0x3c220, 0xffffffff, 0x00000100,
1367 0x3c218, 0xffffffff, 0x06000100,
1368 0x3c204, 0xffffffff, 0x00000100,
1369 0x3c2e0, 0xffffffff, 0x00000100,
1370 0x3c224, 0xffffffff, 0x00000100,
1371 0x3c200, 0xffffffff, 0x00000100,
1372 0x3c230, 0xffffffff, 0x00000100,
1373 0x3c234, 0xffffffff, 0x00000100,
1374 0x3c250, 0xffffffff, 0x00000100,
1375 0x3c254, 0xffffffff, 0x00000100,
1376 0x3c258, 0xffffffff, 0x00000100,
1377 0x3c25c, 0xffffffff, 0x00000100,
1378 0x3c260, 0xffffffff, 0x00000100,
1379 0x3c27c, 0xffffffff, 0x00000100,
1380 0x3c278, 0xffffffff, 0x00000100,
1381 0x3c210, 0xffffffff, 0x06000100,
1382 0x3c290, 0xffffffff, 0x00000100,
1383 0x3c274, 0xffffffff, 0x00000100,
1384 0x3c2b4, 0xffffffff, 0x00000100,
1385 0x3c2b0, 0xffffffff, 0x00000100,
1386 0x3c270, 0xffffffff, 0x00000100,
1387 0x30800, 0xffffffff, 0xe0000000,
1388 0x3c020, 0xffffffff, 0x00010000,
1389 0x3c024, 0xffffffff, 0x00030002,
1390 0x3c028, 0xffffffff, 0x00040007,
1391 0x3c02c, 0xffffffff, 0x00060005,
1392 0x3c030, 0xffffffff, 0x00090008,
1393 0x3c034, 0xffffffff, 0x00010000,
1394 0x3c038, 0xffffffff, 0x00030002,
1395 0x3c03c, 0xffffffff, 0x00040007,
1396 0x3c040, 0xffffffff, 0x00060005,
1397 0x3c044, 0xffffffff, 0x00090008,
1398 0x3c048, 0xffffffff, 0x00010000,
1399 0x3c04c, 0xffffffff, 0x00030002,
1400 0x3c050, 0xffffffff, 0x00040007,
1401 0x3c054, 0xffffffff, 0x00060005,
1402 0x3c058, 0xffffffff, 0x00090008,
1403 0x3c05c, 0xffffffff, 0x00010000,
1404 0x3c060, 0xffffffff, 0x00030002,
1405 0x3c064, 0xffffffff, 0x00040007,
1406 0x3c068, 0xffffffff, 0x00060005,
1407 0x3c06c, 0xffffffff, 0x00090008,
1408 0x3c070, 0xffffffff, 0x00010000,
1409 0x3c074, 0xffffffff, 0x00030002,
1410 0x3c078, 0xffffffff, 0x00040007,
1411 0x3c07c, 0xffffffff, 0x00060005,
1412 0x3c080, 0xffffffff, 0x00090008,
1413 0x3c084, 0xffffffff, 0x00010000,
1414 0x3c088, 0xffffffff, 0x00030002,
1415 0x3c08c, 0xffffffff, 0x00040007,
1416 0x3c090, 0xffffffff, 0x00060005,
1417 0x3c094, 0xffffffff, 0x00090008,
1418 0x3c098, 0xffffffff, 0x00010000,
1419 0x3c09c, 0xffffffff, 0x00030002,
1420 0x3c0a0, 0xffffffff, 0x00040007,
1421 0x3c0a4, 0xffffffff, 0x00060005,
1422 0x3c0a8, 0xffffffff, 0x00090008,
1423 0x3c0ac, 0xffffffff, 0x00010000,
1424 0x3c0b0, 0xffffffff, 0x00030002,
1425 0x3c0b4, 0xffffffff, 0x00040007,
1426 0x3c0b8, 0xffffffff, 0x00060005,
1427 0x3c0bc, 0xffffffff, 0x00090008,
1428 0x3c0c0, 0xffffffff, 0x00010000,
1429 0x3c0c4, 0xffffffff, 0x00030002,
1430 0x3c0c8, 0xffffffff, 0x00040007,
1431 0x3c0cc, 0xffffffff, 0x00060005,
1432 0x3c0d0, 0xffffffff, 0x00090008,
1433 0x3c0d4, 0xffffffff, 0x00010000,
1434 0x3c0d8, 0xffffffff, 0x00030002,
1435 0x3c0dc, 0xffffffff, 0x00040007,
1436 0x3c0e0, 0xffffffff, 0x00060005,
1437 0x3c0e4, 0xffffffff, 0x00090008,
1438 0x3c0e8, 0xffffffff, 0x00010000,
1439 0x3c0ec, 0xffffffff, 0x00030002,
1440 0x3c0f0, 0xffffffff, 0x00040007,
1441 0x3c0f4, 0xffffffff, 0x00060005,
1442 0x3c0f8, 0xffffffff, 0x00090008,
1443 0xc318, 0xffffffff, 0x00020200,
1444 0x3350, 0xffffffff, 0x00000200,
1445 0x15c0, 0xffffffff, 0x00000400,
1446 0x55e8, 0xffffffff, 0x00000000,
1447 0x2f50, 0xffffffff, 0x00000902,
1448 0x3c000, 0xffffffff, 0x96940200,
1449 0x8708, 0xffffffff, 0x00900100,
1450 0xc424, 0xffffffff, 0x0020003f,
1451 0x38, 0xffffffff, 0x0140001c,
1452 0x3c, 0x000f0000, 0x000f0000,
1453 0x220, 0xffffffff, 0xc060000c,
1454 0x224, 0xc0000fff, 0x00000100,
1455 0xf90, 0xffffffff, 0x00000100,
1456 0xf98, 0x00000101, 0x00000000,
1457 0x20a8, 0xffffffff, 0x00000104,
1458 0x55e4, 0xff000fff, 0x00000100,
1459 0x30cc, 0xc0000fff, 0x00000104,
1460 0xc1e4, 0x00000001, 0x00000001,
1461 0xd00c, 0xff000ff0, 0x00000100,
1462 0xd80c, 0xff000ff0, 0x00000100
1463};
1464
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1465static void cik_init_golden_registers(struct radeon_device *rdev)
1466{
1467 switch (rdev->family) {
1468 case CHIP_BONAIRE:
1469 radeon_program_register_sequence(rdev,
1470 bonaire_mgcg_cgcg_init,
1471 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1472 radeon_program_register_sequence(rdev,
1473 bonaire_golden_registers,
1474 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1475 radeon_program_register_sequence(rdev,
1476 bonaire_golden_common_registers,
1477 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1478 radeon_program_register_sequence(rdev,
1479 bonaire_golden_spm_registers,
1480 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1481 break;
1482 case CHIP_KABINI:
1483 radeon_program_register_sequence(rdev,
1484 kalindi_mgcg_cgcg_init,
1485 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1486 radeon_program_register_sequence(rdev,
1487 kalindi_golden_registers,
1488 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1489 radeon_program_register_sequence(rdev,
1490 kalindi_golden_common_registers,
1491 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1492 radeon_program_register_sequence(rdev,
1493 kalindi_golden_spm_registers,
1494 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1495 break;
1496 case CHIP_KAVERI:
1497 radeon_program_register_sequence(rdev,
1498 spectre_mgcg_cgcg_init,
1499 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1500 radeon_program_register_sequence(rdev,
1501 spectre_golden_registers,
1502 (const u32)ARRAY_SIZE(spectre_golden_registers));
1503 radeon_program_register_sequence(rdev,
1504 spectre_golden_common_registers,
1505 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1506 radeon_program_register_sequence(rdev,
1507 spectre_golden_spm_registers,
1508 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1509 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame]1510 case CHIP_HAWAII:
1511 radeon_program_register_sequence(rdev,
1512 hawaii_mgcg_cgcg_init,
1513 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1514 radeon_program_register_sequence(rdev,
1515 hawaii_golden_registers,
1516 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1517 radeon_program_register_sequence(rdev,
1518 hawaii_golden_common_registers,
1519 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1520 radeon_program_register_sequence(rdev,
1521 hawaii_golden_spm_registers,
1522 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1523 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1524 default:
1525 break;
1526 }
1527}
1528
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1529/**
1530 * cik_get_xclk - get the xclk
1531 *
1532 * @rdev: radeon_device pointer
1533 *
1534 * Returns the reference clock used by the gfx engine
1535 * (CIK).
1536 */
1537u32 cik_get_xclk(struct radeon_device *rdev)
1538{
1539 u32 reference_clock = rdev->clock.spll.reference_freq;
1540
1541 if (rdev->flags & RADEON_IS_IGP) {
1542 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1543 return reference_clock / 2;
1544 } else {
1545 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1546 return reference_clock / 4;
1547 }
1548 return reference_clock;
1549}
1550
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1551/**
1552 * cik_mm_rdoorbell - read a doorbell dword
1553 *
1554 * @rdev: radeon_device pointer
1555 * @offset: byte offset into the aperture
1556 *
1557 * Returns the value in the doorbell aperture at the
1558 * requested offset (CIK).
1559 */
1560u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1561{
1562 if (offset < rdev->doorbell.size) {
1563 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1564 } else {
1565 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1566 return 0;
1567 }
1568}
1569
1570/**
1571 * cik_mm_wdoorbell - write a doorbell dword
1572 *
1573 * @rdev: radeon_device pointer
1574 * @offset: byte offset into the aperture
1575 * @v: value to write
1576 *
1577 * Writes @v to the doorbell aperture at the
1578 * requested offset (CIK).
1579 */
1580void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1581{
1582 if (offset < rdev->doorbell.size) {
1583 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1584 } else {
1585 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1586 }
1587}
1588
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1589#define BONAIRE_IO_MC_REGS_SIZE 36
1590
1591static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1592{
1593 {0x00000070, 0x04400000},
1594 {0x00000071, 0x80c01803},
1595 {0x00000072, 0x00004004},
1596 {0x00000073, 0x00000100},
1597 {0x00000074, 0x00ff0000},
1598 {0x00000075, 0x34000000},
1599 {0x00000076, 0x08000014},
1600 {0x00000077, 0x00cc08ec},
1601 {0x00000078, 0x00000400},
1602 {0x00000079, 0x00000000},
1603 {0x0000007a, 0x04090000},
1604 {0x0000007c, 0x00000000},
1605 {0x0000007e, 0x4408a8e8},
1606 {0x0000007f, 0x00000304},
1607 {0x00000080, 0x00000000},
1608 {0x00000082, 0x00000001},
1609 {0x00000083, 0x00000002},
1610 {0x00000084, 0xf3e4f400},
1611 {0x00000085, 0x052024e3},
1612 {0x00000087, 0x00000000},
1613 {0x00000088, 0x01000000},
1614 {0x0000008a, 0x1c0a0000},
1615 {0x0000008b, 0xff010000},
1616 {0x0000008d, 0xffffefff},
1617 {0x0000008e, 0xfff3efff},
1618 {0x0000008f, 0xfff3efbf},
1619 {0x00000092, 0xf7ffffff},
1620 {0x00000093, 0xffffff7f},
1621 {0x00000095, 0x00101101},
1622 {0x00000096, 0x00000fff},
1623 {0x00000097, 0x00116fff},
1624 {0x00000098, 0x60010000},
1625 {0x00000099, 0x10010000},
1626 {0x0000009a, 0x00006000},
1627 {0x0000009b, 0x00001000},
1628 {0x0000009f, 0x00b48000}
1629};
1630
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1631/**
1632 * cik_srbm_select - select specific register instances
1633 *
1634 * @rdev: radeon_device pointer
1635 * @me: selected ME (micro engine)
1636 * @pipe: pipe
1637 * @queue: queue
1638 * @vmid: VMID
1639 *
1640 * Switches the currently active registers instances. Some
1641 * registers are instanced per VMID, others are instanced per
1642 * me/pipe/queue combination.
1643 */
1644static void cik_srbm_select(struct radeon_device *rdev,
1645 u32 me, u32 pipe, u32 queue, u32 vmid)
1646{
1647 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1648 MEID(me & 0x3) |
1649 VMID(vmid & 0xf) |
1650 QUEUEID(queue & 0x7));
1651 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1652}
1653
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1654/* ucode loading */
1655/**
1656 * ci_mc_load_microcode - load MC ucode into the hw
1657 *
1658 * @rdev: radeon_device pointer
1659 *
1660 * Load the GDDR MC ucode into the hw (CIK).
1661 * Returns 0 on success, error on failure.
1662 */
1663static int ci_mc_load_microcode(struct radeon_device *rdev)
1664{
1665 const __be32 *fw_data;
1666 u32 running, blackout = 0;
1667 u32 *io_mc_regs;
1668 int i, ucode_size, regs_size;
1669
1670 if (!rdev->mc_fw)
1671 return -EINVAL;
1672
1673 switch (rdev->family) {
1674 case CHIP_BONAIRE:
1675 default:
1676 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1677 ucode_size = CIK_MC_UCODE_SIZE;
1678 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1679 break;
1680 }
1681
1682 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1683
1684 if (running == 0) {
1685 if (running) {
1686 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1687 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1688 }
1689
1690 /* reset the engine and set to writable */
1691 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1692 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1693
1694 /* load mc io regs */
1695 for (i = 0; i < regs_size; i++) {
1696 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1697 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1698 }
1699 /* load the MC ucode */
1700 fw_data = (const __be32 *)rdev->mc_fw->data;
1701 for (i = 0; i < ucode_size; i++)
1702 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1703
1704 /* put the engine back into the active state */
1705 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1706 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1707 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1708
1709 /* wait for training to complete */
1710 for (i = 0; i < rdev->usec_timeout; i++) {
1711 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1712 break;
1713 udelay(1);
1714 }
1715 for (i = 0; i < rdev->usec_timeout; i++) {
1716 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1717 break;
1718 udelay(1);
1719 }
1720
1721 if (running)
1722 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1723 }
1724
1725 return 0;
1726}
1727
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1728/**
1729 * cik_init_microcode - load ucode images from disk
1730 *
1731 * @rdev: radeon_device pointer
1732 *
1733 * Use the firmware interface to load the ucode images into
1734 * the driver (not loaded into hw).
1735 * Returns 0 on success, error on failure.
1736 */
1737static int cik_init_microcode(struct radeon_device *rdev)
1738{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1739 const char *chip_name;
1740 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1741 mec_req_size, rlc_req_size, mc_req_size,
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1742 sdma_req_size, smc_req_size;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1743 char fw_name[30];
1744 int err;
1745
1746 DRM_DEBUG("\n");
1747
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1748 switch (rdev->family) {
1749 case CHIP_BONAIRE:
1750 chip_name = "BONAIRE";
1751 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1752 me_req_size = CIK_ME_UCODE_SIZE * 4;
1753 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1754 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1755 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1756 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1757 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1758 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1759 break;
1760 case CHIP_KAVERI:
1761 chip_name = "KAVERI";
1762 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1763 me_req_size = CIK_ME_UCODE_SIZE * 4;
1764 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1765 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1766 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1767 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1768 break;
1769 case CHIP_KABINI:
1770 chip_name = "KABINI";
1771 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1772 me_req_size = CIK_ME_UCODE_SIZE * 4;
1773 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1774 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1775 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1776 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1777 break;
1778 default: BUG();
1779 }
1780
1781 DRM_INFO("Loading %s Microcode\n", chip_name);
1782
1783 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1784 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1785 if (err)
1786 goto out;
1787 if (rdev->pfp_fw->size != pfp_req_size) {
1788 printk(KERN_ERR
1789 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1790 rdev->pfp_fw->size, fw_name);
1791 err = -EINVAL;
1792 goto out;
1793 }
1794
1795 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1796 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1797 if (err)
1798 goto out;
1799 if (rdev->me_fw->size != me_req_size) {
1800 printk(KERN_ERR
1801 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1802 rdev->me_fw->size, fw_name);
1803 err = -EINVAL;
1804 }
1805
1806 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1807 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1808 if (err)
1809 goto out;
1810 if (rdev->ce_fw->size != ce_req_size) {
1811 printk(KERN_ERR
1812 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1813 rdev->ce_fw->size, fw_name);
1814 err = -EINVAL;
1815 }
1816
1817 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1818 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1819 if (err)
1820 goto out;
1821 if (rdev->mec_fw->size != mec_req_size) {
1822 printk(KERN_ERR
1823 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1824 rdev->mec_fw->size, fw_name);
1825 err = -EINVAL;
1826 }
1827
1828 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1829 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1830 if (err)
1831 goto out;
1832 if (rdev->rlc_fw->size != rlc_req_size) {
1833 printk(KERN_ERR
1834 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1835 rdev->rlc_fw->size, fw_name);
1836 err = -EINVAL;
1837 }
1838
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1839 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1840 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1841 if (err)
1842 goto out;
1843 if (rdev->sdma_fw->size != sdma_req_size) {
1844 printk(KERN_ERR
1845 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1846 rdev->sdma_fw->size, fw_name);
1847 err = -EINVAL;
1848 }
1849
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1850 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (!(rdev->flags & RADEON_IS_IGP)) {
1852 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1853 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1854 if (err)
1855 goto out;
1856 if (rdev->mc_fw->size != mc_req_size) {
1857 printk(KERN_ERR
1858 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1859 rdev->mc_fw->size, fw_name);
1860 err = -EINVAL;
1861 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1862
1863 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1864 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1865 if (err) {
1866 printk(KERN_ERR
1867 "smc: error loading firmware \"%s\"\n",
1868 fw_name);
1869 release_firmware(rdev->smc_fw);
1870 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1871 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1872 } else if (rdev->smc_fw->size != smc_req_size) {
1873 printk(KERN_ERR
1874 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1875 rdev->smc_fw->size, fw_name);
1876 err = -EINVAL;
1877 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1878 }
1879
1880out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1881 if (err) {
1882 if (err != -EINVAL)
1883 printk(KERN_ERR
1884 "cik_cp: Failed to load firmware \"%s\"\n",
1885 fw_name);
1886 release_firmware(rdev->pfp_fw);
1887 rdev->pfp_fw = NULL;
1888 release_firmware(rdev->me_fw);
1889 rdev->me_fw = NULL;
1890 release_firmware(rdev->ce_fw);
1891 rdev->ce_fw = NULL;
1892 release_firmware(rdev->rlc_fw);
1893 rdev->rlc_fw = NULL;
1894 release_firmware(rdev->mc_fw);
1895 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1896 release_firmware(rdev->smc_fw);
1897 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1898 }
1899 return err;
1900}
1901
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1902/*
1903 * Core functions
1904 */
1905/**
1906 * cik_tiling_mode_table_init - init the hw tiling table
1907 *
1908 * @rdev: radeon_device pointer
1909 *
1910 * Starting with SI, the tiling setup is done globally in a
1911 * set of 32 tiling modes. Rather than selecting each set of
1912 * parameters per surface as on older asics, we just select
1913 * which index in the tiling table we want to use, and the
1914 * surface uses those parameters (CIK).
1915 */
1916static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1917{
1918 const u32 num_tile_mode_states = 32;
1919 const u32 num_secondary_tile_mode_states = 16;
1920 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1921 u32 num_pipe_configs;
1922 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1923 rdev->config.cik.max_shader_engines;
1924
1925 switch (rdev->config.cik.mem_row_size_in_kb) {
1926 case 1:
1927 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1928 break;
1929 case 2:
1930 default:
1931 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1932 break;
1933 case 4:
1934 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1935 break;
1936 }
1937
1938 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1939 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1940 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1941
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1942 if (num_pipe_configs == 16) {
1943 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1944 switch (reg_offset) {
1945 case 0:
1946 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1947 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1948 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1949 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1950 break;
1951 case 1:
1952 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1953 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1954 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1955 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1956 break;
1957 case 2:
1958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1959 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1960 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1961 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1962 break;
1963 case 3:
1964 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1965 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1966 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1967 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1968 break;
1969 case 4:
1970 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1971 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1972 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1973 TILE_SPLIT(split_equal_to_row_size));
1974 break;
1975 case 5:
1976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1977 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1978 break;
1979 case 6:
1980 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1981 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1982 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1983 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1984 break;
1985 case 7:
1986 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1987 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1988 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1989 TILE_SPLIT(split_equal_to_row_size));
1990 break;
1991 case 8:
1992 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1993 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
1994 break;
1995 case 9:
1996 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1997 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1998 break;
1999 case 10:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2004 break;
2005 case 11:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2009 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2010 break;
2011 case 12:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2016 break;
2017 case 13:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2020 break;
2021 case 14:
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2025 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2026 break;
2027 case 16:
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2031 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2032 break;
2033 case 17:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2038 break;
2039 case 27:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2042 break;
2043 case 28:
2044 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2045 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2046 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2047 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2048 break;
2049 case 29:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2052 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2053 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2054 break;
2055 case 30:
2056 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2057 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2058 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2059 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2060 break;
2061 default:
2062 gb_tile_moden = 0;
2063 break;
2064 }
2065 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2066 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2067 }
2068 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2069 switch (reg_offset) {
2070 case 0:
2071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2074 NUM_BANKS(ADDR_SURF_16_BANK));
2075 break;
2076 case 1:
2077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2080 NUM_BANKS(ADDR_SURF_16_BANK));
2081 break;
2082 case 2:
2083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2086 NUM_BANKS(ADDR_SURF_16_BANK));
2087 break;
2088 case 3:
2089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2092 NUM_BANKS(ADDR_SURF_16_BANK));
2093 break;
2094 case 4:
2095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2098 NUM_BANKS(ADDR_SURF_8_BANK));
2099 break;
2100 case 5:
2101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2104 NUM_BANKS(ADDR_SURF_4_BANK));
2105 break;
2106 case 6:
2107 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2108 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2109 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2110 NUM_BANKS(ADDR_SURF_2_BANK));
2111 break;
2112 case 8:
2113 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2114 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2115 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2116 NUM_BANKS(ADDR_SURF_16_BANK));
2117 break;
2118 case 9:
2119 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2120 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2121 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2122 NUM_BANKS(ADDR_SURF_16_BANK));
2123 break;
2124 case 10:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 11:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2134 NUM_BANKS(ADDR_SURF_8_BANK));
2135 break;
2136 case 12:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_4_BANK));
2141 break;
2142 case 13:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_2_BANK));
2147 break;
2148 case 14:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_2_BANK));
2153 break;
2154 default:
2155 gb_tile_moden = 0;
2156 break;
2157 }
2158 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2159 }
2160 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2161 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2162 switch (reg_offset) {
2163 case 0:
2164 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2166 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2167 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2168 break;
2169 case 1:
2170 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2171 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2172 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2173 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2174 break;
2175 case 2:
2176 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2177 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2178 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2179 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2180 break;
2181 case 3:
2182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2183 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2184 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2185 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2186 break;
2187 case 4:
2188 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2189 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2190 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2191 TILE_SPLIT(split_equal_to_row_size));
2192 break;
2193 case 5:
2194 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2195 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2196 break;
2197 case 6:
2198 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2199 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2200 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2201 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2202 break;
2203 case 7:
2204 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2205 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2206 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2207 TILE_SPLIT(split_equal_to_row_size));
2208 break;
2209 case 8:
2210 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2211 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2212 break;
2213 case 9:
2214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2215 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2216 break;
2217 case 10:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2222 break;
2223 case 11:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2228 break;
2229 case 12:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2234 break;
2235 case 13:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2238 break;
2239 case 14:
2240 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2241 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2242 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2243 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2244 break;
2245 case 16:
2246 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2247 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2248 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2249 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2250 break;
2251 case 17:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2256 break;
2257 case 27:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2260 break;
2261 case 28:
2262 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2263 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2264 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2266 break;
2267 case 29:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2271 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2272 break;
2273 case 30:
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2278 break;
2279 default:
2280 gb_tile_moden = 0;
2281 break;
2282 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2283 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2284 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2285 }
2286 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2287 switch (reg_offset) {
2288 case 0:
2289 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2292 NUM_BANKS(ADDR_SURF_16_BANK));
2293 break;
2294 case 1:
2295 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2298 NUM_BANKS(ADDR_SURF_16_BANK));
2299 break;
2300 case 2:
2301 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2304 NUM_BANKS(ADDR_SURF_16_BANK));
2305 break;
2306 case 3:
2307 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2308 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2309 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2310 NUM_BANKS(ADDR_SURF_16_BANK));
2311 break;
2312 case 4:
2313 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2316 NUM_BANKS(ADDR_SURF_8_BANK));
2317 break;
2318 case 5:
2319 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2320 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2321 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2322 NUM_BANKS(ADDR_SURF_4_BANK));
2323 break;
2324 case 6:
2325 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2328 NUM_BANKS(ADDR_SURF_2_BANK));
2329 break;
2330 case 8:
2331 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2332 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2333 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2334 NUM_BANKS(ADDR_SURF_16_BANK));
2335 break;
2336 case 9:
2337 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2338 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2339 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2340 NUM_BANKS(ADDR_SURF_16_BANK));
2341 break;
2342 case 10:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 11:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 12:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2358 NUM_BANKS(ADDR_SURF_8_BANK));
2359 break;
2360 case 13:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2364 NUM_BANKS(ADDR_SURF_4_BANK));
2365 break;
2366 case 14:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_2_BANK));
2371 break;
2372 default:
2373 gb_tile_moden = 0;
2374 break;
2375 }
2376 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2377 }
2378 } else if (num_pipe_configs == 4) {
2379 if (num_rbs == 4) {
2380 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2381 switch (reg_offset) {
2382 case 0:
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2384 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2386 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2387 break;
2388 case 1:
2389 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2392 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2393 break;
2394 case 2:
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2398 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2399 break;
2400 case 3:
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2405 break;
2406 case 4:
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2410 TILE_SPLIT(split_equal_to_row_size));
2411 break;
2412 case 5:
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2415 break;
2416 case 6:
2417 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2420 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2421 break;
2422 case 7:
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2424 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2425 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2426 TILE_SPLIT(split_equal_to_row_size));
2427 break;
2428 case 8:
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2430 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2431 break;
2432 case 9:
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2435 break;
2436 case 10:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2441 break;
2442 case 11:
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2446 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447 break;
2448 case 12:
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2453 break;
2454 case 13:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2457 break;
2458 case 14:
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2462 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463 break;
2464 case 16:
2465 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2466 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2467 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2468 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2469 break;
2470 case 17:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2475 break;
2476 case 27:
2477 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2478 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2479 break;
2480 case 28:
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2485 break;
2486 case 29:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2489 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2490 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2491 break;
2492 case 30:
2493 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2494 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2495 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2497 break;
2498 default:
2499 gb_tile_moden = 0;
2500 break;
2501 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2502 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2503 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2504 }
2505 } else if (num_rbs < 4) {
2506 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2507 switch (reg_offset) {
2508 case 0:
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2513 break;
2514 case 1:
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2518 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2519 break;
2520 case 2:
2521 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2525 break;
2526 case 3:
2527 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2528 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2529 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2530 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2531 break;
2532 case 4:
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2536 TILE_SPLIT(split_equal_to_row_size));
2537 break;
2538 case 5:
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2541 break;
2542 case 6:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2547 break;
2548 case 7:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 8:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2556 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2557 break;
2558 case 9:
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2561 break;
2562 case 10:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2567 break;
2568 case 11:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2573 break;
2574 case 12:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2579 break;
2580 case 13:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2583 break;
2584 case 14:
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2589 break;
2590 case 16:
2591 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2595 break;
2596 case 17:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2601 break;
2602 case 27:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2605 break;
2606 case 28:
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2611 break;
2612 case 29:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2616 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2617 break;
2618 case 30:
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2622 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2623 break;
2624 default:
2625 gb_tile_moden = 0;
2626 break;
2627 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2628 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2629 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2630 }
2631 }
2632 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2633 switch (reg_offset) {
2634 case 0:
2635 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2638 NUM_BANKS(ADDR_SURF_16_BANK));
2639 break;
2640 case 1:
2641 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2643 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2644 NUM_BANKS(ADDR_SURF_16_BANK));
2645 break;
2646 case 2:
2647 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2650 NUM_BANKS(ADDR_SURF_16_BANK));
2651 break;
2652 case 3:
2653 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2656 NUM_BANKS(ADDR_SURF_16_BANK));
2657 break;
2658 case 4:
2659 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2660 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2661 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2662 NUM_BANKS(ADDR_SURF_16_BANK));
2663 break;
2664 case 5:
2665 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2668 NUM_BANKS(ADDR_SURF_8_BANK));
2669 break;
2670 case 6:
2671 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2673 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2674 NUM_BANKS(ADDR_SURF_4_BANK));
2675 break;
2676 case 8:
2677 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2680 NUM_BANKS(ADDR_SURF_16_BANK));
2681 break;
2682 case 9:
2683 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2684 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2685 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2686 NUM_BANKS(ADDR_SURF_16_BANK));
2687 break;
2688 case 10:
2689 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692 NUM_BANKS(ADDR_SURF_16_BANK));
2693 break;
2694 case 11:
2695 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698 NUM_BANKS(ADDR_SURF_16_BANK));
2699 break;
2700 case 12:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 13:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_8_BANK));
2711 break;
2712 case 14:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2716 NUM_BANKS(ADDR_SURF_4_BANK));
2717 break;
2718 default:
2719 gb_tile_moden = 0;
2720 break;
2721 }
2722 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2723 }
2724 } else if (num_pipe_configs == 2) {
2725 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2726 switch (reg_offset) {
2727 case 0:
2728 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2730 PIPE_CONFIG(ADDR_SURF_P2) |
2731 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2732 break;
2733 case 1:
2734 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2735 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2736 PIPE_CONFIG(ADDR_SURF_P2) |
2737 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2738 break;
2739 case 2:
2740 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742 PIPE_CONFIG(ADDR_SURF_P2) |
2743 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2744 break;
2745 case 3:
2746 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2747 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2748 PIPE_CONFIG(ADDR_SURF_P2) |
2749 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2750 break;
2751 case 4:
2752 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754 PIPE_CONFIG(ADDR_SURF_P2) |
2755 TILE_SPLIT(split_equal_to_row_size));
2756 break;
2757 case 5:
2758 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2759 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2760 break;
2761 case 6:
2762 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2763 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2764 PIPE_CONFIG(ADDR_SURF_P2) |
2765 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2766 break;
2767 case 7:
2768 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P2) |
2771 TILE_SPLIT(split_equal_to_row_size));
2772 break;
2773 case 8:
2774 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2775 break;
2776 case 9:
2777 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2778 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2779 break;
2780 case 10:
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P2) |
2784 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2785 break;
2786 case 11:
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P2) |
2790 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2791 break;
2792 case 12:
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2794 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2795 PIPE_CONFIG(ADDR_SURF_P2) |
2796 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2797 break;
2798 case 13:
2799 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2801 break;
2802 case 14:
2803 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2805 PIPE_CONFIG(ADDR_SURF_P2) |
2806 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2807 break;
2808 case 16:
2809 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2811 PIPE_CONFIG(ADDR_SURF_P2) |
2812 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2813 break;
2814 case 17:
2815 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2817 PIPE_CONFIG(ADDR_SURF_P2) |
2818 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2819 break;
2820 case 27:
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2823 break;
2824 case 28:
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P2) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2829 break;
2830 case 29:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P2) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2835 break;
2836 case 30:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 default:
2843 gb_tile_moden = 0;
2844 break;
2845 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2846 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2847 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2848 }
2849 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2850 switch (reg_offset) {
2851 case 0:
2852 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 break;
2857 case 1:
2858 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2859 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2860 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2861 NUM_BANKS(ADDR_SURF_16_BANK));
2862 break;
2863 case 2:
2864 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2867 NUM_BANKS(ADDR_SURF_16_BANK));
2868 break;
2869 case 3:
2870 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2872 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2873 NUM_BANKS(ADDR_SURF_16_BANK));
2874 break;
2875 case 4:
2876 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2879 NUM_BANKS(ADDR_SURF_16_BANK));
2880 break;
2881 case 5:
2882 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2883 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2884 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2885 NUM_BANKS(ADDR_SURF_16_BANK));
2886 break;
2887 case 6:
2888 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2891 NUM_BANKS(ADDR_SURF_8_BANK));
2892 break;
2893 case 8:
2894 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2897 NUM_BANKS(ADDR_SURF_16_BANK));
2898 break;
2899 case 9:
2900 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2901 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2902 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2903 NUM_BANKS(ADDR_SURF_16_BANK));
2904 break;
2905 case 10:
2906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 NUM_BANKS(ADDR_SURF_16_BANK));
2910 break;
2911 case 11:
2912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915 NUM_BANKS(ADDR_SURF_16_BANK));
2916 break;
2917 case 12:
2918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922 break;
2923 case 13:
2924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927 NUM_BANKS(ADDR_SURF_16_BANK));
2928 break;
2929 case 14:
2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2933 NUM_BANKS(ADDR_SURF_8_BANK));
2934 break;
2935 default:
2936 gb_tile_moden = 0;
2937 break;
2938 }
2939 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2940 }
2941 } else
2942 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2943}
2944
2945/**
2946 * cik_select_se_sh - select which SE, SH to address
2947 *
2948 * @rdev: radeon_device pointer
2949 * @se_num: shader engine to address
2950 * @sh_num: sh block to address
2951 *
2952 * Select which SE, SH combinations to address. Certain
2953 * registers are instanced per SE or SH. 0xffffffff means
2954 * broadcast to all SEs or SHs (CIK).
2955 */
2956static void cik_select_se_sh(struct radeon_device *rdev,
2957 u32 se_num, u32 sh_num)
2958{
2959 u32 data = INSTANCE_BROADCAST_WRITES;
2960
2961 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]2962 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2963 else if (se_num == 0xffffffff)
2964 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2965 else if (sh_num == 0xffffffff)
2966 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2967 else
2968 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2969 WREG32(GRBM_GFX_INDEX, data);
2970}
2971
2972/**
2973 * cik_create_bitmask - create a bitmask
2974 *
2975 * @bit_width: length of the mask
2976 *
2977 * create a variable length bit mask (CIK).
2978 * Returns the bitmask.
2979 */
2980static u32 cik_create_bitmask(u32 bit_width)
2981{
2982 u32 i, mask = 0;
2983
2984 for (i = 0; i < bit_width; i++) {
2985 mask <<= 1;
2986 mask |= 1;
2987 }
2988 return mask;
2989}
2990
2991/**
2992 * cik_select_se_sh - select which SE, SH to address
2993 *
2994 * @rdev: radeon_device pointer
2995 * @max_rb_num: max RBs (render backends) for the asic
2996 * @se_num: number of SEs (shader engines) for the asic
2997 * @sh_per_se: number of SH blocks per SE for the asic
2998 *
2999 * Calculates the bitmask of disabled RBs (CIK).
3000 * Returns the disabled RB bitmask.
3001 */
3002static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3003 u32 max_rb_num, u32 se_num,
3004 u32 sh_per_se)
3005{
3006 u32 data, mask;
3007
3008 data = RREG32(CC_RB_BACKEND_DISABLE);
3009 if (data & 1)
3010 data &= BACKEND_DISABLE_MASK;
3011 else
3012 data = 0;
3013 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3014
3015 data >>= BACKEND_DISABLE_SHIFT;
3016
3017 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3018
3019 return data & mask;
3020}
3021
3022/**
3023 * cik_setup_rb - setup the RBs on the asic
3024 *
3025 * @rdev: radeon_device pointer
3026 * @se_num: number of SEs (shader engines) for the asic
3027 * @sh_per_se: number of SH blocks per SE for the asic
3028 * @max_rb_num: max RBs (render backends) for the asic
3029 *
3030 * Configures per-SE/SH RB registers (CIK).
3031 */
3032static void cik_setup_rb(struct radeon_device *rdev,
3033 u32 se_num, u32 sh_per_se,
3034 u32 max_rb_num)
3035{
3036 int i, j;
3037 u32 data, mask;
3038 u32 disabled_rbs = 0;
3039 u32 enabled_rbs = 0;
3040
3041 for (i = 0; i < se_num; i++) {
3042 for (j = 0; j < sh_per_se; j++) {
3043 cik_select_se_sh(rdev, i, j);
3044 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame^]3045 if (rdev->family == CHIP_HAWAII)
3046 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3047 else
3048 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3049 }
3050 }
3051 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3052
3053 mask = 1;
3054 for (i = 0; i < max_rb_num; i++) {
3055 if (!(disabled_rbs & mask))
3056 enabled_rbs |= mask;
3057 mask <<= 1;
3058 }
3059
3060 for (i = 0; i < se_num; i++) {
3061 cik_select_se_sh(rdev, i, 0xffffffff);
3062 data = 0;
3063 for (j = 0; j < sh_per_se; j++) {
3064 switch (enabled_rbs & 3) {
Alex Deucherfc821b72013-08-07 20:14:08 -0400[diff] [blame^]3065 case 0:
3066 if (j == 0)
3067 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3068 else
3069 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3070 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3071 case 1:
3072 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3073 break;
3074 case 2:
3075 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3076 break;
3077 case 3:
3078 default:
3079 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3080 break;
3081 }
3082 enabled_rbs >>= 2;
3083 }
3084 WREG32(PA_SC_RASTER_CONFIG, data);
3085 }
3086 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3087}
3088
3089/**
3090 * cik_gpu_init - setup the 3D engine
3091 *
3092 * @rdev: radeon_device pointer
3093 *
3094 * Configures the 3D engine and tiling configuration
3095 * registers so that the 3D engine is usable.
3096 */
3097static void cik_gpu_init(struct radeon_device *rdev)
3098{
3099 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3100 u32 mc_shared_chmap, mc_arb_ramcfg;
3101 u32 hdp_host_path_cntl;
3102 u32 tmp;
3103 int i, j;
3104
3105 switch (rdev->family) {
3106 case CHIP_BONAIRE:
3107 rdev->config.cik.max_shader_engines = 2;
3108 rdev->config.cik.max_tile_pipes = 4;
3109 rdev->config.cik.max_cu_per_sh = 7;
3110 rdev->config.cik.max_sh_per_se = 1;
3111 rdev->config.cik.max_backends_per_se = 2;
3112 rdev->config.cik.max_texture_channel_caches = 4;
3113 rdev->config.cik.max_gprs = 256;
3114 rdev->config.cik.max_gs_threads = 32;
3115 rdev->config.cik.max_hw_contexts = 8;
3116
3117 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3118 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3119 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3120 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3121 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3122 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3123 case CHIP_HAWAII:
3124 rdev->config.cik.max_shader_engines = 4;
3125 rdev->config.cik.max_tile_pipes = 16;
3126 rdev->config.cik.max_cu_per_sh = 11;
3127 rdev->config.cik.max_sh_per_se = 1;
3128 rdev->config.cik.max_backends_per_se = 4;
3129 rdev->config.cik.max_texture_channel_caches = 16;
3130 rdev->config.cik.max_gprs = 256;
3131 rdev->config.cik.max_gs_threads = 32;
3132 rdev->config.cik.max_hw_contexts = 8;
3133
3134 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3135 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3136 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3137 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3138 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3139 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3140 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3141 rdev->config.cik.max_shader_engines = 1;
3142 rdev->config.cik.max_tile_pipes = 4;
3143 if ((rdev->pdev->device == 0x1304) ||
3144 (rdev->pdev->device == 0x1305) ||
3145 (rdev->pdev->device == 0x130C) ||
3146 (rdev->pdev->device == 0x130F) ||
3147 (rdev->pdev->device == 0x1310) ||
3148 (rdev->pdev->device == 0x1311) ||
3149 (rdev->pdev->device == 0x131C)) {
3150 rdev->config.cik.max_cu_per_sh = 8;
3151 rdev->config.cik.max_backends_per_se = 2;
3152 } else if ((rdev->pdev->device == 0x1309) ||
3153 (rdev->pdev->device == 0x130A) ||
3154 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3155 (rdev->pdev->device == 0x1313) ||
3156 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3157 rdev->config.cik.max_cu_per_sh = 6;
3158 rdev->config.cik.max_backends_per_se = 2;
3159 } else if ((rdev->pdev->device == 0x1306) ||
3160 (rdev->pdev->device == 0x1307) ||
3161 (rdev->pdev->device == 0x130B) ||
3162 (rdev->pdev->device == 0x130E) ||
3163 (rdev->pdev->device == 0x1315) ||
3164 (rdev->pdev->device == 0x131B)) {
3165 rdev->config.cik.max_cu_per_sh = 4;
3166 rdev->config.cik.max_backends_per_se = 1;
3167 } else {
3168 rdev->config.cik.max_cu_per_sh = 3;
3169 rdev->config.cik.max_backends_per_se = 1;
3170 }
3171 rdev->config.cik.max_sh_per_se = 1;
3172 rdev->config.cik.max_texture_channel_caches = 4;
3173 rdev->config.cik.max_gprs = 256;
3174 rdev->config.cik.max_gs_threads = 16;
3175 rdev->config.cik.max_hw_contexts = 8;
3176
3177 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3178 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3179 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3180 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3181 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3182 break;
3183 case CHIP_KABINI:
3184 default:
3185 rdev->config.cik.max_shader_engines = 1;
3186 rdev->config.cik.max_tile_pipes = 2;
3187 rdev->config.cik.max_cu_per_sh = 2;
3188 rdev->config.cik.max_sh_per_se = 1;
3189 rdev->config.cik.max_backends_per_se = 1;
3190 rdev->config.cik.max_texture_channel_caches = 2;
3191 rdev->config.cik.max_gprs = 256;
3192 rdev->config.cik.max_gs_threads = 16;
3193 rdev->config.cik.max_hw_contexts = 8;
3194
3195 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3196 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3197 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3198 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3199 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3200 break;
3201 }
3202
3203 /* Initialize HDP */
3204 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3205 WREG32((0x2c14 + j), 0x00000000);
3206 WREG32((0x2c18 + j), 0x00000000);
3207 WREG32((0x2c1c + j), 0x00000000);
3208 WREG32((0x2c20 + j), 0x00000000);
3209 WREG32((0x2c24 + j), 0x00000000);
3210 }
3211
3212 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3213
3214 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3215
3216 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3217 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3218
3219 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3220 rdev->config.cik.mem_max_burst_length_bytes = 256;
3221 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3222 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3223 if (rdev->config.cik.mem_row_size_in_kb > 4)
3224 rdev->config.cik.mem_row_size_in_kb = 4;
3225 /* XXX use MC settings? */
3226 rdev->config.cik.shader_engine_tile_size = 32;
3227 rdev->config.cik.num_gpus = 1;
3228 rdev->config.cik.multi_gpu_tile_size = 64;
3229
3230 /* fix up row size */
3231 gb_addr_config &= ~ROW_SIZE_MASK;
3232 switch (rdev->config.cik.mem_row_size_in_kb) {
3233 case 1:
3234 default:
3235 gb_addr_config |= ROW_SIZE(0);
3236 break;
3237 case 2:
3238 gb_addr_config |= ROW_SIZE(1);
3239 break;
3240 case 4:
3241 gb_addr_config |= ROW_SIZE(2);
3242 break;
3243 }
3244
3245 /* setup tiling info dword. gb_addr_config is not adequate since it does
3246 * not have bank info, so create a custom tiling dword.
3247 * bits 3:0 num_pipes
3248 * bits 7:4 num_banks
3249 * bits 11:8 group_size
3250 * bits 15:12 row_size
3251 */
3252 rdev->config.cik.tile_config = 0;
3253 switch (rdev->config.cik.num_tile_pipes) {
3254 case 1:
3255 rdev->config.cik.tile_config |= (0 << 0);
3256 break;
3257 case 2:
3258 rdev->config.cik.tile_config |= (1 << 0);
3259 break;
3260 case 4:
3261 rdev->config.cik.tile_config |= (2 << 0);
3262 break;
3263 case 8:
3264 default:
3265 /* XXX what about 12? */
3266 rdev->config.cik.tile_config |= (3 << 0);
3267 break;
3268 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3269 rdev->config.cik.tile_config |=
3270 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3271 rdev->config.cik.tile_config |=
3272 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3273 rdev->config.cik.tile_config |=
3274 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3275
3276 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3277 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3278 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3279 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3280 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3281 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3282 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3283 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3284
3285 cik_tiling_mode_table_init(rdev);
3286
3287 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3288 rdev->config.cik.max_sh_per_se,
3289 rdev->config.cik.max_backends_per_se);
3290
3291 /* set HW defaults for 3D engine */
3292 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3293
3294 WREG32(SX_DEBUG_1, 0x20);
3295
3296 WREG32(TA_CNTL_AUX, 0x00010000);
3297
3298 tmp = RREG32(SPI_CONFIG_CNTL);
3299 tmp |= 0x03000000;
3300 WREG32(SPI_CONFIG_CNTL, tmp);
3301
3302 WREG32(SQ_CONFIG, 1);
3303
3304 WREG32(DB_DEBUG, 0);
3305
3306 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3307 tmp |= 0x00000400;
3308 WREG32(DB_DEBUG2, tmp);
3309
3310 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3311 tmp |= 0x00020200;
3312 WREG32(DB_DEBUG3, tmp);
3313
3314 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3315 tmp |= 0x00018208;
3316 WREG32(CB_HW_CONTROL, tmp);
3317
3318 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3319
3320 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3321 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3322 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3323 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3324
3325 WREG32(VGT_NUM_INSTANCES, 1);
3326
3327 WREG32(CP_PERFMON_CNTL, 0);
3328
3329 WREG32(SQ_CONFIG, 0);
3330
3331 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3332 FORCE_EOV_MAX_REZ_CNT(255)));
3333
3334 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3335 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3336
3337 WREG32(VGT_GS_VERTEX_REUSE, 16);
3338 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3339
3340 tmp = RREG32(HDP_MISC_CNTL);
3341 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3342 WREG32(HDP_MISC_CNTL, tmp);
3343
3344 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3345 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3346
3347 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3348 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3349
3350 udelay(50);
3351}
3352
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3353/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3354 * GPU scratch registers helpers function.
3355 */
3356/**
3357 * cik_scratch_init - setup driver info for CP scratch regs
3358 *
3359 * @rdev: radeon_device pointer
3360 *
3361 * Set up the number and offset of the CP scratch registers.
3362 * NOTE: use of CP scratch registers is a legacy inferface and
3363 * is not used by default on newer asics (r6xx+). On newer asics,
3364 * memory buffers are used for fences rather than scratch regs.
3365 */
3366static void cik_scratch_init(struct radeon_device *rdev)
3367{
3368 int i;
3369
3370 rdev->scratch.num_reg = 7;
3371 rdev->scratch.reg_base = SCRATCH_REG0;
3372 for (i = 0; i < rdev->scratch.num_reg; i++) {
3373 rdev->scratch.free[i] = true;
3374 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3375 }
3376}
3377
3378/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3379 * cik_ring_test - basic gfx ring test
3380 *
3381 * @rdev: radeon_device pointer
3382 * @ring: radeon_ring structure holding ring information
3383 *
3384 * Allocate a scratch register and write to it using the gfx ring (CIK).
3385 * Provides a basic gfx ring test to verify that the ring is working.
3386 * Used by cik_cp_gfx_resume();
3387 * Returns 0 on success, error on failure.
3388 */
3389int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3390{
3391 uint32_t scratch;
3392 uint32_t tmp = 0;
3393 unsigned i;
3394 int r;
3395
3396 r = radeon_scratch_get(rdev, &scratch);
3397 if (r) {
3398 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3399 return r;
3400 }
3401 WREG32(scratch, 0xCAFEDEAD);
3402 r = radeon_ring_lock(rdev, ring, 3);
3403 if (r) {
3404 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3405 radeon_scratch_free(rdev, scratch);
3406 return r;
3407 }
3408 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3409 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3410 radeon_ring_write(ring, 0xDEADBEEF);
3411 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3412
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3413 for (i = 0; i < rdev->usec_timeout; i++) {
3414 tmp = RREG32(scratch);
3415 if (tmp == 0xDEADBEEF)
3416 break;
3417 DRM_UDELAY(1);
3418 }
3419 if (i < rdev->usec_timeout) {
3420 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3421 } else {
3422 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3423 ring->idx, scratch, tmp);
3424 r = -EINVAL;
3425 }
3426 radeon_scratch_free(rdev, scratch);
3427 return r;
3428}
3429
3430/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3431 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3432 *
3433 * @rdev: radeon_device pointer
3434 * @fence: radeon fence object
3435 *
3436 * Emits a fence sequnce number on the gfx ring and flushes
3437 * GPU caches.
3438 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3439void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3440 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3441{
3442 struct radeon_ring *ring = &rdev->ring[fence->ring];
3443 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3444
3445 /* EVENT_WRITE_EOP - flush caches, send int */
3446 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3447 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3448 EOP_TC_ACTION_EN |
3449 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3450 EVENT_INDEX(5)));
3451 radeon_ring_write(ring, addr & 0xfffffffc);
3452 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3453 radeon_ring_write(ring, fence->seq);
3454 radeon_ring_write(ring, 0);
3455 /* HDP flush */
3456 /* We should be using the new WAIT_REG_MEM special op packet here
3457 * but it causes the CP to hang
3458 */
3459 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3460 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3461 WRITE_DATA_DST_SEL(0)));
3462 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3463 radeon_ring_write(ring, 0);
3464 radeon_ring_write(ring, 0);
3465}
3466
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3467/**
3468 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3469 *
3470 * @rdev: radeon_device pointer
3471 * @fence: radeon fence object
3472 *
3473 * Emits a fence sequnce number on the compute ring and flushes
3474 * GPU caches.
3475 */
3476void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3477 struct radeon_fence *fence)
3478{
3479 struct radeon_ring *ring = &rdev->ring[fence->ring];
3480 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3481
3482 /* RELEASE_MEM - flush caches, send int */
3483 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3484 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3485 EOP_TC_ACTION_EN |
3486 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3487 EVENT_INDEX(5)));
3488 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3489 radeon_ring_write(ring, addr & 0xfffffffc);
3490 radeon_ring_write(ring, upper_32_bits(addr));
3491 radeon_ring_write(ring, fence->seq);
3492 radeon_ring_write(ring, 0);
3493 /* HDP flush */
3494 /* We should be using the new WAIT_REG_MEM special op packet here
3495 * but it causes the CP to hang
3496 */
3497 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3498 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3499 WRITE_DATA_DST_SEL(0)));
3500 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3501 radeon_ring_write(ring, 0);
3502 radeon_ring_write(ring, 0);
3503}
3504
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3505void cik_semaphore_ring_emit(struct radeon_device *rdev,
3506 struct radeon_ring *ring,
3507 struct radeon_semaphore *semaphore,
3508 bool emit_wait)
3509{
3510 uint64_t addr = semaphore->gpu_addr;
3511 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3512
3513 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3514 radeon_ring_write(ring, addr & 0xffffffff);
3515 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3516}
3517
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3518/**
3519 * cik_copy_cpdma - copy pages using the CP DMA engine
3520 *
3521 * @rdev: radeon_device pointer
3522 * @src_offset: src GPU address
3523 * @dst_offset: dst GPU address
3524 * @num_gpu_pages: number of GPU pages to xfer
3525 * @fence: radeon fence object
3526 *
3527 * Copy GPU paging using the CP DMA engine (CIK+).
3528 * Used by the radeon ttm implementation to move pages if
3529 * registered as the asic copy callback.
3530 */
3531int cik_copy_cpdma(struct radeon_device *rdev,
3532 uint64_t src_offset, uint64_t dst_offset,
3533 unsigned num_gpu_pages,
3534 struct radeon_fence **fence)
3535{
3536 struct radeon_semaphore *sem = NULL;
3537 int ring_index = rdev->asic->copy.blit_ring_index;
3538 struct radeon_ring *ring = &rdev->ring[ring_index];
3539 u32 size_in_bytes, cur_size_in_bytes, control;
3540 int i, num_loops;
3541 int r = 0;
3542
3543 r = radeon_semaphore_create(rdev, &sem);
3544 if (r) {
3545 DRM_ERROR("radeon: moving bo (%d).\n", r);
3546 return r;
3547 }
3548
3549 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3550 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3551 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3552 if (r) {
3553 DRM_ERROR("radeon: moving bo (%d).\n", r);
3554 radeon_semaphore_free(rdev, &sem, NULL);
3555 return r;
3556 }
3557
3558 if (radeon_fence_need_sync(*fence, ring->idx)) {
3559 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3560 ring->idx);
3561 radeon_fence_note_sync(*fence, ring->idx);
3562 } else {
3563 radeon_semaphore_free(rdev, &sem, NULL);
3564 }
3565
3566 for (i = 0; i < num_loops; i++) {
3567 cur_size_in_bytes = size_in_bytes;
3568 if (cur_size_in_bytes > 0x1fffff)
3569 cur_size_in_bytes = 0x1fffff;
3570 size_in_bytes -= cur_size_in_bytes;
3571 control = 0;
3572 if (size_in_bytes == 0)
3573 control |= PACKET3_DMA_DATA_CP_SYNC;
3574 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3575 radeon_ring_write(ring, control);
3576 radeon_ring_write(ring, lower_32_bits(src_offset));
3577 radeon_ring_write(ring, upper_32_bits(src_offset));
3578 radeon_ring_write(ring, lower_32_bits(dst_offset));
3579 radeon_ring_write(ring, upper_32_bits(dst_offset));
3580 radeon_ring_write(ring, cur_size_in_bytes);
3581 src_offset += cur_size_in_bytes;
3582 dst_offset += cur_size_in_bytes;
3583 }
3584
3585 r = radeon_fence_emit(rdev, fence, ring->idx);
3586 if (r) {
3587 radeon_ring_unlock_undo(rdev, ring);
3588 return r;
3589 }
3590
3591 radeon_ring_unlock_commit(rdev, ring);
3592 radeon_semaphore_free(rdev, &sem, *fence);
3593
3594 return r;
3595}
3596
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3597/*
3598 * IB stuff
3599 */
3600/**
3601 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3602 *
3603 * @rdev: radeon_device pointer
3604 * @ib: radeon indirect buffer object
3605 *
3606 * Emits an DE (drawing engine) or CE (constant engine) IB
3607 * on the gfx ring. IBs are usually generated by userspace
3608 * acceleration drivers and submitted to the kernel for
3609 * sheduling on the ring. This function schedules the IB
3610 * on the gfx ring for execution by the GPU.
3611 */
3612void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3613{
3614 struct radeon_ring *ring = &rdev->ring[ib->ring];
3615 u32 header, control = INDIRECT_BUFFER_VALID;
3616
3617 if (ib->is_const_ib) {
3618 /* set switch buffer packet before const IB */
3619 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3620 radeon_ring_write(ring, 0);
3621
3622 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3623 } else {
3624 u32 next_rptr;
3625 if (ring->rptr_save_reg) {
3626 next_rptr = ring->wptr + 3 + 4;
3627 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3628 radeon_ring_write(ring, ((ring->rptr_save_reg -
3629 PACKET3_SET_UCONFIG_REG_START) >> 2));
3630 radeon_ring_write(ring, next_rptr);
3631 } else if (rdev->wb.enabled) {
3632 next_rptr = ring->wptr + 5 + 4;
3633 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3634 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3635 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3636 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3637 radeon_ring_write(ring, next_rptr);
3638 }
3639
3640 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3641 }
3642
3643 control |= ib->length_dw |
3644 (ib->vm ? (ib->vm->id << 24) : 0);
3645
3646 radeon_ring_write(ring, header);
3647 radeon_ring_write(ring,
3648#ifdef __BIG_ENDIAN
3649 (2 << 0) |
3650#endif
3651 (ib->gpu_addr & 0xFFFFFFFC));
3652 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3653 radeon_ring_write(ring, control);
3654}
3655
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3656/**
3657 * cik_ib_test - basic gfx ring IB test
3658 *
3659 * @rdev: radeon_device pointer
3660 * @ring: radeon_ring structure holding ring information
3661 *
3662 * Allocate an IB and execute it on the gfx ring (CIK).
3663 * Provides a basic gfx ring test to verify that IBs are working.
3664 * Returns 0 on success, error on failure.
3665 */
3666int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3667{
3668 struct radeon_ib ib;
3669 uint32_t scratch;
3670 uint32_t tmp = 0;
3671 unsigned i;
3672 int r;
3673
3674 r = radeon_scratch_get(rdev, &scratch);
3675 if (r) {
3676 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3677 return r;
3678 }
3679 WREG32(scratch, 0xCAFEDEAD);
3680 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3681 if (r) {
3682 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3683 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3684 return r;
3685 }
3686 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3687 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3688 ib.ptr[2] = 0xDEADBEEF;
3689 ib.length_dw = 3;
3690 r = radeon_ib_schedule(rdev, &ib, NULL);
3691 if (r) {
3692 radeon_scratch_free(rdev, scratch);
3693 radeon_ib_free(rdev, &ib);
3694 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3695 return r;
3696 }
3697 r = radeon_fence_wait(ib.fence, false);
3698 if (r) {
3699 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3700 radeon_scratch_free(rdev, scratch);
3701 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3702 return r;
3703 }
3704 for (i = 0; i < rdev->usec_timeout; i++) {
3705 tmp = RREG32(scratch);
3706 if (tmp == 0xDEADBEEF)
3707 break;
3708 DRM_UDELAY(1);
3709 }
3710 if (i < rdev->usec_timeout) {
3711 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3712 } else {
3713 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3714 scratch, tmp);
3715 r = -EINVAL;
3716 }
3717 radeon_scratch_free(rdev, scratch);
3718 radeon_ib_free(rdev, &ib);
3719 return r;
3720}
3721
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3722/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3723 * CP.
3724 * On CIK, gfx and compute now have independant command processors.
3725 *
3726 * GFX
3727 * Gfx consists of a single ring and can process both gfx jobs and
3728 * compute jobs. The gfx CP consists of three microengines (ME):
3729 * PFP - Pre-Fetch Parser
3730 * ME - Micro Engine
3731 * CE - Constant Engine
3732 * The PFP and ME make up what is considered the Drawing Engine (DE).
3733 * The CE is an asynchronous engine used for updating buffer desciptors
3734 * used by the DE so that they can be loaded into cache in parallel
3735 * while the DE is processing state update packets.
3736 *
3737 * Compute
3738 * The compute CP consists of two microengines (ME):
3739 * MEC1 - Compute MicroEngine 1
3740 * MEC2 - Compute MicroEngine 2
3741 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3742 * The queues are exposed to userspace and are programmed directly
3743 * by the compute runtime.
3744 */
3745/**
3746 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3747 *
3748 * @rdev: radeon_device pointer
3749 * @enable: enable or disable the MEs
3750 *
3751 * Halts or unhalts the gfx MEs.
3752 */
3753static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3754{
3755 if (enable)
3756 WREG32(CP_ME_CNTL, 0);
3757 else {
3758 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3759 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3760 }
3761 udelay(50);
3762}
3763
3764/**
3765 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3766 *
3767 * @rdev: radeon_device pointer
3768 *
3769 * Loads the gfx PFP, ME, and CE ucode.
3770 * Returns 0 for success, -EINVAL if the ucode is not available.
3771 */
3772static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3773{
3774 const __be32 *fw_data;
3775 int i;
3776
3777 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3778 return -EINVAL;
3779
3780 cik_cp_gfx_enable(rdev, false);
3781
3782 /* PFP */
3783 fw_data = (const __be32 *)rdev->pfp_fw->data;
3784 WREG32(CP_PFP_UCODE_ADDR, 0);
3785 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3786 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3787 WREG32(CP_PFP_UCODE_ADDR, 0);
3788
3789 /* CE */
3790 fw_data = (const __be32 *)rdev->ce_fw->data;
3791 WREG32(CP_CE_UCODE_ADDR, 0);
3792 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3793 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3794 WREG32(CP_CE_UCODE_ADDR, 0);
3795
3796 /* ME */
3797 fw_data = (const __be32 *)rdev->me_fw->data;
3798 WREG32(CP_ME_RAM_WADDR, 0);
3799 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3800 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3801 WREG32(CP_ME_RAM_WADDR, 0);
3802
3803 WREG32(CP_PFP_UCODE_ADDR, 0);
3804 WREG32(CP_CE_UCODE_ADDR, 0);
3805 WREG32(CP_ME_RAM_WADDR, 0);
3806 WREG32(CP_ME_RAM_RADDR, 0);
3807 return 0;
3808}
3809
3810/**
3811 * cik_cp_gfx_start - start the gfx ring
3812 *
3813 * @rdev: radeon_device pointer
3814 *
3815 * Enables the ring and loads the clear state context and other
3816 * packets required to init the ring.
3817 * Returns 0 for success, error for failure.
3818 */
3819static int cik_cp_gfx_start(struct radeon_device *rdev)
3820{
3821 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3822 int r, i;
3823
3824 /* init the CP */
3825 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3826 WREG32(CP_ENDIAN_SWAP, 0);
3827 WREG32(CP_DEVICE_ID, 1);
3828
3829 cik_cp_gfx_enable(rdev, true);
3830
3831 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3832 if (r) {
3833 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3834 return r;
3835 }
3836
3837 /* init the CE partitions. CE only used for gfx on CIK */
3838 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3839 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3840 radeon_ring_write(ring, 0xc000);
3841 radeon_ring_write(ring, 0xc000);
3842
3843 /* setup clear context state */
3844 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3845 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3846
3847 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3848 radeon_ring_write(ring, 0x80000000);
3849 radeon_ring_write(ring, 0x80000000);
3850
3851 for (i = 0; i < cik_default_size; i++)
3852 radeon_ring_write(ring, cik_default_state[i]);
3853
3854 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3855 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3856
3857 /* set clear context state */
3858 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3859 radeon_ring_write(ring, 0);
3860
3861 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3862 radeon_ring_write(ring, 0x00000316);
3863 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3864 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3865
3866 radeon_ring_unlock_commit(rdev, ring);
3867
3868 return 0;
3869}
3870
3871/**
3872 * cik_cp_gfx_fini - stop the gfx ring
3873 *
3874 * @rdev: radeon_device pointer
3875 *
3876 * Stop the gfx ring and tear down the driver ring
3877 * info.
3878 */
3879static void cik_cp_gfx_fini(struct radeon_device *rdev)
3880{
3881 cik_cp_gfx_enable(rdev, false);
3882 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3883}
3884
3885/**
3886 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3887 *
3888 * @rdev: radeon_device pointer
3889 *
3890 * Program the location and size of the gfx ring buffer
3891 * and test it to make sure it's working.
3892 * Returns 0 for success, error for failure.
3893 */
3894static int cik_cp_gfx_resume(struct radeon_device *rdev)
3895{
3896 struct radeon_ring *ring;
3897 u32 tmp;
3898 u32 rb_bufsz;
3899 u64 rb_addr;
3900 int r;
3901
3902 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3903 if (rdev->family != CHIP_HAWAII)
3904 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3905
3906 /* Set the write pointer delay */
3907 WREG32(CP_RB_WPTR_DELAY, 0);
3908
3909 /* set the RB to use vmid 0 */
3910 WREG32(CP_RB_VMID, 0);
3911
3912 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3913
3914 /* ring 0 - compute and gfx */
3915 /* Set ring buffer size */
3916 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3917 rb_bufsz = order_base_2(ring->ring_size / 8);
3918 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3919#ifdef __BIG_ENDIAN
3920 tmp |= BUF_SWAP_32BIT;
3921#endif
3922 WREG32(CP_RB0_CNTL, tmp);
3923
3924 /* Initialize the ring buffer's read and write pointers */
3925 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3926 ring->wptr = 0;
3927 WREG32(CP_RB0_WPTR, ring->wptr);
3928
3929 /* set the wb address wether it's enabled or not */
3930 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3931 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3932
3933 /* scratch register shadowing is no longer supported */
3934 WREG32(SCRATCH_UMSK, 0);
3935
3936 if (!rdev->wb.enabled)
3937 tmp |= RB_NO_UPDATE;
3938
3939 mdelay(1);
3940 WREG32(CP_RB0_CNTL, tmp);
3941
3942 rb_addr = ring->gpu_addr >> 8;
3943 WREG32(CP_RB0_BASE, rb_addr);
3944 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3945
3946 ring->rptr = RREG32(CP_RB0_RPTR);
3947
3948 /* start the ring */
3949 cik_cp_gfx_start(rdev);
3950 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3951 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3952 if (r) {
3953 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3954 return r;
3955 }
3956 return 0;
3957}
3958
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3959u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3960 struct radeon_ring *ring)
3961{
3962 u32 rptr;
3963
3964
3965
3966 if (rdev->wb.enabled) {
3967 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3968 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3969 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3970 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3971 rptr = RREG32(CP_HQD_PQ_RPTR);
3972 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3973 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3974 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3975
3976 return rptr;
3977}
3978
3979u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3980 struct radeon_ring *ring)
3981{
3982 u32 wptr;
3983
3984 if (rdev->wb.enabled) {
3985 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3986 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3987 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3988 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3989 wptr = RREG32(CP_HQD_PQ_WPTR);
3990 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3991 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3992 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3993
3994 return wptr;
3995}
3996
3997void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3998 struct radeon_ring *ring)
3999{
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]4000 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4001 WDOORBELL32(ring->doorbell_offset, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4002}
4003
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4004/**
4005 * cik_cp_compute_enable - enable/disable the compute CP MEs
4006 *
4007 * @rdev: radeon_device pointer
4008 * @enable: enable or disable the MEs
4009 *
4010 * Halts or unhalts the compute MEs.
4011 */
4012static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4013{
4014 if (enable)
4015 WREG32(CP_MEC_CNTL, 0);
4016 else
4017 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4018 udelay(50);
4019}
4020
4021/**
4022 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4023 *
4024 * @rdev: radeon_device pointer
4025 *
4026 * Loads the compute MEC1&2 ucode.
4027 * Returns 0 for success, -EINVAL if the ucode is not available.
4028 */
4029static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4030{
4031 const __be32 *fw_data;
4032 int i;
4033
4034 if (!rdev->mec_fw)
4035 return -EINVAL;
4036
4037 cik_cp_compute_enable(rdev, false);
4038
4039 /* MEC1 */
4040 fw_data = (const __be32 *)rdev->mec_fw->data;
4041 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4042 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4043 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4044 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4045
4046 if (rdev->family == CHIP_KAVERI) {
4047 /* MEC2 */
4048 fw_data = (const __be32 *)rdev->mec_fw->data;
4049 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4050 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4051 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4052 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4053 }
4054
4055 return 0;
4056}
4057
4058/**
4059 * cik_cp_compute_start - start the compute queues
4060 *
4061 * @rdev: radeon_device pointer
4062 *
4063 * Enable the compute queues.
4064 * Returns 0 for success, error for failure.
4065 */
4066static int cik_cp_compute_start(struct radeon_device *rdev)
4067{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4068 cik_cp_compute_enable(rdev, true);
4069
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4070 return 0;
4071}
4072
4073/**
4074 * cik_cp_compute_fini - stop the compute queues
4075 *
4076 * @rdev: radeon_device pointer
4077 *
4078 * Stop the compute queues and tear down the driver queue
4079 * info.
4080 */
4081static void cik_cp_compute_fini(struct radeon_device *rdev)
4082{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4083 int i, idx, r;
4084
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4085 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4086
4087 for (i = 0; i < 2; i++) {
4088 if (i == 0)
4089 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4090 else
4091 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4092
4093 if (rdev->ring[idx].mqd_obj) {
4094 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4095 if (unlikely(r != 0))
4096 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4097
4098 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4099 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4100
4101 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4102 rdev->ring[idx].mqd_obj = NULL;
4103 }
4104 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4105}
4106
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4107static void cik_mec_fini(struct radeon_device *rdev)
4108{
4109 int r;
4110
4111 if (rdev->mec.hpd_eop_obj) {
4112 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4113 if (unlikely(r != 0))
4114 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4115 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4116 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4117
4118 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4119 rdev->mec.hpd_eop_obj = NULL;
4120 }
4121}
4122
4123#define MEC_HPD_SIZE 2048
4124
4125static int cik_mec_init(struct radeon_device *rdev)
4126{
4127 int r;
4128 u32 *hpd;
4129
4130 /*
4131 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4132 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4133 */
4134 if (rdev->family == CHIP_KAVERI)
4135 rdev->mec.num_mec = 2;
4136 else
4137 rdev->mec.num_mec = 1;
4138 rdev->mec.num_pipe = 4;
4139 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4140
4141 if (rdev->mec.hpd_eop_obj == NULL) {
4142 r = radeon_bo_create(rdev,
4143 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4144 PAGE_SIZE, true,
4145 RADEON_GEM_DOMAIN_GTT, NULL,
4146 &rdev->mec.hpd_eop_obj);
4147 if (r) {
4148 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4149 return r;
4150 }
4151 }
4152
4153 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4154 if (unlikely(r != 0)) {
4155 cik_mec_fini(rdev);
4156 return r;
4157 }
4158 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4159 &rdev->mec.hpd_eop_gpu_addr);
4160 if (r) {
4161 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4162 cik_mec_fini(rdev);
4163 return r;
4164 }
4165 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4166 if (r) {
4167 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4168 cik_mec_fini(rdev);
4169 return r;
4170 }
4171
4172 /* clear memory. Not sure if this is required or not */
4173 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4174
4175 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4176 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4177
4178 return 0;
4179}
4180
4181struct hqd_registers
4182{
4183 u32 cp_mqd_base_addr;
4184 u32 cp_mqd_base_addr_hi;
4185 u32 cp_hqd_active;
4186 u32 cp_hqd_vmid;
4187 u32 cp_hqd_persistent_state;
4188 u32 cp_hqd_pipe_priority;
4189 u32 cp_hqd_queue_priority;
4190 u32 cp_hqd_quantum;
4191 u32 cp_hqd_pq_base;
4192 u32 cp_hqd_pq_base_hi;
4193 u32 cp_hqd_pq_rptr;
4194 u32 cp_hqd_pq_rptr_report_addr;
4195 u32 cp_hqd_pq_rptr_report_addr_hi;
4196 u32 cp_hqd_pq_wptr_poll_addr;
4197 u32 cp_hqd_pq_wptr_poll_addr_hi;
4198 u32 cp_hqd_pq_doorbell_control;
4199 u32 cp_hqd_pq_wptr;
4200 u32 cp_hqd_pq_control;
4201 u32 cp_hqd_ib_base_addr;
4202 u32 cp_hqd_ib_base_addr_hi;
4203 u32 cp_hqd_ib_rptr;
4204 u32 cp_hqd_ib_control;
4205 u32 cp_hqd_iq_timer;
4206 u32 cp_hqd_iq_rptr;
4207 u32 cp_hqd_dequeue_request;
4208 u32 cp_hqd_dma_offload;
4209 u32 cp_hqd_sema_cmd;
4210 u32 cp_hqd_msg_type;
4211 u32 cp_hqd_atomic0_preop_lo;
4212 u32 cp_hqd_atomic0_preop_hi;
4213 u32 cp_hqd_atomic1_preop_lo;
4214 u32 cp_hqd_atomic1_preop_hi;
4215 u32 cp_hqd_hq_scheduler0;
4216 u32 cp_hqd_hq_scheduler1;
4217 u32 cp_mqd_control;
4218};
4219
4220struct bonaire_mqd
4221{
4222 u32 header;
4223 u32 dispatch_initiator;
4224 u32 dimensions[3];
4225 u32 start_idx[3];
4226 u32 num_threads[3];
4227 u32 pipeline_stat_enable;
4228 u32 perf_counter_enable;
4229 u32 pgm[2];
4230 u32 tba[2];
4231 u32 tma[2];
4232 u32 pgm_rsrc[2];
4233 u32 vmid;
4234 u32 resource_limits;
4235 u32 static_thread_mgmt01[2];
4236 u32 tmp_ring_size;
4237 u32 static_thread_mgmt23[2];
4238 u32 restart[3];
4239 u32 thread_trace_enable;
4240 u32 reserved1;
4241 u32 user_data[16];
4242 u32 vgtcs_invoke_count[2];
4243 struct hqd_registers queue_state;
4244 u32 dequeue_cntr;
4245 u32 interrupt_queue[64];
4246};
4247
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4248/**
4249 * cik_cp_compute_resume - setup the compute queue registers
4250 *
4251 * @rdev: radeon_device pointer
4252 *
4253 * Program the compute queues and test them to make sure they
4254 * are working.
4255 * Returns 0 for success, error for failure.
4256 */
4257static int cik_cp_compute_resume(struct radeon_device *rdev)
4258{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4259 int r, i, idx;
4260 u32 tmp;
4261 bool use_doorbell = true;
4262 u64 hqd_gpu_addr;
4263 u64 mqd_gpu_addr;
4264 u64 eop_gpu_addr;
4265 u64 wb_gpu_addr;
4266 u32 *buf;
4267 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4268
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4269 r = cik_cp_compute_start(rdev);
4270 if (r)
4271 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4272
4273 /* fix up chicken bits */
4274 tmp = RREG32(CP_CPF_DEBUG);
4275 tmp |= (1 << 23);
4276 WREG32(CP_CPF_DEBUG, tmp);
4277
4278 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4279 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4280 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4281 int me = (i < 4) ? 1 : 2;
4282 int pipe = (i < 4) ? i : (i - 4);
4283
4284 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4285
4286 cik_srbm_select(rdev, me, pipe, 0, 0);
4287
4288 /* write the EOP addr */
4289 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4290 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4291
4292 /* set the VMID assigned */
4293 WREG32(CP_HPD_EOP_VMID, 0);
4294
4295 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4296 tmp = RREG32(CP_HPD_EOP_CONTROL);
4297 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4298 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4299 WREG32(CP_HPD_EOP_CONTROL, tmp);
4300 }
4301 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4302 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4303
4304 /* init the queues. Just two for now. */
4305 for (i = 0; i < 2; i++) {
4306 if (i == 0)
4307 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4308 else
4309 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4310
4311 if (rdev->ring[idx].mqd_obj == NULL) {
4312 r = radeon_bo_create(rdev,
4313 sizeof(struct bonaire_mqd),
4314 PAGE_SIZE, true,
4315 RADEON_GEM_DOMAIN_GTT, NULL,
4316 &rdev->ring[idx].mqd_obj);
4317 if (r) {
4318 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4319 return r;
4320 }
4321 }
4322
4323 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4324 if (unlikely(r != 0)) {
4325 cik_cp_compute_fini(rdev);
4326 return r;
4327 }
4328 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4329 &mqd_gpu_addr);
4330 if (r) {
4331 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4332 cik_cp_compute_fini(rdev);
4333 return r;
4334 }
4335 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4336 if (r) {
4337 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4338 cik_cp_compute_fini(rdev);
4339 return r;
4340 }
4341
4342 /* doorbell offset */
4343 rdev->ring[idx].doorbell_offset =
4344 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
4345
4346 /* init the mqd struct */
4347 memset(buf, 0, sizeof(struct bonaire_mqd));
4348
4349 mqd = (struct bonaire_mqd *)buf;
4350 mqd->header = 0xC0310800;
4351 mqd->static_thread_mgmt01[0] = 0xffffffff;
4352 mqd->static_thread_mgmt01[1] = 0xffffffff;
4353 mqd->static_thread_mgmt23[0] = 0xffffffff;
4354 mqd->static_thread_mgmt23[1] = 0xffffffff;
4355
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4356 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4357 cik_srbm_select(rdev, rdev->ring[idx].me,
4358 rdev->ring[idx].pipe,
4359 rdev->ring[idx].queue, 0);
4360
4361 /* disable wptr polling */
4362 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4363 tmp &= ~WPTR_POLL_EN;
4364 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4365
4366 /* enable doorbell? */
4367 mqd->queue_state.cp_hqd_pq_doorbell_control =
4368 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4369 if (use_doorbell)
4370 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4371 else
4372 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4373 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4374 mqd->queue_state.cp_hqd_pq_doorbell_control);
4375
4376 /* disable the queue if it's active */
4377 mqd->queue_state.cp_hqd_dequeue_request = 0;
4378 mqd->queue_state.cp_hqd_pq_rptr = 0;
4379 mqd->queue_state.cp_hqd_pq_wptr= 0;
4380 if (RREG32(CP_HQD_ACTIVE) & 1) {
4381 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4382 for (i = 0; i < rdev->usec_timeout; i++) {
4383 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4384 break;
4385 udelay(1);
4386 }
4387 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4388 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4389 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4390 }
4391
4392 /* set the pointer to the MQD */
4393 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4394 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4395 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4396 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4397 /* set MQD vmid to 0 */
4398 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4399 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4400 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4401
4402 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4403 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4404 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4405 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4406 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4407 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4408
4409 /* set up the HQD, this is similar to CP_RB0_CNTL */
4410 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4411 mqd->queue_state.cp_hqd_pq_control &=
4412 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4413
4414 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4415 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4416 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4417 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4418#ifdef __BIG_ENDIAN
4419 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4420#endif
4421 mqd->queue_state.cp_hqd_pq_control &=
4422 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4423 mqd->queue_state.cp_hqd_pq_control |=
4424 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4425 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4426
4427 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4428 if (i == 0)
4429 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4430 else
4431 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4432 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4433 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4434 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4435 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4436 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4437
4438 /* set the wb address wether it's enabled or not */
4439 if (i == 0)
4440 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4441 else
4442 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4443 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4444 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4445 upper_32_bits(wb_gpu_addr) & 0xffff;
4446 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4447 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4448 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4449 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4450
4451 /* enable the doorbell if requested */
4452 if (use_doorbell) {
4453 mqd->queue_state.cp_hqd_pq_doorbell_control =
4454 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4455 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4456 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4457 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4458 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4459 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4460 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4461
4462 } else {
4463 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4464 }
4465 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4466 mqd->queue_state.cp_hqd_pq_doorbell_control);
4467
4468 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4469 rdev->ring[idx].wptr = 0;
4470 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4471 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4472 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4473 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4474
4475 /* set the vmid for the queue */
4476 mqd->queue_state.cp_hqd_vmid = 0;
4477 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4478
4479 /* activate the queue */
4480 mqd->queue_state.cp_hqd_active = 1;
4481 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4482
4483 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4484 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4485
4486 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4487 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4488
4489 rdev->ring[idx].ready = true;
4490 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4491 if (r)
4492 rdev->ring[idx].ready = false;
4493 }
4494
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4495 return 0;
4496}
4497
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4498static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4499{
4500 cik_cp_gfx_enable(rdev, enable);
4501 cik_cp_compute_enable(rdev, enable);
4502}
4503
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4504static int cik_cp_load_microcode(struct radeon_device *rdev)
4505{
4506 int r;
4507
4508 r = cik_cp_gfx_load_microcode(rdev);
4509 if (r)
4510 return r;
4511 r = cik_cp_compute_load_microcode(rdev);
4512 if (r)
4513 return r;
4514
4515 return 0;
4516}
4517
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4518static void cik_cp_fini(struct radeon_device *rdev)
4519{
4520 cik_cp_gfx_fini(rdev);
4521 cik_cp_compute_fini(rdev);
4522}
4523
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4524static int cik_cp_resume(struct radeon_device *rdev)
4525{
4526 int r;
4527
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4528 cik_enable_gui_idle_interrupt(rdev, false);
4529
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4530 r = cik_cp_load_microcode(rdev);
4531 if (r)
4532 return r;
4533
4534 r = cik_cp_gfx_resume(rdev);
4535 if (r)
4536 return r;
4537 r = cik_cp_compute_resume(rdev);
4538 if (r)
4539 return r;
4540
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4541 cik_enable_gui_idle_interrupt(rdev, true);
4542
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4543 return 0;
4544}
4545
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4546static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4547{
4548 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4549 RREG32(GRBM_STATUS));
4550 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4551 RREG32(GRBM_STATUS2));
4552 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4553 RREG32(GRBM_STATUS_SE0));
4554 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4555 RREG32(GRBM_STATUS_SE1));
4556 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4557 RREG32(GRBM_STATUS_SE2));
4558 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4559 RREG32(GRBM_STATUS_SE3));
4560 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4561 RREG32(SRBM_STATUS));
4562 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4563 RREG32(SRBM_STATUS2));
4564 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4565 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4566 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4567 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4568 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4569 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4570 RREG32(CP_STALLED_STAT1));
4571 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4572 RREG32(CP_STALLED_STAT2));
4573 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4574 RREG32(CP_STALLED_STAT3));
4575 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4576 RREG32(CP_CPF_BUSY_STAT));
4577 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4578 RREG32(CP_CPF_STALLED_STAT1));
4579 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4580 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4581 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4582 RREG32(CP_CPC_STALLED_STAT1));
4583 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4584}
4585
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4586/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4587 * cik_gpu_check_soft_reset - check which blocks are busy
4588 *
4589 * @rdev: radeon_device pointer
4590 *
4591 * Check which blocks are busy and return the relevant reset
4592 * mask to be used by cik_gpu_soft_reset().
4593 * Returns a mask of the blocks to be reset.
4594 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4595u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4596{
4597 u32 reset_mask = 0;
4598 u32 tmp;
4599
4600 /* GRBM_STATUS */
4601 tmp = RREG32(GRBM_STATUS);
4602 if (tmp & (PA_BUSY | SC_BUSY |
4603 BCI_BUSY | SX_BUSY |
4604 TA_BUSY | VGT_BUSY |
4605 DB_BUSY | CB_BUSY |
4606 GDS_BUSY | SPI_BUSY |
4607 IA_BUSY | IA_BUSY_NO_DMA))
4608 reset_mask |= RADEON_RESET_GFX;
4609
4610 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4611 reset_mask |= RADEON_RESET_CP;
4612
4613 /* GRBM_STATUS2 */
4614 tmp = RREG32(GRBM_STATUS2);
4615 if (tmp & RLC_BUSY)
4616 reset_mask |= RADEON_RESET_RLC;
4617
4618 /* SDMA0_STATUS_REG */
4619 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4620 if (!(tmp & SDMA_IDLE))
4621 reset_mask |= RADEON_RESET_DMA;
4622
4623 /* SDMA1_STATUS_REG */
4624 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4625 if (!(tmp & SDMA_IDLE))
4626 reset_mask |= RADEON_RESET_DMA1;
4627
4628 /* SRBM_STATUS2 */
4629 tmp = RREG32(SRBM_STATUS2);
4630 if (tmp & SDMA_BUSY)
4631 reset_mask |= RADEON_RESET_DMA;
4632
4633 if (tmp & SDMA1_BUSY)
4634 reset_mask |= RADEON_RESET_DMA1;
4635
4636 /* SRBM_STATUS */
4637 tmp = RREG32(SRBM_STATUS);
4638
4639 if (tmp & IH_BUSY)
4640 reset_mask |= RADEON_RESET_IH;
4641
4642 if (tmp & SEM_BUSY)
4643 reset_mask |= RADEON_RESET_SEM;
4644
4645 if (tmp & GRBM_RQ_PENDING)
4646 reset_mask |= RADEON_RESET_GRBM;
4647
4648 if (tmp & VMC_BUSY)
4649 reset_mask |= RADEON_RESET_VMC;
4650
4651 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4652 MCC_BUSY | MCD_BUSY))
4653 reset_mask |= RADEON_RESET_MC;
4654
4655 if (evergreen_is_display_hung(rdev))
4656 reset_mask |= RADEON_RESET_DISPLAY;
4657
4658 /* Skip MC reset as it's mostly likely not hung, just busy */
4659 if (reset_mask & RADEON_RESET_MC) {
4660 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4661 reset_mask &= ~RADEON_RESET_MC;
4662 }
4663
4664 return reset_mask;
4665}
4666
4667/**
4668 * cik_gpu_soft_reset - soft reset GPU
4669 *
4670 * @rdev: radeon_device pointer
4671 * @reset_mask: mask of which blocks to reset
4672 *
4673 * Soft reset the blocks specified in @reset_mask.
4674 */
4675static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4676{
4677 struct evergreen_mc_save save;
4678 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4679 u32 tmp;
4680
4681 if (reset_mask == 0)
4682 return;
4683
4684 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4685
4686 cik_print_gpu_status_regs(rdev);
4687 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4688 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4689 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4690 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4691
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4692 /* disable CG/PG */
4693 cik_fini_pg(rdev);
4694 cik_fini_cg(rdev);
4695
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4696 /* stop the rlc */
4697 cik_rlc_stop(rdev);
4698
4699 /* Disable GFX parsing/prefetching */
4700 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4701
4702 /* Disable MEC parsing/prefetching */
4703 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4704
4705 if (reset_mask & RADEON_RESET_DMA) {
4706 /* sdma0 */
4707 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4708 tmp |= SDMA_HALT;
4709 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4710 }
4711 if (reset_mask & RADEON_RESET_DMA1) {
4712 /* sdma1 */
4713 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4714 tmp |= SDMA_HALT;
4715 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4716 }
4717
4718 evergreen_mc_stop(rdev, &save);
4719 if (evergreen_mc_wait_for_idle(rdev)) {
4720 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4721 }
4722
4723 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4724 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4725
4726 if (reset_mask & RADEON_RESET_CP) {
4727 grbm_soft_reset |= SOFT_RESET_CP;
4728
4729 srbm_soft_reset |= SOFT_RESET_GRBM;
4730 }
4731
4732 if (reset_mask & RADEON_RESET_DMA)
4733 srbm_soft_reset |= SOFT_RESET_SDMA;
4734
4735 if (reset_mask & RADEON_RESET_DMA1)
4736 srbm_soft_reset |= SOFT_RESET_SDMA1;
4737
4738 if (reset_mask & RADEON_RESET_DISPLAY)
4739 srbm_soft_reset |= SOFT_RESET_DC;
4740
4741 if (reset_mask & RADEON_RESET_RLC)
4742 grbm_soft_reset |= SOFT_RESET_RLC;
4743
4744 if (reset_mask & RADEON_RESET_SEM)
4745 srbm_soft_reset |= SOFT_RESET_SEM;
4746
4747 if (reset_mask & RADEON_RESET_IH)
4748 srbm_soft_reset |= SOFT_RESET_IH;
4749
4750 if (reset_mask & RADEON_RESET_GRBM)
4751 srbm_soft_reset |= SOFT_RESET_GRBM;
4752
4753 if (reset_mask & RADEON_RESET_VMC)
4754 srbm_soft_reset |= SOFT_RESET_VMC;
4755
4756 if (!(rdev->flags & RADEON_IS_IGP)) {
4757 if (reset_mask & RADEON_RESET_MC)
4758 srbm_soft_reset |= SOFT_RESET_MC;
4759 }
4760
4761 if (grbm_soft_reset) {
4762 tmp = RREG32(GRBM_SOFT_RESET);
4763 tmp |= grbm_soft_reset;
4764 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4765 WREG32(GRBM_SOFT_RESET, tmp);
4766 tmp = RREG32(GRBM_SOFT_RESET);
4767
4768 udelay(50);
4769
4770 tmp &= ~grbm_soft_reset;
4771 WREG32(GRBM_SOFT_RESET, tmp);
4772 tmp = RREG32(GRBM_SOFT_RESET);
4773 }
4774
4775 if (srbm_soft_reset) {
4776 tmp = RREG32(SRBM_SOFT_RESET);
4777 tmp |= srbm_soft_reset;
4778 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4779 WREG32(SRBM_SOFT_RESET, tmp);
4780 tmp = RREG32(SRBM_SOFT_RESET);
4781
4782 udelay(50);
4783
4784 tmp &= ~srbm_soft_reset;
4785 WREG32(SRBM_SOFT_RESET, tmp);
4786 tmp = RREG32(SRBM_SOFT_RESET);
4787 }
4788
4789 /* Wait a little for things to settle down */
4790 udelay(50);
4791
4792 evergreen_mc_resume(rdev, &save);
4793 udelay(50);
4794
4795 cik_print_gpu_status_regs(rdev);
4796}
4797
4798/**
4799 * cik_asic_reset - soft reset GPU
4800 *
4801 * @rdev: radeon_device pointer
4802 *
4803 * Look up which blocks are hung and attempt
4804 * to reset them.
4805 * Returns 0 for success.
4806 */
4807int cik_asic_reset(struct radeon_device *rdev)
4808{
4809 u32 reset_mask;
4810
4811 reset_mask = cik_gpu_check_soft_reset(rdev);
4812
4813 if (reset_mask)
4814 r600_set_bios_scratch_engine_hung(rdev, true);
4815
4816 cik_gpu_soft_reset(rdev, reset_mask);
4817
4818 reset_mask = cik_gpu_check_soft_reset(rdev);
4819
4820 if (!reset_mask)
4821 r600_set_bios_scratch_engine_hung(rdev, false);
4822
4823 return 0;
4824}
4825
4826/**
4827 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4828 *
4829 * @rdev: radeon_device pointer
4830 * @ring: radeon_ring structure holding ring information
4831 *
4832 * Check if the 3D engine is locked up (CIK).
4833 * Returns true if the engine is locked, false if not.
4834 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4835bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4836{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4837 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4838
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4839 if (!(reset_mask & (RADEON_RESET_GFX |
4840 RADEON_RESET_COMPUTE |
4841 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4842 radeon_ring_lockup_update(ring);
4843 return false;
4844 }
4845 /* force CP activities */
4846 radeon_ring_force_activity(rdev, ring);
4847 return radeon_ring_test_lockup(rdev, ring);
4848}
4849
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4850/* MC */
4851/**
4852 * cik_mc_program - program the GPU memory controller
4853 *
4854 * @rdev: radeon_device pointer
4855 *
4856 * Set the location of vram, gart, and AGP in the GPU's
4857 * physical address space (CIK).
4858 */
4859static void cik_mc_program(struct radeon_device *rdev)
4860{
4861 struct evergreen_mc_save save;
4862 u32 tmp;
4863 int i, j;
4864
4865 /* Initialize HDP */
4866 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4867 WREG32((0x2c14 + j), 0x00000000);
4868 WREG32((0x2c18 + j), 0x00000000);
4869 WREG32((0x2c1c + j), 0x00000000);
4870 WREG32((0x2c20 + j), 0x00000000);
4871 WREG32((0x2c24 + j), 0x00000000);
4872 }
4873 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4874
4875 evergreen_mc_stop(rdev, &save);
4876 if (radeon_mc_wait_for_idle(rdev)) {
4877 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4878 }
4879 /* Lockout access through VGA aperture*/
4880 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4881 /* Update configuration */
4882 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4883 rdev->mc.vram_start >> 12);
4884 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4885 rdev->mc.vram_end >> 12);
4886 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4887 rdev->vram_scratch.gpu_addr >> 12);
4888 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4889 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4890 WREG32(MC_VM_FB_LOCATION, tmp);
4891 /* XXX double check these! */
4892 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4893 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4894 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4895 WREG32(MC_VM_AGP_BASE, 0);
4896 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4897 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4898 if (radeon_mc_wait_for_idle(rdev)) {
4899 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4900 }
4901 evergreen_mc_resume(rdev, &save);
4902 /* we need to own VRAM, so turn off the VGA renderer here
4903 * to stop it overwriting our objects */
4904 rv515_vga_render_disable(rdev);
4905}
4906
4907/**
4908 * cik_mc_init - initialize the memory controller driver params
4909 *
4910 * @rdev: radeon_device pointer
4911 *
4912 * Look up the amount of vram, vram width, and decide how to place
4913 * vram and gart within the GPU's physical address space (CIK).
4914 * Returns 0 for success.
4915 */
4916static int cik_mc_init(struct radeon_device *rdev)
4917{
4918 u32 tmp;
4919 int chansize, numchan;
4920
4921 /* Get VRAM informations */
4922 rdev->mc.vram_is_ddr = true;
4923 tmp = RREG32(MC_ARB_RAMCFG);
4924 if (tmp & CHANSIZE_MASK) {
4925 chansize = 64;
4926 } else {
4927 chansize = 32;
4928 }
4929 tmp = RREG32(MC_SHARED_CHMAP);
4930 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4931 case 0:
4932 default:
4933 numchan = 1;
4934 break;
4935 case 1:
4936 numchan = 2;
4937 break;
4938 case 2:
4939 numchan = 4;
4940 break;
4941 case 3:
4942 numchan = 8;
4943 break;
4944 case 4:
4945 numchan = 3;
4946 break;
4947 case 5:
4948 numchan = 6;
4949 break;
4950 case 6:
4951 numchan = 10;
4952 break;
4953 case 7:
4954 numchan = 12;
4955 break;
4956 case 8:
4957 numchan = 16;
4958 break;
4959 }
4960 rdev->mc.vram_width = numchan * chansize;
4961 /* Could aper size report 0 ? */
4962 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4963 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4964 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]4965 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
4966 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4967 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4968 si_vram_gtt_location(rdev, &rdev->mc);
4969 radeon_update_bandwidth_info(rdev);
4970
4971 return 0;
4972}
4973
4974/*
4975 * GART
4976 * VMID 0 is the physical GPU addresses as used by the kernel.
4977 * VMIDs 1-15 are used for userspace clients and are handled
4978 * by the radeon vm/hsa code.
4979 */
4980/**
4981 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4982 *
4983 * @rdev: radeon_device pointer
4984 *
4985 * Flush the TLB for the VMID 0 page table (CIK).
4986 */
4987void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4988{
4989 /* flush hdp cache */
4990 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4991
4992 /* bits 0-15 are the VM contexts0-15 */
4993 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4994}
4995
4996/**
4997 * cik_pcie_gart_enable - gart enable
4998 *
4999 * @rdev: radeon_device pointer
5000 *
5001 * This sets up the TLBs, programs the page tables for VMID0,
5002 * sets up the hw for VMIDs 1-15 which are allocated on
5003 * demand, and sets up the global locations for the LDS, GDS,
5004 * and GPUVM for FSA64 clients (CIK).
5005 * Returns 0 for success, errors for failure.
5006 */
5007static int cik_pcie_gart_enable(struct radeon_device *rdev)
5008{
5009 int r, i;
5010
5011 if (rdev->gart.robj == NULL) {
5012 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5013 return -EINVAL;
5014 }
5015 r = radeon_gart_table_vram_pin(rdev);
5016 if (r)
5017 return r;
5018 radeon_gart_restore(rdev);
5019 /* Setup TLB control */
5020 WREG32(MC_VM_MX_L1_TLB_CNTL,
5021 (0xA << 7) |
5022 ENABLE_L1_TLB |
5023 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5024 ENABLE_ADVANCED_DRIVER_MODEL |
5025 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5026 /* Setup L2 cache */
5027 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5028 ENABLE_L2_FRAGMENT_PROCESSING |
5029 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5030 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5031 EFFECTIVE_L2_QUEUE_SIZE(7) |
5032 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5033 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5034 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5035 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5036 /* setup context0 */
5037 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5038 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5039 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5040 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5041 (u32)(rdev->dummy_page.addr >> 12));
5042 WREG32(VM_CONTEXT0_CNTL2, 0);
5043 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5044 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5045
5046 WREG32(0x15D4, 0);
5047 WREG32(0x15D8, 0);
5048 WREG32(0x15DC, 0);
5049
5050 /* empty context1-15 */
5051 /* FIXME start with 4G, once using 2 level pt switch to full
5052 * vm size space
5053 */
5054 /* set vm size, must be a multiple of 4 */
5055 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5056 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5057 for (i = 1; i < 16; i++) {
5058 if (i < 8)
5059 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5060 rdev->gart.table_addr >> 12);
5061 else
5062 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5063 rdev->gart.table_addr >> 12);
5064 }
5065
5066 /* enable context1-15 */
5067 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5068 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5069 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5070 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5071 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5072 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5073 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5074 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5075 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5076 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5077 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5078 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5079 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5080 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5081 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5082 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5083
5084 /* TC cache setup ??? */
5085 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5086 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5087 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5088
5089 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5090 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5091 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5092 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5093 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5094
5095 WREG32(TC_CFG_L1_VOLATILE, 0);
5096 WREG32(TC_CFG_L2_VOLATILE, 0);
5097
5098 if (rdev->family == CHIP_KAVERI) {
5099 u32 tmp = RREG32(CHUB_CONTROL);
5100 tmp &= ~BYPASS_VM;
5101 WREG32(CHUB_CONTROL, tmp);
5102 }
5103
5104 /* XXX SH_MEM regs */
5105 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5106 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5107 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5108 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5109 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5110 WREG32(SH_MEM_CONFIG, 0);
5111 WREG32(SH_MEM_APE1_BASE, 1);
5112 WREG32(SH_MEM_APE1_LIMIT, 0);
5113 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5114 /* SDMA GFX */
5115 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5116 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5117 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5118 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5119 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5120 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5121 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5122 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5123
5124 cik_pcie_gart_tlb_flush(rdev);
5125 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5126 (unsigned)(rdev->mc.gtt_size >> 20),
5127 (unsigned long long)rdev->gart.table_addr);
5128 rdev->gart.ready = true;
5129 return 0;
5130}
5131
5132/**
5133 * cik_pcie_gart_disable - gart disable
5134 *
5135 * @rdev: radeon_device pointer
5136 *
5137 * This disables all VM page table (CIK).
5138 */
5139static void cik_pcie_gart_disable(struct radeon_device *rdev)
5140{
5141 /* Disable all tables */
5142 WREG32(VM_CONTEXT0_CNTL, 0);
5143 WREG32(VM_CONTEXT1_CNTL, 0);
5144 /* Setup TLB control */
5145 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5146 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5147 /* Setup L2 cache */
5148 WREG32(VM_L2_CNTL,
5149 ENABLE_L2_FRAGMENT_PROCESSING |
5150 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5151 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5152 EFFECTIVE_L2_QUEUE_SIZE(7) |
5153 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5154 WREG32(VM_L2_CNTL2, 0);
5155 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5156 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5157 radeon_gart_table_vram_unpin(rdev);
5158}
5159
5160/**
5161 * cik_pcie_gart_fini - vm fini callback
5162 *
5163 * @rdev: radeon_device pointer
5164 *
5165 * Tears down the driver GART/VM setup (CIK).
5166 */
5167static void cik_pcie_gart_fini(struct radeon_device *rdev)
5168{
5169 cik_pcie_gart_disable(rdev);
5170 radeon_gart_table_vram_free(rdev);
5171 radeon_gart_fini(rdev);
5172}
5173
5174/* vm parser */
5175/**
5176 * cik_ib_parse - vm ib_parse callback
5177 *
5178 * @rdev: radeon_device pointer
5179 * @ib: indirect buffer pointer
5180 *
5181 * CIK uses hw IB checking so this is a nop (CIK).
5182 */
5183int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5184{
5185 return 0;
5186}
5187
5188/*
5189 * vm
5190 * VMID 0 is the physical GPU addresses as used by the kernel.
5191 * VMIDs 1-15 are used for userspace clients and are handled
5192 * by the radeon vm/hsa code.
5193 */
5194/**
5195 * cik_vm_init - cik vm init callback
5196 *
5197 * @rdev: radeon_device pointer
5198 *
5199 * Inits cik specific vm parameters (number of VMs, base of vram for
5200 * VMIDs 1-15) (CIK).
5201 * Returns 0 for success.
5202 */
5203int cik_vm_init(struct radeon_device *rdev)
5204{
5205 /* number of VMs */
5206 rdev->vm_manager.nvm = 16;
5207 /* base offset of vram pages */
5208 if (rdev->flags & RADEON_IS_IGP) {
5209 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5210 tmp <<= 22;
5211 rdev->vm_manager.vram_base_offset = tmp;
5212 } else
5213 rdev->vm_manager.vram_base_offset = 0;
5214
5215 return 0;
5216}
5217
5218/**
5219 * cik_vm_fini - cik vm fini callback
5220 *
5221 * @rdev: radeon_device pointer
5222 *
5223 * Tear down any asic specific VM setup (CIK).
5224 */
5225void cik_vm_fini(struct radeon_device *rdev)
5226{
5227}
5228
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5229/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5230 * cik_vm_decode_fault - print human readable fault info
5231 *
5232 * @rdev: radeon_device pointer
5233 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5234 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5235 *
5236 * Print human readable fault information (CIK).
5237 */
5238static void cik_vm_decode_fault(struct radeon_device *rdev,
5239 u32 status, u32 addr, u32 mc_client)
5240{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5241 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5242 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5243 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5244 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5245 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5246
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5247 if (rdev->family == CHIP_HAWAII)
5248 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5249 else
5250 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5251
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5252 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5253 protections, vmid, addr,
5254 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5255 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5256}
5257
5258/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5259 * cik_vm_flush - cik vm flush using the CP
5260 *
5261 * @rdev: radeon_device pointer
5262 *
5263 * Update the page table base and flush the VM TLB
5264 * using the CP (CIK).
5265 */
5266void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5267{
5268 struct radeon_ring *ring = &rdev->ring[ridx];
5269
5270 if (vm == NULL)
5271 return;
5272
5273 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5274 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5275 WRITE_DATA_DST_SEL(0)));
5276 if (vm->id < 8) {
5277 radeon_ring_write(ring,
5278 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5279 } else {
5280 radeon_ring_write(ring,
5281 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5282 }
5283 radeon_ring_write(ring, 0);
5284 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5285
5286 /* update SH_MEM_* regs */
5287 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5288 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5289 WRITE_DATA_DST_SEL(0)));
5290 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5291 radeon_ring_write(ring, 0);
5292 radeon_ring_write(ring, VMID(vm->id));
5293
5294 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5295 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5296 WRITE_DATA_DST_SEL(0)));
5297 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5298 radeon_ring_write(ring, 0);
5299
5300 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5301 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5302 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5303 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5304
5305 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5306 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5307 WRITE_DATA_DST_SEL(0)));
5308 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5309 radeon_ring_write(ring, 0);
5310 radeon_ring_write(ring, VMID(0));
5311
5312 /* HDP flush */
5313 /* We should be using the WAIT_REG_MEM packet here like in
5314 * cik_fence_ring_emit(), but it causes the CP to hang in this
5315 * context...
5316 */
5317 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5318 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5319 WRITE_DATA_DST_SEL(0)));
5320 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5321 radeon_ring_write(ring, 0);
5322 radeon_ring_write(ring, 0);
5323
5324 /* bits 0-15 are the VM contexts0-15 */
5325 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5326 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5327 WRITE_DATA_DST_SEL(0)));
5328 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5329 radeon_ring_write(ring, 0);
5330 radeon_ring_write(ring, 1 << vm->id);
5331
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5332 /* compute doesn't have PFP */
5333 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5334 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5335 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5336 radeon_ring_write(ring, 0x0);
5337 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5338}
5339
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5340/*
5341 * RLC
5342 * The RLC is a multi-purpose microengine that handles a
5343 * variety of functions, the most important of which is
5344 * the interrupt controller.
5345 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5346static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5347 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5348{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5349 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5350
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5351 if (enable)
5352 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5353 else
5354 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5355 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5356}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5357
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5358static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5359{
5360 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5361
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5362 tmp = RREG32(RLC_LB_CNTL);
5363 if (enable)
5364 tmp |= LOAD_BALANCE_ENABLE;
5365 else
5366 tmp &= ~LOAD_BALANCE_ENABLE;
5367 WREG32(RLC_LB_CNTL, tmp);
5368}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5369
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5370static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5371{
5372 u32 i, j, k;
5373 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5374
5375 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5376 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5377 cik_select_se_sh(rdev, i, j);
5378 for (k = 0; k < rdev->usec_timeout; k++) {
5379 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5380 break;
5381 udelay(1);
5382 }
5383 }
5384 }
5385 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5386
5387 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5388 for (k = 0; k < rdev->usec_timeout; k++) {
5389 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5390 break;
5391 udelay(1);
5392 }
5393}
5394
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5395static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5396{
5397 u32 tmp;
5398
5399 tmp = RREG32(RLC_CNTL);
5400 if (tmp != rlc)
5401 WREG32(RLC_CNTL, rlc);
5402}
5403
5404static u32 cik_halt_rlc(struct radeon_device *rdev)
5405{
5406 u32 data, orig;
5407
5408 orig = data = RREG32(RLC_CNTL);
5409
5410 if (data & RLC_ENABLE) {
5411 u32 i;
5412
5413 data &= ~RLC_ENABLE;
5414 WREG32(RLC_CNTL, data);
5415
5416 for (i = 0; i < rdev->usec_timeout; i++) {
5417 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5418 break;
5419 udelay(1);
5420 }
5421
5422 cik_wait_for_rlc_serdes(rdev);
5423 }
5424
5425 return orig;
5426}
5427
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5428void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5429{
5430 u32 tmp, i, mask;
5431
5432 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5433 WREG32(RLC_GPR_REG2, tmp);
5434
5435 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5436 for (i = 0; i < rdev->usec_timeout; i++) {
5437 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5438 break;
5439 udelay(1);
5440 }
5441
5442 for (i = 0; i < rdev->usec_timeout; i++) {
5443 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5444 break;
5445 udelay(1);
5446 }
5447}
5448
5449void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5450{
5451 u32 tmp;
5452
5453 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5454 WREG32(RLC_GPR_REG2, tmp);
5455}
5456
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5457/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5458 * cik_rlc_stop - stop the RLC ME
5459 *
5460 * @rdev: radeon_device pointer
5461 *
5462 * Halt the RLC ME (MicroEngine) (CIK).
5463 */
5464static void cik_rlc_stop(struct radeon_device *rdev)
5465{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5466 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5467
5468 cik_enable_gui_idle_interrupt(rdev, false);
5469
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5470 cik_wait_for_rlc_serdes(rdev);
5471}
5472
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5473/**
5474 * cik_rlc_start - start the RLC ME
5475 *
5476 * @rdev: radeon_device pointer
5477 *
5478 * Unhalt the RLC ME (MicroEngine) (CIK).
5479 */
5480static void cik_rlc_start(struct radeon_device *rdev)
5481{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5482 WREG32(RLC_CNTL, RLC_ENABLE);
5483
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5484 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5485
5486 udelay(50);
5487}
5488
5489/**
5490 * cik_rlc_resume - setup the RLC hw
5491 *
5492 * @rdev: radeon_device pointer
5493 *
5494 * Initialize the RLC registers, load the ucode,
5495 * and start the RLC (CIK).
5496 * Returns 0 for success, -EINVAL if the ucode is not available.
5497 */
5498static int cik_rlc_resume(struct radeon_device *rdev)
5499{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5500 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5501 const __be32 *fw_data;
5502
5503 if (!rdev->rlc_fw)
5504 return -EINVAL;
5505
5506 switch (rdev->family) {
5507 case CHIP_BONAIRE:
5508 default:
5509 size = BONAIRE_RLC_UCODE_SIZE;
5510 break;
5511 case CHIP_KAVERI:
5512 size = KV_RLC_UCODE_SIZE;
5513 break;
5514 case CHIP_KABINI:
5515 size = KB_RLC_UCODE_SIZE;
5516 break;
5517 }
5518
5519 cik_rlc_stop(rdev);
5520
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5521 /* disable CG */
5522 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5523 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5524
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5525 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5526
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5527 cik_init_pg(rdev);
5528
5529 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5530
5531 WREG32(RLC_LB_CNTR_INIT, 0);
5532 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5533
5534 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5535 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5536 WREG32(RLC_LB_PARAMS, 0x00600408);
5537 WREG32(RLC_LB_CNTL, 0x80000004);
5538
5539 WREG32(RLC_MC_CNTL, 0);
5540 WREG32(RLC_UCODE_CNTL, 0);
5541
5542 fw_data = (const __be32 *)rdev->rlc_fw->data;
5543 WREG32(RLC_GPM_UCODE_ADDR, 0);
5544 for (i = 0; i < size; i++)
5545 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5546 WREG32(RLC_GPM_UCODE_ADDR, 0);
5547
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5548 /* XXX - find out what chips support lbpw */
5549 cik_enable_lbpw(rdev, false);
5550
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5551 if (rdev->family == CHIP_BONAIRE)
5552 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5553
5554 cik_rlc_start(rdev);
5555
5556 return 0;
5557}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5558
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5559static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5560{
5561 u32 data, orig, tmp, tmp2;
5562
5563 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5564
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5565 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5566 cik_enable_gui_idle_interrupt(rdev, true);
5567
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5568 tmp = cik_halt_rlc(rdev);
5569
5570 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5571 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5572 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5573 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5574 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5575
5576 cik_update_rlc(rdev, tmp);
5577
5578 data |= CGCG_EN | CGLS_EN;
5579 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5580 cik_enable_gui_idle_interrupt(rdev, false);
5581
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5582 RREG32(CB_CGTT_SCLK_CTRL);
5583 RREG32(CB_CGTT_SCLK_CTRL);
5584 RREG32(CB_CGTT_SCLK_CTRL);
5585 RREG32(CB_CGTT_SCLK_CTRL);
5586
5587 data &= ~(CGCG_EN | CGLS_EN);
5588 }
5589
5590 if (orig != data)
5591 WREG32(RLC_CGCG_CGLS_CTRL, data);
5592
5593}
5594
5595static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5596{
5597 u32 data, orig, tmp = 0;
5598
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5599 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5600 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5601 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5602 orig = data = RREG32(CP_MEM_SLP_CNTL);
5603 data |= CP_MEM_LS_EN;
5604 if (orig != data)
5605 WREG32(CP_MEM_SLP_CNTL, data);
5606 }
5607 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5608
5609 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5610 data &= 0xfffffffd;
5611 if (orig != data)
5612 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5613
5614 tmp = cik_halt_rlc(rdev);
5615
5616 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5617 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5618 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5619 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5620 WREG32(RLC_SERDES_WR_CTRL, data);
5621
5622 cik_update_rlc(rdev, tmp);
5623
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5624 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5625 orig = data = RREG32(CGTS_SM_CTRL_REG);
5626 data &= ~SM_MODE_MASK;
5627 data |= SM_MODE(0x2);
5628 data |= SM_MODE_ENABLE;
5629 data &= ~CGTS_OVERRIDE;
5630 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5631 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5632 data &= ~CGTS_LS_OVERRIDE;
5633 data &= ~ON_MONITOR_ADD_MASK;
5634 data |= ON_MONITOR_ADD_EN;
5635 data |= ON_MONITOR_ADD(0x96);
5636 if (orig != data)
5637 WREG32(CGTS_SM_CTRL_REG, data);
5638 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5639 } else {
5640 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5641 data |= 0x00000002;
5642 if (orig != data)
5643 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5644
5645 data = RREG32(RLC_MEM_SLP_CNTL);
5646 if (data & RLC_MEM_LS_EN) {
5647 data &= ~RLC_MEM_LS_EN;
5648 WREG32(RLC_MEM_SLP_CNTL, data);
5649 }
5650
5651 data = RREG32(CP_MEM_SLP_CNTL);
5652 if (data & CP_MEM_LS_EN) {
5653 data &= ~CP_MEM_LS_EN;
5654 WREG32(CP_MEM_SLP_CNTL, data);
5655 }
5656
5657 orig = data = RREG32(CGTS_SM_CTRL_REG);
5658 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5659 if (orig != data)
5660 WREG32(CGTS_SM_CTRL_REG, data);
5661
5662 tmp = cik_halt_rlc(rdev);
5663
5664 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5665 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5666 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5667 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5668 WREG32(RLC_SERDES_WR_CTRL, data);
5669
5670 cik_update_rlc(rdev, tmp);
5671 }
5672}
5673
5674static const u32 mc_cg_registers[] =
5675{
5676 MC_HUB_MISC_HUB_CG,
5677 MC_HUB_MISC_SIP_CG,
5678 MC_HUB_MISC_VM_CG,
5679 MC_XPB_CLK_GAT,
5680 ATC_MISC_CG,
5681 MC_CITF_MISC_WR_CG,
5682 MC_CITF_MISC_RD_CG,
5683 MC_CITF_MISC_VM_CG,
5684 VM_L2_CG,
5685};
5686
5687static void cik_enable_mc_ls(struct radeon_device *rdev,
5688 bool enable)
5689{
5690 int i;
5691 u32 orig, data;
5692
5693 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5694 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5695 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5696 data |= MC_LS_ENABLE;
5697 else
5698 data &= ~MC_LS_ENABLE;
5699 if (data != orig)
5700 WREG32(mc_cg_registers[i], data);
5701 }
5702}
5703
5704static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5705 bool enable)
5706{
5707 int i;
5708 u32 orig, data;
5709
5710 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5711 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5712 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5713 data |= MC_CG_ENABLE;
5714 else
5715 data &= ~MC_CG_ENABLE;
5716 if (data != orig)
5717 WREG32(mc_cg_registers[i], data);
5718 }
5719}
5720
5721static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5722 bool enable)
5723{
5724 u32 orig, data;
5725
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5726 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5727 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5728 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5729 } else {
5730 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5731 data |= 0xff000000;
5732 if (data != orig)
5733 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5734
5735 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5736 data |= 0xff000000;
5737 if (data != orig)
5738 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5739 }
5740}
5741
5742static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5743 bool enable)
5744{
5745 u32 orig, data;
5746
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5747 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5748 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5749 data |= 0x100;
5750 if (orig != data)
5751 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5752
5753 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5754 data |= 0x100;
5755 if (orig != data)
5756 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5757 } else {
5758 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5759 data &= ~0x100;
5760 if (orig != data)
5761 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5762
5763 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5764 data &= ~0x100;
5765 if (orig != data)
5766 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5767 }
5768}
5769
5770static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5771 bool enable)
5772{
5773 u32 orig, data;
5774
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5775 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5776 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5777 data = 0xfff;
5778 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5779
5780 orig = data = RREG32(UVD_CGC_CTRL);
5781 data |= DCM;
5782 if (orig != data)
5783 WREG32(UVD_CGC_CTRL, data);
5784 } else {
5785 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5786 data &= ~0xfff;
5787 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5788
5789 orig = data = RREG32(UVD_CGC_CTRL);
5790 data &= ~DCM;
5791 if (orig != data)
5792 WREG32(UVD_CGC_CTRL, data);
5793 }
5794}
5795
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5796static void cik_enable_bif_mgls(struct radeon_device *rdev,
5797 bool enable)
5798{
5799 u32 orig, data;
5800
5801 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5802
5803 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5804 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5805 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5806 else
5807 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5808 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5809
5810 if (orig != data)
5811 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5812}
5813
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5814static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5815 bool enable)
5816{
5817 u32 orig, data;
5818
5819 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5820
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5821 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5822 data &= ~CLOCK_GATING_DIS;
5823 else
5824 data |= CLOCK_GATING_DIS;
5825
5826 if (orig != data)
5827 WREG32(HDP_HOST_PATH_CNTL, data);
5828}
5829
5830static void cik_enable_hdp_ls(struct radeon_device *rdev,
5831 bool enable)
5832{
5833 u32 orig, data;
5834
5835 orig = data = RREG32(HDP_MEM_POWER_LS);
5836
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5837 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5838 data |= HDP_LS_ENABLE;
5839 else
5840 data &= ~HDP_LS_ENABLE;
5841
5842 if (orig != data)
5843 WREG32(HDP_MEM_POWER_LS, data);
5844}
5845
5846void cik_update_cg(struct radeon_device *rdev,
5847 u32 block, bool enable)
5848{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5849
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5850 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5851 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5852 /* order matters! */
5853 if (enable) {
5854 cik_enable_mgcg(rdev, true);
5855 cik_enable_cgcg(rdev, true);
5856 } else {
5857 cik_enable_cgcg(rdev, false);
5858 cik_enable_mgcg(rdev, false);
5859 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5860 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5861 }
5862
5863 if (block & RADEON_CG_BLOCK_MC) {
5864 if (!(rdev->flags & RADEON_IS_IGP)) {
5865 cik_enable_mc_mgcg(rdev, enable);
5866 cik_enable_mc_ls(rdev, enable);
5867 }
5868 }
5869
5870 if (block & RADEON_CG_BLOCK_SDMA) {
5871 cik_enable_sdma_mgcg(rdev, enable);
5872 cik_enable_sdma_mgls(rdev, enable);
5873 }
5874
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5875 if (block & RADEON_CG_BLOCK_BIF) {
5876 cik_enable_bif_mgls(rdev, enable);
5877 }
5878
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5879 if (block & RADEON_CG_BLOCK_UVD) {
5880 if (rdev->has_uvd)
5881 cik_enable_uvd_mgcg(rdev, enable);
5882 }
5883
5884 if (block & RADEON_CG_BLOCK_HDP) {
5885 cik_enable_hdp_mgcg(rdev, enable);
5886 cik_enable_hdp_ls(rdev, enable);
5887 }
5888}
5889
5890static void cik_init_cg(struct radeon_device *rdev)
5891{
5892
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5893 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5894
5895 if (rdev->has_uvd)
5896 si_init_uvd_internal_cg(rdev);
5897
5898 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5899 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5900 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5901 RADEON_CG_BLOCK_UVD |
5902 RADEON_CG_BLOCK_HDP), true);
5903}
5904
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5905static void cik_fini_cg(struct radeon_device *rdev)
5906{
5907 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5908 RADEON_CG_BLOCK_SDMA |
5909 RADEON_CG_BLOCK_BIF |
5910 RADEON_CG_BLOCK_UVD |
5911 RADEON_CG_BLOCK_HDP), false);
5912
5913 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5914}
5915
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5916static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5917 bool enable)
5918{
5919 u32 data, orig;
5920
5921 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5922 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5923 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5924 else
5925 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5926 if (orig != data)
5927 WREG32(RLC_PG_CNTL, data);
5928}
5929
5930static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5931 bool enable)
5932{
5933 u32 data, orig;
5934
5935 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5936 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5937 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5938 else
5939 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5940 if (orig != data)
5941 WREG32(RLC_PG_CNTL, data);
5942}
5943
5944static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5945{
5946 u32 data, orig;
5947
5948 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5949 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5950 data &= ~DISABLE_CP_PG;
5951 else
5952 data |= DISABLE_CP_PG;
5953 if (orig != data)
5954 WREG32(RLC_PG_CNTL, data);
5955}
5956
5957static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5958{
5959 u32 data, orig;
5960
5961 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5962 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5963 data &= ~DISABLE_GDS_PG;
5964 else
5965 data |= DISABLE_GDS_PG;
5966 if (orig != data)
5967 WREG32(RLC_PG_CNTL, data);
5968}
5969
5970#define CP_ME_TABLE_SIZE 96
5971#define CP_ME_TABLE_OFFSET 2048
5972#define CP_MEC_TABLE_OFFSET 4096
5973
5974void cik_init_cp_pg_table(struct radeon_device *rdev)
5975{
5976 const __be32 *fw_data;
5977 volatile u32 *dst_ptr;
5978 int me, i, max_me = 4;
5979 u32 bo_offset = 0;
5980 u32 table_offset;
5981
5982 if (rdev->family == CHIP_KAVERI)
5983 max_me = 5;
5984
5985 if (rdev->rlc.cp_table_ptr == NULL)
5986 return;
5987
5988 /* write the cp table buffer */
5989 dst_ptr = rdev->rlc.cp_table_ptr;
5990 for (me = 0; me < max_me; me++) {
5991 if (me == 0) {
5992 fw_data = (const __be32 *)rdev->ce_fw->data;
5993 table_offset = CP_ME_TABLE_OFFSET;
5994 } else if (me == 1) {
5995 fw_data = (const __be32 *)rdev->pfp_fw->data;
5996 table_offset = CP_ME_TABLE_OFFSET;
5997 } else if (me == 2) {
5998 fw_data = (const __be32 *)rdev->me_fw->data;
5999 table_offset = CP_ME_TABLE_OFFSET;
6000 } else {
6001 fw_data = (const __be32 *)rdev->mec_fw->data;
6002 table_offset = CP_MEC_TABLE_OFFSET;
6003 }
6004
6005 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6006 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6007 }
6008 bo_offset += CP_ME_TABLE_SIZE;
6009 }
6010}
6011
6012static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6013 bool enable)
6014{
6015 u32 data, orig;
6016
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6017 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6018 orig = data = RREG32(RLC_PG_CNTL);
6019 data |= GFX_PG_ENABLE;
6020 if (orig != data)
6021 WREG32(RLC_PG_CNTL, data);
6022
6023 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6024 data |= AUTO_PG_EN;
6025 if (orig != data)
6026 WREG32(RLC_AUTO_PG_CTRL, data);
6027 } else {
6028 orig = data = RREG32(RLC_PG_CNTL);
6029 data &= ~GFX_PG_ENABLE;
6030 if (orig != data)
6031 WREG32(RLC_PG_CNTL, data);
6032
6033 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6034 data &= ~AUTO_PG_EN;
6035 if (orig != data)
6036 WREG32(RLC_AUTO_PG_CTRL, data);
6037
6038 data = RREG32(DB_RENDER_CONTROL);
6039 }
6040}
6041
6042static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6043{
6044 u32 mask = 0, tmp, tmp1;
6045 int i;
6046
6047 cik_select_se_sh(rdev, se, sh);
6048 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6049 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6050 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6051
6052 tmp &= 0xffff0000;
6053
6054 tmp |= tmp1;
6055 tmp >>= 16;
6056
6057 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6058 mask <<= 1;
6059 mask |= 1;
6060 }
6061
6062 return (~tmp) & mask;
6063}
6064
6065static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6066{
6067 u32 i, j, k, active_cu_number = 0;
6068 u32 mask, counter, cu_bitmap;
6069 u32 tmp = 0;
6070
6071 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6072 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6073 mask = 1;
6074 cu_bitmap = 0;
6075 counter = 0;
6076 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6077 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6078 if (counter < 2)
6079 cu_bitmap |= mask;
6080 counter ++;
6081 }
6082 mask <<= 1;
6083 }
6084
6085 active_cu_number += counter;
6086 tmp |= (cu_bitmap << (i * 16 + j * 8));
6087 }
6088 }
6089
6090 WREG32(RLC_PG_AO_CU_MASK, tmp);
6091
6092 tmp = RREG32(RLC_MAX_PG_CU);
6093 tmp &= ~MAX_PU_CU_MASK;
6094 tmp |= MAX_PU_CU(active_cu_number);
6095 WREG32(RLC_MAX_PG_CU, tmp);
6096}
6097
6098static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6099 bool enable)
6100{
6101 u32 data, orig;
6102
6103 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6104 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6105 data |= STATIC_PER_CU_PG_ENABLE;
6106 else
6107 data &= ~STATIC_PER_CU_PG_ENABLE;
6108 if (orig != data)
6109 WREG32(RLC_PG_CNTL, data);
6110}
6111
6112static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6113 bool enable)
6114{
6115 u32 data, orig;
6116
6117 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6118 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6119 data |= DYN_PER_CU_PG_ENABLE;
6120 else
6121 data &= ~DYN_PER_CU_PG_ENABLE;
6122 if (orig != data)
6123 WREG32(RLC_PG_CNTL, data);
6124}
6125
6126#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6127#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6128
6129static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6130{
6131 u32 data, orig;
6132 u32 i;
6133
6134 if (rdev->rlc.cs_data) {
6135 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6136 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6137 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6138 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6139 } else {
6140 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6141 for (i = 0; i < 3; i++)
6142 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6143 }
6144 if (rdev->rlc.reg_list) {
6145 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6146 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6147 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6148 }
6149
6150 orig = data = RREG32(RLC_PG_CNTL);
6151 data |= GFX_PG_SRC;
6152 if (orig != data)
6153 WREG32(RLC_PG_CNTL, data);
6154
6155 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6156 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6157
6158 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6159 data &= ~IDLE_POLL_COUNT_MASK;
6160 data |= IDLE_POLL_COUNT(0x60);
6161 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6162
6163 data = 0x10101010;
6164 WREG32(RLC_PG_DELAY, data);
6165
6166 data = RREG32(RLC_PG_DELAY_2);
6167 data &= ~0xff;
6168 data |= 0x3;
6169 WREG32(RLC_PG_DELAY_2, data);
6170
6171 data = RREG32(RLC_AUTO_PG_CTRL);
6172 data &= ~GRBM_REG_SGIT_MASK;
6173 data |= GRBM_REG_SGIT(0x700);
6174 WREG32(RLC_AUTO_PG_CTRL, data);
6175
6176}
6177
6178static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6179{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6180 cik_enable_gfx_cgpg(rdev, enable);
6181 cik_enable_gfx_static_mgpg(rdev, enable);
6182 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6183}
6184
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6185u32 cik_get_csb_size(struct radeon_device *rdev)
6186{
6187 u32 count = 0;
6188 const struct cs_section_def *sect = NULL;
6189 const struct cs_extent_def *ext = NULL;
6190
6191 if (rdev->rlc.cs_data == NULL)
6192 return 0;
6193
6194 /* begin clear state */
6195 count += 2;
6196 /* context control state */
6197 count += 3;
6198
6199 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6200 for (ext = sect->section; ext->extent != NULL; ++ext) {
6201 if (sect->id == SECT_CONTEXT)
6202 count += 2 + ext->reg_count;
6203 else
6204 return 0;
6205 }
6206 }
6207 /* pa_sc_raster_config/pa_sc_raster_config1 */
6208 count += 4;
6209 /* end clear state */
6210 count += 2;
6211 /* clear state */
6212 count += 2;
6213
6214 return count;
6215}
6216
6217void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6218{
6219 u32 count = 0, i;
6220 const struct cs_section_def *sect = NULL;
6221 const struct cs_extent_def *ext = NULL;
6222
6223 if (rdev->rlc.cs_data == NULL)
6224 return;
6225 if (buffer == NULL)
6226 return;
6227
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6228 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6229 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6230
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6231 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6232 buffer[count++] = cpu_to_le32(0x80000000);
6233 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6234
6235 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6236 for (ext = sect->section; ext->extent != NULL; ++ext) {
6237 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6238 buffer[count++] =
6239 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6240 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6241 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6242 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6243 } else {
6244 return;
6245 }
6246 }
6247 }
6248
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6249 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6250 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6251 switch (rdev->family) {
6252 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6253 buffer[count++] = cpu_to_le32(0x16000012);
6254 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6255 break;
6256 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6257 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6258 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6259 break;
6260 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6261 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6262 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6263 break;
6264 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6265 buffer[count++] = cpu_to_le32(0x00000000);
6266 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6267 break;
6268 }
6269
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6270 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6271 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6272
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6273 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6274 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6275}
6276
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6277static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6278{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6279 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6280 cik_enable_sck_slowdown_on_pu(rdev, true);
6281 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6282 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6283 cik_init_gfx_cgpg(rdev);
6284 cik_enable_cp_pg(rdev, true);
6285 cik_enable_gds_pg(rdev, true);
6286 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6287 cik_init_ao_cu_mask(rdev);
6288 cik_update_gfx_pg(rdev, true);
6289 }
6290}
6291
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6292static void cik_fini_pg(struct radeon_device *rdev)
6293{
6294 if (rdev->pg_flags) {
6295 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6296 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6297 cik_enable_cp_pg(rdev, false);
6298 cik_enable_gds_pg(rdev, false);
6299 }
6300 }
6301}
6302
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6303/*
6304 * Interrupts
6305 * Starting with r6xx, interrupts are handled via a ring buffer.
6306 * Ring buffers are areas of GPU accessible memory that the GPU
6307 * writes interrupt vectors into and the host reads vectors out of.
6308 * There is a rptr (read pointer) that determines where the
6309 * host is currently reading, and a wptr (write pointer)
6310 * which determines where the GPU has written. When the
6311 * pointers are equal, the ring is idle. When the GPU
6312 * writes vectors to the ring buffer, it increments the
6313 * wptr. When there is an interrupt, the host then starts
6314 * fetching commands and processing them until the pointers are
6315 * equal again at which point it updates the rptr.
6316 */
6317
6318/**
6319 * cik_enable_interrupts - Enable the interrupt ring buffer
6320 *
6321 * @rdev: radeon_device pointer
6322 *
6323 * Enable the interrupt ring buffer (CIK).
6324 */
6325static void cik_enable_interrupts(struct radeon_device *rdev)
6326{
6327 u32 ih_cntl = RREG32(IH_CNTL);
6328 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6329
6330 ih_cntl |= ENABLE_INTR;
6331 ih_rb_cntl |= IH_RB_ENABLE;
6332 WREG32(IH_CNTL, ih_cntl);
6333 WREG32(IH_RB_CNTL, ih_rb_cntl);
6334 rdev->ih.enabled = true;
6335}
6336
6337/**
6338 * cik_disable_interrupts - Disable the interrupt ring buffer
6339 *
6340 * @rdev: radeon_device pointer
6341 *
6342 * Disable the interrupt ring buffer (CIK).
6343 */
6344static void cik_disable_interrupts(struct radeon_device *rdev)
6345{
6346 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6347 u32 ih_cntl = RREG32(IH_CNTL);
6348
6349 ih_rb_cntl &= ~IH_RB_ENABLE;
6350 ih_cntl &= ~ENABLE_INTR;
6351 WREG32(IH_RB_CNTL, ih_rb_cntl);
6352 WREG32(IH_CNTL, ih_cntl);
6353 /* set rptr, wptr to 0 */
6354 WREG32(IH_RB_RPTR, 0);
6355 WREG32(IH_RB_WPTR, 0);
6356 rdev->ih.enabled = false;
6357 rdev->ih.rptr = 0;
6358}
6359
6360/**
6361 * cik_disable_interrupt_state - Disable all interrupt sources
6362 *
6363 * @rdev: radeon_device pointer
6364 *
6365 * Clear all interrupt enable bits used by the driver (CIK).
6366 */
6367static void cik_disable_interrupt_state(struct radeon_device *rdev)
6368{
6369 u32 tmp;
6370
6371 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6372 tmp = RREG32(CP_INT_CNTL_RING0) &
6373 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6374 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6375 /* sdma */
6376 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6377 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6378 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6379 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6380 /* compute queues */
6381 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6382 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6383 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6384 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6385 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6386 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6387 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6388 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6389 /* grbm */
6390 WREG32(GRBM_INT_CNTL, 0);
6391 /* vline/vblank, etc. */
6392 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6393 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6394 if (rdev->num_crtc >= 4) {
6395 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6396 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6397 }
6398 if (rdev->num_crtc >= 6) {
6399 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6400 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6401 }
6402
6403 /* dac hotplug */
6404 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6405
6406 /* digital hotplug */
6407 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6408 WREG32(DC_HPD1_INT_CONTROL, tmp);
6409 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6410 WREG32(DC_HPD2_INT_CONTROL, tmp);
6411 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6412 WREG32(DC_HPD3_INT_CONTROL, tmp);
6413 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6414 WREG32(DC_HPD4_INT_CONTROL, tmp);
6415 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6416 WREG32(DC_HPD5_INT_CONTROL, tmp);
6417 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6418 WREG32(DC_HPD6_INT_CONTROL, tmp);
6419
6420}
6421
6422/**
6423 * cik_irq_init - init and enable the interrupt ring
6424 *
6425 * @rdev: radeon_device pointer
6426 *
6427 * Allocate a ring buffer for the interrupt controller,
6428 * enable the RLC, disable interrupts, enable the IH
6429 * ring buffer and enable it (CIK).
6430 * Called at device load and reume.
6431 * Returns 0 for success, errors for failure.
6432 */
6433static int cik_irq_init(struct radeon_device *rdev)
6434{
6435 int ret = 0;
6436 int rb_bufsz;
6437 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6438
6439 /* allocate ring */
6440 ret = r600_ih_ring_alloc(rdev);
6441 if (ret)
6442 return ret;
6443
6444 /* disable irqs */
6445 cik_disable_interrupts(rdev);
6446
6447 /* init rlc */
6448 ret = cik_rlc_resume(rdev);
6449 if (ret) {
6450 r600_ih_ring_fini(rdev);
6451 return ret;
6452 }
6453
6454 /* setup interrupt control */
6455 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6456 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6457 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6458 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6459 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6460 */
6461 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6462 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6463 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6464 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6465
6466 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6467 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6468
6469 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6470 IH_WPTR_OVERFLOW_CLEAR |
6471 (rb_bufsz << 1));
6472
6473 if (rdev->wb.enabled)
6474 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6475
6476 /* set the writeback address whether it's enabled or not */
6477 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6478 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6479
6480 WREG32(IH_RB_CNTL, ih_rb_cntl);
6481
6482 /* set rptr, wptr to 0 */
6483 WREG32(IH_RB_RPTR, 0);
6484 WREG32(IH_RB_WPTR, 0);
6485
6486 /* Default settings for IH_CNTL (disabled at first) */
6487 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6488 /* RPTR_REARM only works if msi's are enabled */
6489 if (rdev->msi_enabled)
6490 ih_cntl |= RPTR_REARM;
6491 WREG32(IH_CNTL, ih_cntl);
6492
6493 /* force the active interrupt state to all disabled */
6494 cik_disable_interrupt_state(rdev);
6495
6496 pci_set_master(rdev->pdev);
6497
6498 /* enable irqs */
6499 cik_enable_interrupts(rdev);
6500
6501 return ret;
6502}
6503
6504/**
6505 * cik_irq_set - enable/disable interrupt sources
6506 *
6507 * @rdev: radeon_device pointer
6508 *
6509 * Enable interrupt sources on the GPU (vblanks, hpd,
6510 * etc.) (CIK).
6511 * Returns 0 for success, errors for failure.
6512 */
6513int cik_irq_set(struct radeon_device *rdev)
6514{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6515 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6516 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6517 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6518 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6519 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6520 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6521 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6522 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6523
6524 if (!rdev->irq.installed) {
6525 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6526 return -EINVAL;
6527 }
6528 /* don't enable anything if the ih is disabled */
6529 if (!rdev->ih.enabled) {
6530 cik_disable_interrupts(rdev);
6531 /* force the active interrupt state to all disabled */
6532 cik_disable_interrupt_state(rdev);
6533 return 0;
6534 }
6535
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6536 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6537 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6538 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6539
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6540 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6541 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6542 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6543 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6544 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6545 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6546
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6547 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6548 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6549
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6550 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6551 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6552 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6553 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6554 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6555 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6556 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6557 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6558
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6559 if (rdev->flags & RADEON_IS_IGP)
6560 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6561 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6562 else
6563 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6564 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6565
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6566 /* enable CP interrupts on all rings */
6567 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6568 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6569 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6570 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6571 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6572 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6573 DRM_DEBUG("si_irq_set: sw int cp1\n");
6574 if (ring->me == 1) {
6575 switch (ring->pipe) {
6576 case 0:
6577 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6578 break;
6579 case 1:
6580 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6581 break;
6582 case 2:
6583 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6584 break;
6585 case 3:
6586 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6587 break;
6588 default:
6589 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6590 break;
6591 }
6592 } else if (ring->me == 2) {
6593 switch (ring->pipe) {
6594 case 0:
6595 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6596 break;
6597 case 1:
6598 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6599 break;
6600 case 2:
6601 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6602 break;
6603 case 3:
6604 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6605 break;
6606 default:
6607 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6608 break;
6609 }
6610 } else {
6611 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6612 }
6613 }
6614 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6615 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6616 DRM_DEBUG("si_irq_set: sw int cp2\n");
6617 if (ring->me == 1) {
6618 switch (ring->pipe) {
6619 case 0:
6620 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6621 break;
6622 case 1:
6623 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6624 break;
6625 case 2:
6626 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6627 break;
6628 case 3:
6629 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6630 break;
6631 default:
6632 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6633 break;
6634 }
6635 } else if (ring->me == 2) {
6636 switch (ring->pipe) {
6637 case 0:
6638 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6639 break;
6640 case 1:
6641 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6642 break;
6643 case 2:
6644 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6645 break;
6646 case 3:
6647 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6648 break;
6649 default:
6650 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6651 break;
6652 }
6653 } else {
6654 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6655 }
6656 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6657
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6658 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6659 DRM_DEBUG("cik_irq_set: sw int dma\n");
6660 dma_cntl |= TRAP_ENABLE;
6661 }
6662
6663 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6664 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6665 dma_cntl1 |= TRAP_ENABLE;
6666 }
6667
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6668 if (rdev->irq.crtc_vblank_int[0] ||
6669 atomic_read(&rdev->irq.pflip[0])) {
6670 DRM_DEBUG("cik_irq_set: vblank 0\n");
6671 crtc1 |= VBLANK_INTERRUPT_MASK;
6672 }
6673 if (rdev->irq.crtc_vblank_int[1] ||
6674 atomic_read(&rdev->irq.pflip[1])) {
6675 DRM_DEBUG("cik_irq_set: vblank 1\n");
6676 crtc2 |= VBLANK_INTERRUPT_MASK;
6677 }
6678 if (rdev->irq.crtc_vblank_int[2] ||
6679 atomic_read(&rdev->irq.pflip[2])) {
6680 DRM_DEBUG("cik_irq_set: vblank 2\n");
6681 crtc3 |= VBLANK_INTERRUPT_MASK;
6682 }
6683 if (rdev->irq.crtc_vblank_int[3] ||
6684 atomic_read(&rdev->irq.pflip[3])) {
6685 DRM_DEBUG("cik_irq_set: vblank 3\n");
6686 crtc4 |= VBLANK_INTERRUPT_MASK;
6687 }
6688 if (rdev->irq.crtc_vblank_int[4] ||
6689 atomic_read(&rdev->irq.pflip[4])) {
6690 DRM_DEBUG("cik_irq_set: vblank 4\n");
6691 crtc5 |= VBLANK_INTERRUPT_MASK;
6692 }
6693 if (rdev->irq.crtc_vblank_int[5] ||
6694 atomic_read(&rdev->irq.pflip[5])) {
6695 DRM_DEBUG("cik_irq_set: vblank 5\n");
6696 crtc6 |= VBLANK_INTERRUPT_MASK;
6697 }
6698 if (rdev->irq.hpd[0]) {
6699 DRM_DEBUG("cik_irq_set: hpd 1\n");
6700 hpd1 |= DC_HPDx_INT_EN;
6701 }
6702 if (rdev->irq.hpd[1]) {
6703 DRM_DEBUG("cik_irq_set: hpd 2\n");
6704 hpd2 |= DC_HPDx_INT_EN;
6705 }
6706 if (rdev->irq.hpd[2]) {
6707 DRM_DEBUG("cik_irq_set: hpd 3\n");
6708 hpd3 |= DC_HPDx_INT_EN;
6709 }
6710 if (rdev->irq.hpd[3]) {
6711 DRM_DEBUG("cik_irq_set: hpd 4\n");
6712 hpd4 |= DC_HPDx_INT_EN;
6713 }
6714 if (rdev->irq.hpd[4]) {
6715 DRM_DEBUG("cik_irq_set: hpd 5\n");
6716 hpd5 |= DC_HPDx_INT_EN;
6717 }
6718 if (rdev->irq.hpd[5]) {
6719 DRM_DEBUG("cik_irq_set: hpd 6\n");
6720 hpd6 |= DC_HPDx_INT_EN;
6721 }
6722
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6723 if (rdev->irq.dpm_thermal) {
6724 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6725 if (rdev->flags & RADEON_IS_IGP)
6726 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6727 else
6728 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6729 }
6730
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6731 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6732
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6733 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6734 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6735
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6736 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6737 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6738 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6739 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6740 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6741 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6742 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6743 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6744
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6745 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6746
6747 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6748 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6749 if (rdev->num_crtc >= 4) {
6750 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6751 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6752 }
6753 if (rdev->num_crtc >= 6) {
6754 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6755 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6756 }
6757
6758 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6759 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6760 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6761 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6762 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6763 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6764
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6765 if (rdev->flags & RADEON_IS_IGP)
6766 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6767 else
6768 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6769
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6770 return 0;
6771}
6772
6773/**
6774 * cik_irq_ack - ack interrupt sources
6775 *
6776 * @rdev: radeon_device pointer
6777 *
6778 * Ack interrupt sources on the GPU (vblanks, hpd,
6779 * etc.) (CIK). Certain interrupts sources are sw
6780 * generated and do not require an explicit ack.
6781 */
6782static inline void cik_irq_ack(struct radeon_device *rdev)
6783{
6784 u32 tmp;
6785
6786 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6787 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6788 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6789 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6790 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6791 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6792 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6793
6794 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6795 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6796 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6797 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6798 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6799 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6800 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6801 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6802
6803 if (rdev->num_crtc >= 4) {
6804 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6805 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6806 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6807 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6808 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6809 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6810 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6811 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6812 }
6813
6814 if (rdev->num_crtc >= 6) {
6815 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6816 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6817 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6818 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6819 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6820 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6821 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6822 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6823 }
6824
6825 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6826 tmp = RREG32(DC_HPD1_INT_CONTROL);
6827 tmp |= DC_HPDx_INT_ACK;
6828 WREG32(DC_HPD1_INT_CONTROL, tmp);
6829 }
6830 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6831 tmp = RREG32(DC_HPD2_INT_CONTROL);
6832 tmp |= DC_HPDx_INT_ACK;
6833 WREG32(DC_HPD2_INT_CONTROL, tmp);
6834 }
6835 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6836 tmp = RREG32(DC_HPD3_INT_CONTROL);
6837 tmp |= DC_HPDx_INT_ACK;
6838 WREG32(DC_HPD3_INT_CONTROL, tmp);
6839 }
6840 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6841 tmp = RREG32(DC_HPD4_INT_CONTROL);
6842 tmp |= DC_HPDx_INT_ACK;
6843 WREG32(DC_HPD4_INT_CONTROL, tmp);
6844 }
6845 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6846 tmp = RREG32(DC_HPD5_INT_CONTROL);
6847 tmp |= DC_HPDx_INT_ACK;
6848 WREG32(DC_HPD5_INT_CONTROL, tmp);
6849 }
6850 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6851 tmp = RREG32(DC_HPD5_INT_CONTROL);
6852 tmp |= DC_HPDx_INT_ACK;
6853 WREG32(DC_HPD6_INT_CONTROL, tmp);
6854 }
6855}
6856
6857/**
6858 * cik_irq_disable - disable interrupts
6859 *
6860 * @rdev: radeon_device pointer
6861 *
6862 * Disable interrupts on the hw (CIK).
6863 */
6864static void cik_irq_disable(struct radeon_device *rdev)
6865{
6866 cik_disable_interrupts(rdev);
6867 /* Wait and acknowledge irq */
6868 mdelay(1);
6869 cik_irq_ack(rdev);
6870 cik_disable_interrupt_state(rdev);
6871}
6872
6873/**
6874 * cik_irq_disable - disable interrupts for suspend
6875 *
6876 * @rdev: radeon_device pointer
6877 *
6878 * Disable interrupts and stop the RLC (CIK).
6879 * Used for suspend.
6880 */
6881static void cik_irq_suspend(struct radeon_device *rdev)
6882{
6883 cik_irq_disable(rdev);
6884 cik_rlc_stop(rdev);
6885}
6886
6887/**
6888 * cik_irq_fini - tear down interrupt support
6889 *
6890 * @rdev: radeon_device pointer
6891 *
6892 * Disable interrupts on the hw and free the IH ring
6893 * buffer (CIK).
6894 * Used for driver unload.
6895 */
6896static void cik_irq_fini(struct radeon_device *rdev)
6897{
6898 cik_irq_suspend(rdev);
6899 r600_ih_ring_fini(rdev);
6900}
6901
6902/**
6903 * cik_get_ih_wptr - get the IH ring buffer wptr
6904 *
6905 * @rdev: radeon_device pointer
6906 *
6907 * Get the IH ring buffer wptr from either the register
6908 * or the writeback memory buffer (CIK). Also check for
6909 * ring buffer overflow and deal with it.
6910 * Used by cik_irq_process().
6911 * Returns the value of the wptr.
6912 */
6913static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6914{
6915 u32 wptr, tmp;
6916
6917 if (rdev->wb.enabled)
6918 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6919 else
6920 wptr = RREG32(IH_RB_WPTR);
6921
6922 if (wptr & RB_OVERFLOW) {
6923 /* When a ring buffer overflow happen start parsing interrupt
6924 * from the last not overwritten vector (wptr + 16). Hopefully
6925 * this should allow us to catchup.
6926 */
6927 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6928 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6929 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6930 tmp = RREG32(IH_RB_CNTL);
6931 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6932 WREG32(IH_RB_CNTL, tmp);
6933 }
6934 return (wptr & rdev->ih.ptr_mask);
6935}
6936
6937/* CIK IV Ring
6938 * Each IV ring entry is 128 bits:
6939 * [7:0] - interrupt source id
6940 * [31:8] - reserved
6941 * [59:32] - interrupt source data
6942 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6943 * [71:64] - RINGID
6944 * CP:
6945 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6946 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6947 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6948 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6949 * PIPE_ID - ME0 0=3D
6950 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6951 * SDMA:
6952 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6953 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6954 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6955 * [79:72] - VMID
6956 * [95:80] - PASID
6957 * [127:96] - reserved
6958 */
6959/**
6960 * cik_irq_process - interrupt handler
6961 *
6962 * @rdev: radeon_device pointer
6963 *
6964 * Interrupt hander (CIK). Walk the IH ring,
6965 * ack interrupts and schedule work to handle
6966 * interrupt events.
6967 * Returns irq process return code.
6968 */
6969int cik_irq_process(struct radeon_device *rdev)
6970{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6971 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6972 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6973 u32 wptr;
6974 u32 rptr;
6975 u32 src_id, src_data, ring_id;
6976 u8 me_id, pipe_id, queue_id;
6977 u32 ring_index;
6978 bool queue_hotplug = false;
6979 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]6980 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6981 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6982
6983 if (!rdev->ih.enabled || rdev->shutdown)
6984 return IRQ_NONE;
6985
6986 wptr = cik_get_ih_wptr(rdev);
6987
6988restart_ih:
6989 /* is somebody else already processing irqs? */
6990 if (atomic_xchg(&rdev->ih.lock, 1))
6991 return IRQ_NONE;
6992
6993 rptr = rdev->ih.rptr;
6994 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6995
6996 /* Order reading of wptr vs. reading of IH ring data */
6997 rmb();
6998
6999 /* display interrupts */
7000 cik_irq_ack(rdev);
7001
7002 while (rptr != wptr) {
7003 /* wptr/rptr are in bytes! */
7004 ring_index = rptr / 4;
7005 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7006 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7007 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7008
7009 switch (src_id) {
7010 case 1: /* D1 vblank/vline */
7011 switch (src_data) {
7012 case 0: /* D1 vblank */
7013 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7014 if (rdev->irq.crtc_vblank_int[0]) {
7015 drm_handle_vblank(rdev->ddev, 0);
7016 rdev->pm.vblank_sync = true;
7017 wake_up(&rdev->irq.vblank_queue);
7018 }
7019 if (atomic_read(&rdev->irq.pflip[0]))
7020 radeon_crtc_handle_flip(rdev, 0);
7021 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7022 DRM_DEBUG("IH: D1 vblank\n");
7023 }
7024 break;
7025 case 1: /* D1 vline */
7026 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7027 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7028 DRM_DEBUG("IH: D1 vline\n");
7029 }
7030 break;
7031 default:
7032 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7033 break;
7034 }
7035 break;
7036 case 2: /* D2 vblank/vline */
7037 switch (src_data) {
7038 case 0: /* D2 vblank */
7039 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7040 if (rdev->irq.crtc_vblank_int[1]) {
7041 drm_handle_vblank(rdev->ddev, 1);
7042 rdev->pm.vblank_sync = true;
7043 wake_up(&rdev->irq.vblank_queue);
7044 }
7045 if (atomic_read(&rdev->irq.pflip[1]))
7046 radeon_crtc_handle_flip(rdev, 1);
7047 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7048 DRM_DEBUG("IH: D2 vblank\n");
7049 }
7050 break;
7051 case 1: /* D2 vline */
7052 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7053 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7054 DRM_DEBUG("IH: D2 vline\n");
7055 }
7056 break;
7057 default:
7058 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7059 break;
7060 }
7061 break;
7062 case 3: /* D3 vblank/vline */
7063 switch (src_data) {
7064 case 0: /* D3 vblank */
7065 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7066 if (rdev->irq.crtc_vblank_int[2]) {
7067 drm_handle_vblank(rdev->ddev, 2);
7068 rdev->pm.vblank_sync = true;
7069 wake_up(&rdev->irq.vblank_queue);
7070 }
7071 if (atomic_read(&rdev->irq.pflip[2]))
7072 radeon_crtc_handle_flip(rdev, 2);
7073 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7074 DRM_DEBUG("IH: D3 vblank\n");
7075 }
7076 break;
7077 case 1: /* D3 vline */
7078 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7079 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7080 DRM_DEBUG("IH: D3 vline\n");
7081 }
7082 break;
7083 default:
7084 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7085 break;
7086 }
7087 break;
7088 case 4: /* D4 vblank/vline */
7089 switch (src_data) {
7090 case 0: /* D4 vblank */
7091 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7092 if (rdev->irq.crtc_vblank_int[3]) {
7093 drm_handle_vblank(rdev->ddev, 3);
7094 rdev->pm.vblank_sync = true;
7095 wake_up(&rdev->irq.vblank_queue);
7096 }
7097 if (atomic_read(&rdev->irq.pflip[3]))
7098 radeon_crtc_handle_flip(rdev, 3);
7099 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7100 DRM_DEBUG("IH: D4 vblank\n");
7101 }
7102 break;
7103 case 1: /* D4 vline */
7104 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7105 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7106 DRM_DEBUG("IH: D4 vline\n");
7107 }
7108 break;
7109 default:
7110 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7111 break;
7112 }
7113 break;
7114 case 5: /* D5 vblank/vline */
7115 switch (src_data) {
7116 case 0: /* D5 vblank */
7117 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7118 if (rdev->irq.crtc_vblank_int[4]) {
7119 drm_handle_vblank(rdev->ddev, 4);
7120 rdev->pm.vblank_sync = true;
7121 wake_up(&rdev->irq.vblank_queue);
7122 }
7123 if (atomic_read(&rdev->irq.pflip[4]))
7124 radeon_crtc_handle_flip(rdev, 4);
7125 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7126 DRM_DEBUG("IH: D5 vblank\n");
7127 }
7128 break;
7129 case 1: /* D5 vline */
7130 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7131 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7132 DRM_DEBUG("IH: D5 vline\n");
7133 }
7134 break;
7135 default:
7136 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7137 break;
7138 }
7139 break;
7140 case 6: /* D6 vblank/vline */
7141 switch (src_data) {
7142 case 0: /* D6 vblank */
7143 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7144 if (rdev->irq.crtc_vblank_int[5]) {
7145 drm_handle_vblank(rdev->ddev, 5);
7146 rdev->pm.vblank_sync = true;
7147 wake_up(&rdev->irq.vblank_queue);
7148 }
7149 if (atomic_read(&rdev->irq.pflip[5]))
7150 radeon_crtc_handle_flip(rdev, 5);
7151 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7152 DRM_DEBUG("IH: D6 vblank\n");
7153 }
7154 break;
7155 case 1: /* D6 vline */
7156 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7157 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7158 DRM_DEBUG("IH: D6 vline\n");
7159 }
7160 break;
7161 default:
7162 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7163 break;
7164 }
7165 break;
7166 case 42: /* HPD hotplug */
7167 switch (src_data) {
7168 case 0:
7169 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7170 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7171 queue_hotplug = true;
7172 DRM_DEBUG("IH: HPD1\n");
7173 }
7174 break;
7175 case 1:
7176 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7177 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7178 queue_hotplug = true;
7179 DRM_DEBUG("IH: HPD2\n");
7180 }
7181 break;
7182 case 2:
7183 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7184 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7185 queue_hotplug = true;
7186 DRM_DEBUG("IH: HPD3\n");
7187 }
7188 break;
7189 case 3:
7190 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7191 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7192 queue_hotplug = true;
7193 DRM_DEBUG("IH: HPD4\n");
7194 }
7195 break;
7196 case 4:
7197 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7198 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7199 queue_hotplug = true;
7200 DRM_DEBUG("IH: HPD5\n");
7201 }
7202 break;
7203 case 5:
7204 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7205 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7206 queue_hotplug = true;
7207 DRM_DEBUG("IH: HPD6\n");
7208 }
7209 break;
7210 default:
7211 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7212 break;
7213 }
7214 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7215 case 124: /* UVD */
7216 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7217 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7218 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7219 case 146:
7220 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7221 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7222 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7223 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7224 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7225 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7226 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7227 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7228 status);
7229 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7230 /* reset addr and status */
7231 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7232 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7233 case 176: /* GFX RB CP_INT */
7234 case 177: /* GFX IB CP_INT */
7235 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7236 break;
7237 case 181: /* CP EOP event */
7238 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7239 /* XXX check the bitfield order! */
7240 me_id = (ring_id & 0x60) >> 5;
7241 pipe_id = (ring_id & 0x18) >> 3;
7242 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7243 switch (me_id) {
7244 case 0:
7245 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7246 break;
7247 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7248 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7249 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7250 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7251 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7252 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7253 break;
7254 }
7255 break;
7256 case 184: /* CP Privileged reg access */
7257 DRM_ERROR("Illegal register access in command stream\n");
7258 /* XXX check the bitfield order! */
7259 me_id = (ring_id & 0x60) >> 5;
7260 pipe_id = (ring_id & 0x18) >> 3;
7261 queue_id = (ring_id & 0x7) >> 0;
7262 switch (me_id) {
7263 case 0:
7264 /* This results in a full GPU reset, but all we need to do is soft
7265 * reset the CP for gfx
7266 */
7267 queue_reset = true;
7268 break;
7269 case 1:
7270 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7271 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7272 break;
7273 case 2:
7274 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7275 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7276 break;
7277 }
7278 break;
7279 case 185: /* CP Privileged inst */
7280 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7281 /* XXX check the bitfield order! */
7282 me_id = (ring_id & 0x60) >> 5;
7283 pipe_id = (ring_id & 0x18) >> 3;
7284 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7285 switch (me_id) {
7286 case 0:
7287 /* This results in a full GPU reset, but all we need to do is soft
7288 * reset the CP for gfx
7289 */
7290 queue_reset = true;
7291 break;
7292 case 1:
7293 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7294 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7295 break;
7296 case 2:
7297 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7298 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7299 break;
7300 }
7301 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7302 case 224: /* SDMA trap event */
7303 /* XXX check the bitfield order! */
7304 me_id = (ring_id & 0x3) >> 0;
7305 queue_id = (ring_id & 0xc) >> 2;
7306 DRM_DEBUG("IH: SDMA trap\n");
7307 switch (me_id) {
7308 case 0:
7309 switch (queue_id) {
7310 case 0:
7311 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7312 break;
7313 case 1:
7314 /* XXX compute */
7315 break;
7316 case 2:
7317 /* XXX compute */
7318 break;
7319 }
7320 break;
7321 case 1:
7322 switch (queue_id) {
7323 case 0:
7324 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7325 break;
7326 case 1:
7327 /* XXX compute */
7328 break;
7329 case 2:
7330 /* XXX compute */
7331 break;
7332 }
7333 break;
7334 }
7335 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7336 case 230: /* thermal low to high */
7337 DRM_DEBUG("IH: thermal low to high\n");
7338 rdev->pm.dpm.thermal.high_to_low = false;
7339 queue_thermal = true;
7340 break;
7341 case 231: /* thermal high to low */
7342 DRM_DEBUG("IH: thermal high to low\n");
7343 rdev->pm.dpm.thermal.high_to_low = true;
7344 queue_thermal = true;
7345 break;
7346 case 233: /* GUI IDLE */
7347 DRM_DEBUG("IH: GUI idle\n");
7348 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7349 case 241: /* SDMA Privileged inst */
7350 case 247: /* SDMA Privileged inst */
7351 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7352 /* XXX check the bitfield order! */
7353 me_id = (ring_id & 0x3) >> 0;
7354 queue_id = (ring_id & 0xc) >> 2;
7355 switch (me_id) {
7356 case 0:
7357 switch (queue_id) {
7358 case 0:
7359 queue_reset = true;
7360 break;
7361 case 1:
7362 /* XXX compute */
7363 queue_reset = true;
7364 break;
7365 case 2:
7366 /* XXX compute */
7367 queue_reset = true;
7368 break;
7369 }
7370 break;
7371 case 1:
7372 switch (queue_id) {
7373 case 0:
7374 queue_reset = true;
7375 break;
7376 case 1:
7377 /* XXX compute */
7378 queue_reset = true;
7379 break;
7380 case 2:
7381 /* XXX compute */
7382 queue_reset = true;
7383 break;
7384 }
7385 break;
7386 }
7387 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7388 default:
7389 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7390 break;
7391 }
7392
7393 /* wptr/rptr are in bytes! */
7394 rptr += 16;
7395 rptr &= rdev->ih.ptr_mask;
7396 }
7397 if (queue_hotplug)
7398 schedule_work(&rdev->hotplug_work);
7399 if (queue_reset)
7400 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7401 if (queue_thermal)
7402 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7403 rdev->ih.rptr = rptr;
7404 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7405 atomic_set(&rdev->ih.lock, 0);
7406
7407 /* make sure wptr hasn't changed while processing */
7408 wptr = cik_get_ih_wptr(rdev);
7409 if (wptr != rptr)
7410 goto restart_ih;
7411
7412 return IRQ_HANDLED;
7413}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7414
7415/*
7416 * startup/shutdown callbacks
7417 */
7418/**
7419 * cik_startup - program the asic to a functional state
7420 *
7421 * @rdev: radeon_device pointer
7422 *
7423 * Programs the asic to a functional state (CIK).
7424 * Called by cik_init() and cik_resume().
7425 * Returns 0 for success, error for failure.
7426 */
7427static int cik_startup(struct radeon_device *rdev)
7428{
7429 struct radeon_ring *ring;
7430 int r;
7431
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7432 /* enable pcie gen2/3 link */
7433 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7434 /* enable aspm */
7435 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7436
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7437 /* scratch needs to be initialized before MC */
7438 r = r600_vram_scratch_init(rdev);
7439 if (r)
7440 return r;
7441
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7442 cik_mc_program(rdev);
7443
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7444 if (rdev->flags & RADEON_IS_IGP) {
7445 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7446 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7447 r = cik_init_microcode(rdev);
7448 if (r) {
7449 DRM_ERROR("Failed to load firmware!\n");
7450 return r;
7451 }
7452 }
7453 } else {
7454 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7455 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7456 !rdev->mc_fw) {
7457 r = cik_init_microcode(rdev);
7458 if (r) {
7459 DRM_ERROR("Failed to load firmware!\n");
7460 return r;
7461 }
7462 }
7463
7464 r = ci_mc_load_microcode(rdev);
7465 if (r) {
7466 DRM_ERROR("Failed to load MC firmware!\n");
7467 return r;
7468 }
7469 }
7470
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7471 r = cik_pcie_gart_enable(rdev);
7472 if (r)
7473 return r;
7474 cik_gpu_init(rdev);
7475
7476 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7477 if (rdev->flags & RADEON_IS_IGP) {
7478 if (rdev->family == CHIP_KAVERI) {
7479 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7480 rdev->rlc.reg_list_size =
7481 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7482 } else {
7483 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7484 rdev->rlc.reg_list_size =
7485 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7486 }
7487 }
7488 rdev->rlc.cs_data = ci_cs_data;
7489 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7490 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7491 if (r) {
7492 DRM_ERROR("Failed to init rlc BOs!\n");
7493 return r;
7494 }
7495
7496 /* allocate wb buffer */
7497 r = radeon_wb_init(rdev);
7498 if (r)
7499 return r;
7500
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7501 /* allocate mec buffers */
7502 r = cik_mec_init(rdev);
7503 if (r) {
7504 DRM_ERROR("Failed to init MEC BOs!\n");
7505 return r;
7506 }
7507
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7508 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7509 if (r) {
7510 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7511 return r;
7512 }
7513
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7514 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7515 if (r) {
7516 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7517 return r;
7518 }
7519
7520 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7521 if (r) {
7522 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7523 return r;
7524 }
7525
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7526 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7527 if (r) {
7528 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7529 return r;
7530 }
7531
7532 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7533 if (r) {
7534 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7535 return r;
7536 }
7537
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7538 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7539 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7540 r = uvd_v4_2_resume(rdev);
7541 if (!r) {
7542 r = radeon_fence_driver_start_ring(rdev,
7543 R600_RING_TYPE_UVD_INDEX);
7544 if (r)
7545 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7546 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7547 }
7548 if (r)
7549 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7550
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7551 /* Enable IRQ */
7552 if (!rdev->irq.installed) {
7553 r = radeon_irq_kms_init(rdev);
7554 if (r)
7555 return r;
7556 }
7557
7558 r = cik_irq_init(rdev);
7559 if (r) {
7560 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7561 radeon_irq_kms_fini(rdev);
7562 return r;
7563 }
7564 cik_irq_set(rdev);
7565
7566 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7567 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7568 CP_RB0_RPTR, CP_RB0_WPTR,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7569 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7570 if (r)
7571 return r;
7572
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7573 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7574 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7575 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7576 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7577 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7578 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7579 if (r)
7580 return r;
7581 ring->me = 1; /* first MEC */
7582 ring->pipe = 0; /* first pipe */
7583 ring->queue = 0; /* first queue */
7584 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7585
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7586 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7587 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7588 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7589 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7590 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7591 if (r)
7592 return r;
7593 /* dGPU only have 1 MEC */
7594 ring->me = 1; /* first MEC */
7595 ring->pipe = 0; /* first pipe */
7596 ring->queue = 1; /* second queue */
7597 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7598
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7599 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7600 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7601 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7602 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7603 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7604 if (r)
7605 return r;
7606
7607 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7608 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7609 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7610 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7611 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7612 if (r)
7613 return r;
7614
7615 r = cik_cp_resume(rdev);
7616 if (r)
7617 return r;
7618
7619 r = cik_sdma_resume(rdev);
7620 if (r)
7621 return r;
7622
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7623 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7624 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7625 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7626 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7627 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7628 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7629 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7630 if (r)
7631 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7632 }
7633
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7634 r = radeon_ib_pool_init(rdev);
7635 if (r) {
7636 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7637 return r;
7638 }
7639
7640 r = radeon_vm_manager_init(rdev);
7641 if (r) {
7642 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7643 return r;
7644 }
7645
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7646 r = dce6_audio_init(rdev);
7647 if (r)
7648 return r;
7649
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7650 return 0;
7651}
7652
7653/**
7654 * cik_resume - resume the asic to a functional state
7655 *
7656 * @rdev: radeon_device pointer
7657 *
7658 * Programs the asic to a functional state (CIK).
7659 * Called at resume.
7660 * Returns 0 for success, error for failure.
7661 */
7662int cik_resume(struct radeon_device *rdev)
7663{
7664 int r;
7665
7666 /* post card */
7667 atom_asic_init(rdev->mode_info.atom_context);
7668
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7669 /* init golden registers */
7670 cik_init_golden_registers(rdev);
7671
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7672 rdev->accel_working = true;
7673 r = cik_startup(rdev);
7674 if (r) {
7675 DRM_ERROR("cik startup failed on resume\n");
7676 rdev->accel_working = false;
7677 return r;
7678 }
7679
7680 return r;
7681
7682}
7683
7684/**
7685 * cik_suspend - suspend the asic
7686 *
7687 * @rdev: radeon_device pointer
7688 *
7689 * Bring the chip into a state suitable for suspend (CIK).
7690 * Called at suspend.
7691 * Returns 0 for success.
7692 */
7693int cik_suspend(struct radeon_device *rdev)
7694{
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7695 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7696 radeon_vm_manager_fini(rdev);
7697 cik_cp_enable(rdev, false);
7698 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7699 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7700 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7701 cik_fini_pg(rdev);
7702 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7703 cik_irq_suspend(rdev);
7704 radeon_wb_disable(rdev);
7705 cik_pcie_gart_disable(rdev);
7706 return 0;
7707}
7708
7709/* Plan is to move initialization in that function and use
7710 * helper function so that radeon_device_init pretty much
7711 * do nothing more than calling asic specific function. This
7712 * should also allow to remove a bunch of callback function
7713 * like vram_info.
7714 */
7715/**
7716 * cik_init - asic specific driver and hw init
7717 *
7718 * @rdev: radeon_device pointer
7719 *
7720 * Setup asic specific driver variables and program the hw
7721 * to a functional state (CIK).
7722 * Called at driver startup.
7723 * Returns 0 for success, errors for failure.
7724 */
7725int cik_init(struct radeon_device *rdev)
7726{
7727 struct radeon_ring *ring;
7728 int r;
7729
7730 /* Read BIOS */
7731 if (!radeon_get_bios(rdev)) {
7732 if (ASIC_IS_AVIVO(rdev))
7733 return -EINVAL;
7734 }
7735 /* Must be an ATOMBIOS */
7736 if (!rdev->is_atom_bios) {
7737 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7738 return -EINVAL;
7739 }
7740 r = radeon_atombios_init(rdev);
7741 if (r)
7742 return r;
7743
7744 /* Post card if necessary */
7745 if (!radeon_card_posted(rdev)) {
7746 if (!rdev->bios) {
7747 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7748 return -EINVAL;
7749 }
7750 DRM_INFO("GPU not posted. posting now...\n");
7751 atom_asic_init(rdev->mode_info.atom_context);
7752 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7753 /* init golden registers */
7754 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7755 /* Initialize scratch registers */
7756 cik_scratch_init(rdev);
7757 /* Initialize surface registers */
7758 radeon_surface_init(rdev);
7759 /* Initialize clocks */
7760 radeon_get_clock_info(rdev->ddev);
7761
7762 /* Fence driver */
7763 r = radeon_fence_driver_init(rdev);
7764 if (r)
7765 return r;
7766
7767 /* initialize memory controller */
7768 r = cik_mc_init(rdev);
7769 if (r)
7770 return r;
7771 /* Memory manager */
7772 r = radeon_bo_init(rdev);
7773 if (r)
7774 return r;
7775
7776 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7777 ring->ring_obj = NULL;
7778 r600_ring_init(rdev, ring, 1024 * 1024);
7779
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7780 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7781 ring->ring_obj = NULL;
7782 r600_ring_init(rdev, ring, 1024 * 1024);
7783 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7784 if (r)
7785 return r;
7786
7787 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7788 ring->ring_obj = NULL;
7789 r600_ring_init(rdev, ring, 1024 * 1024);
7790 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7791 if (r)
7792 return r;
7793
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7794 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7795 ring->ring_obj = NULL;
7796 r600_ring_init(rdev, ring, 256 * 1024);
7797
7798 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7799 ring->ring_obj = NULL;
7800 r600_ring_init(rdev, ring, 256 * 1024);
7801
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7802 r = radeon_uvd_init(rdev);
7803 if (!r) {
7804 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7805 ring->ring_obj = NULL;
7806 r600_ring_init(rdev, ring, 4096);
7807 }
7808
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7809 rdev->ih.ring_obj = NULL;
7810 r600_ih_ring_init(rdev, 64 * 1024);
7811
7812 r = r600_pcie_gart_init(rdev);
7813 if (r)
7814 return r;
7815
7816 rdev->accel_working = true;
7817 r = cik_startup(rdev);
7818 if (r) {
7819 dev_err(rdev->dev, "disabling GPU acceleration\n");
7820 cik_cp_fini(rdev);
7821 cik_sdma_fini(rdev);
7822 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7823 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7824 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7825 radeon_wb_fini(rdev);
7826 radeon_ib_pool_fini(rdev);
7827 radeon_vm_manager_fini(rdev);
7828 radeon_irq_kms_fini(rdev);
7829 cik_pcie_gart_fini(rdev);
7830 rdev->accel_working = false;
7831 }
7832
7833 /* Don't start up if the MC ucode is missing.
7834 * The default clocks and voltages before the MC ucode
7835 * is loaded are not suffient for advanced operations.
7836 */
7837 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7838 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7839 return -EINVAL;
7840 }
7841
7842 return 0;
7843}
7844
7845/**
7846 * cik_fini - asic specific driver and hw fini
7847 *
7848 * @rdev: radeon_device pointer
7849 *
7850 * Tear down the asic specific driver variables and program the hw
7851 * to an idle state (CIK).
7852 * Called at driver unload.
7853 */
7854void cik_fini(struct radeon_device *rdev)
7855{
7856 cik_cp_fini(rdev);
7857 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7858 cik_fini_pg(rdev);
7859 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7860 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7861 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7862 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7863 radeon_wb_fini(rdev);
7864 radeon_vm_manager_fini(rdev);
7865 radeon_ib_pool_fini(rdev);
7866 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7867 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7868 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7869 cik_pcie_gart_fini(rdev);
7870 r600_vram_scratch_fini(rdev);
7871 radeon_gem_fini(rdev);
7872 radeon_fence_driver_fini(rdev);
7873 radeon_bo_fini(rdev);
7874 radeon_atombios_fini(rdev);
7875 kfree(rdev->bios);
7876 rdev->bios = NULL;
7877}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7878
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7879void dce8_program_fmt(struct drm_encoder *encoder)
7880{
7881 struct drm_device *dev = encoder->dev;
7882 struct radeon_device *rdev = dev->dev_private;
7883 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7884 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7885 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7886 int bpc = 0;
7887 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7888 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7889
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7890 if (connector) {
7891 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7892 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7893 dither = radeon_connector->dither;
7894 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7895
7896 /* LVDS/eDP FMT is set up by atom */
7897 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7898 return;
7899
7900 /* not needed for analog */
7901 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7902 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7903 return;
7904
7905 if (bpc == 0)
7906 return;
7907
7908 switch (bpc) {
7909 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7910 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7911 /* XXX sort out optimal dither settings */
7912 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7913 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7914 else
7915 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7916 break;
7917 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7918 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7919 /* XXX sort out optimal dither settings */
7920 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7921 FMT_RGB_RANDOM_ENABLE |
7922 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7923 else
7924 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7925 break;
7926 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7927 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7928 /* XXX sort out optimal dither settings */
7929 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7930 FMT_RGB_RANDOM_ENABLE |
7931 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7932 else
7933 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7934 break;
7935 default:
7936 /* not needed */
7937 break;
7938 }
7939
7940 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7941}
7942
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7943/* display watermark setup */
7944/**
7945 * dce8_line_buffer_adjust - Set up the line buffer
7946 *
7947 * @rdev: radeon_device pointer
7948 * @radeon_crtc: the selected display controller
7949 * @mode: the current display mode on the selected display
7950 * controller
7951 *
7952 * Setup up the line buffer allocation for
7953 * the selected display controller (CIK).
7954 * Returns the line buffer size in pixels.
7955 */
7956static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7957 struct radeon_crtc *radeon_crtc,
7958 struct drm_display_mode *mode)
7959{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7960 u32 tmp, buffer_alloc, i;
7961 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7962 /*
7963 * Line Buffer Setup
7964 * There are 6 line buffers, one for each display controllers.
7965 * There are 3 partitions per LB. Select the number of partitions
7966 * to enable based on the display width. For display widths larger
7967 * than 4096, you need use to use 2 display controllers and combine
7968 * them using the stereo blender.
7969 */
7970 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7971 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7972 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7973 buffer_alloc = 2;
7974 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7975 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7976 buffer_alloc = 2;
7977 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7978 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7979 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7980 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7981 DRM_DEBUG_KMS("Mode too big for LB!\n");
7982 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7983 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7984 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7985 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7986 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7987 buffer_alloc = 0;
7988 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7989
7990 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7991 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7992
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7993 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7994 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7995 for (i = 0; i < rdev->usec_timeout; i++) {
7996 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7997 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7998 break;
7999 udelay(1);
8000 }
8001
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8002 if (radeon_crtc->base.enabled && mode) {
8003 switch (tmp) {
8004 case 0:
8005 default:
8006 return 4096 * 2;
8007 case 1:
8008 return 1920 * 2;
8009 case 2:
8010 return 2560 * 2;
8011 }
8012 }
8013
8014 /* controller not enabled, so no lb used */
8015 return 0;
8016}
8017
8018/**
8019 * cik_get_number_of_dram_channels - get the number of dram channels
8020 *
8021 * @rdev: radeon_device pointer
8022 *
8023 * Look up the number of video ram channels (CIK).
8024 * Used for display watermark bandwidth calculations
8025 * Returns the number of dram channels
8026 */
8027static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8028{
8029 u32 tmp = RREG32(MC_SHARED_CHMAP);
8030
8031 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8032 case 0:
8033 default:
8034 return 1;
8035 case 1:
8036 return 2;
8037 case 2:
8038 return 4;
8039 case 3:
8040 return 8;
8041 case 4:
8042 return 3;
8043 case 5:
8044 return 6;
8045 case 6:
8046 return 10;
8047 case 7:
8048 return 12;
8049 case 8:
8050 return 16;
8051 }
8052}
8053
8054struct dce8_wm_params {
8055 u32 dram_channels; /* number of dram channels */
8056 u32 yclk; /* bandwidth per dram data pin in kHz */
8057 u32 sclk; /* engine clock in kHz */
8058 u32 disp_clk; /* display clock in kHz */
8059 u32 src_width; /* viewport width */
8060 u32 active_time; /* active display time in ns */
8061 u32 blank_time; /* blank time in ns */
8062 bool interlaced; /* mode is interlaced */
8063 fixed20_12 vsc; /* vertical scale ratio */
8064 u32 num_heads; /* number of active crtcs */
8065 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8066 u32 lb_size; /* line buffer allocated to pipe */
8067 u32 vtaps; /* vertical scaler taps */
8068};
8069
8070/**
8071 * dce8_dram_bandwidth - get the dram bandwidth
8072 *
8073 * @wm: watermark calculation data
8074 *
8075 * Calculate the raw dram bandwidth (CIK).
8076 * Used for display watermark bandwidth calculations
8077 * Returns the dram bandwidth in MBytes/s
8078 */
8079static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8080{
8081 /* Calculate raw DRAM Bandwidth */
8082 fixed20_12 dram_efficiency; /* 0.7 */
8083 fixed20_12 yclk, dram_channels, bandwidth;
8084 fixed20_12 a;
8085
8086 a.full = dfixed_const(1000);
8087 yclk.full = dfixed_const(wm->yclk);
8088 yclk.full = dfixed_div(yclk, a);
8089 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8090 a.full = dfixed_const(10);
8091 dram_efficiency.full = dfixed_const(7);
8092 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8093 bandwidth.full = dfixed_mul(dram_channels, yclk);
8094 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8095
8096 return dfixed_trunc(bandwidth);
8097}
8098
8099/**
8100 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8101 *
8102 * @wm: watermark calculation data
8103 *
8104 * Calculate the dram bandwidth used for display (CIK).
8105 * Used for display watermark bandwidth calculations
8106 * Returns the dram bandwidth for display in MBytes/s
8107 */
8108static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8109{
8110 /* Calculate DRAM Bandwidth and the part allocated to display. */
8111 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8112 fixed20_12 yclk, dram_channels, bandwidth;
8113 fixed20_12 a;
8114
8115 a.full = dfixed_const(1000);
8116 yclk.full = dfixed_const(wm->yclk);
8117 yclk.full = dfixed_div(yclk, a);
8118 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8119 a.full = dfixed_const(10);
8120 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8121 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8122 bandwidth.full = dfixed_mul(dram_channels, yclk);
8123 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8124
8125 return dfixed_trunc(bandwidth);
8126}
8127
8128/**
8129 * dce8_data_return_bandwidth - get the data return bandwidth
8130 *
8131 * @wm: watermark calculation data
8132 *
8133 * Calculate the data return bandwidth used for display (CIK).
8134 * Used for display watermark bandwidth calculations
8135 * Returns the data return bandwidth in MBytes/s
8136 */
8137static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8138{
8139 /* Calculate the display Data return Bandwidth */
8140 fixed20_12 return_efficiency; /* 0.8 */
8141 fixed20_12 sclk, bandwidth;
8142 fixed20_12 a;
8143
8144 a.full = dfixed_const(1000);
8145 sclk.full = dfixed_const(wm->sclk);
8146 sclk.full = dfixed_div(sclk, a);
8147 a.full = dfixed_const(10);
8148 return_efficiency.full = dfixed_const(8);
8149 return_efficiency.full = dfixed_div(return_efficiency, a);
8150 a.full = dfixed_const(32);
8151 bandwidth.full = dfixed_mul(a, sclk);
8152 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8153
8154 return dfixed_trunc(bandwidth);
8155}
8156
8157/**
8158 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8159 *
8160 * @wm: watermark calculation data
8161 *
8162 * Calculate the dmif bandwidth used for display (CIK).
8163 * Used for display watermark bandwidth calculations
8164 * Returns the dmif bandwidth in MBytes/s
8165 */
8166static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8167{
8168 /* Calculate the DMIF Request Bandwidth */
8169 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8170 fixed20_12 disp_clk, bandwidth;
8171 fixed20_12 a, b;
8172
8173 a.full = dfixed_const(1000);
8174 disp_clk.full = dfixed_const(wm->disp_clk);
8175 disp_clk.full = dfixed_div(disp_clk, a);
8176 a.full = dfixed_const(32);
8177 b.full = dfixed_mul(a, disp_clk);
8178
8179 a.full = dfixed_const(10);
8180 disp_clk_request_efficiency.full = dfixed_const(8);
8181 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8182
8183 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8184
8185 return dfixed_trunc(bandwidth);
8186}
8187
8188/**
8189 * dce8_available_bandwidth - get the min available bandwidth
8190 *
8191 * @wm: watermark calculation data
8192 *
8193 * Calculate the min available bandwidth used for display (CIK).
8194 * Used for display watermark bandwidth calculations
8195 * Returns the min available bandwidth in MBytes/s
8196 */
8197static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8198{
8199 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8200 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8201 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8202 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8203
8204 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8205}
8206
8207/**
8208 * dce8_average_bandwidth - get the average available bandwidth
8209 *
8210 * @wm: watermark calculation data
8211 *
8212 * Calculate the average available bandwidth used for display (CIK).
8213 * Used for display watermark bandwidth calculations
8214 * Returns the average available bandwidth in MBytes/s
8215 */
8216static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8217{
8218 /* Calculate the display mode Average Bandwidth
8219 * DisplayMode should contain the source and destination dimensions,
8220 * timing, etc.
8221 */
8222 fixed20_12 bpp;
8223 fixed20_12 line_time;
8224 fixed20_12 src_width;
8225 fixed20_12 bandwidth;
8226 fixed20_12 a;
8227
8228 a.full = dfixed_const(1000);
8229 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8230 line_time.full = dfixed_div(line_time, a);
8231 bpp.full = dfixed_const(wm->bytes_per_pixel);
8232 src_width.full = dfixed_const(wm->src_width);
8233 bandwidth.full = dfixed_mul(src_width, bpp);
8234 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8235 bandwidth.full = dfixed_div(bandwidth, line_time);
8236
8237 return dfixed_trunc(bandwidth);
8238}
8239
8240/**
8241 * dce8_latency_watermark - get the latency watermark
8242 *
8243 * @wm: watermark calculation data
8244 *
8245 * Calculate the latency watermark (CIK).
8246 * Used for display watermark bandwidth calculations
8247 * Returns the latency watermark in ns
8248 */
8249static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8250{
8251 /* First calculate the latency in ns */
8252 u32 mc_latency = 2000; /* 2000 ns. */
8253 u32 available_bandwidth = dce8_available_bandwidth(wm);
8254 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8255 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8256 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8257 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8258 (wm->num_heads * cursor_line_pair_return_time);
8259 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8260 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8261 u32 tmp, dmif_size = 12288;
8262 fixed20_12 a, b, c;
8263
8264 if (wm->num_heads == 0)
8265 return 0;
8266
8267 a.full = dfixed_const(2);
8268 b.full = dfixed_const(1);
8269 if ((wm->vsc.full > a.full) ||
8270 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8271 (wm->vtaps >= 5) ||
8272 ((wm->vsc.full >= a.full) && wm->interlaced))
8273 max_src_lines_per_dst_line = 4;
8274 else
8275 max_src_lines_per_dst_line = 2;
8276
8277 a.full = dfixed_const(available_bandwidth);
8278 b.full = dfixed_const(wm->num_heads);
8279 a.full = dfixed_div(a, b);
8280
8281 b.full = dfixed_const(mc_latency + 512);
8282 c.full = dfixed_const(wm->disp_clk);
8283 b.full = dfixed_div(b, c);
8284
8285 c.full = dfixed_const(dmif_size);
8286 b.full = dfixed_div(c, b);
8287
8288 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8289
8290 b.full = dfixed_const(1000);
8291 c.full = dfixed_const(wm->disp_clk);
8292 b.full = dfixed_div(c, b);
8293 c.full = dfixed_const(wm->bytes_per_pixel);
8294 b.full = dfixed_mul(b, c);
8295
8296 lb_fill_bw = min(tmp, dfixed_trunc(b));
8297
8298 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8299 b.full = dfixed_const(1000);
8300 c.full = dfixed_const(lb_fill_bw);
8301 b.full = dfixed_div(c, b);
8302 a.full = dfixed_div(a, b);
8303 line_fill_time = dfixed_trunc(a);
8304
8305 if (line_fill_time < wm->active_time)
8306 return latency;
8307 else
8308 return latency + (line_fill_time - wm->active_time);
8309
8310}
8311
8312/**
8313 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8314 * average and available dram bandwidth
8315 *
8316 * @wm: watermark calculation data
8317 *
8318 * Check if the display average bandwidth fits in the display
8319 * dram bandwidth (CIK).
8320 * Used for display watermark bandwidth calculations
8321 * Returns true if the display fits, false if not.
8322 */
8323static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8324{
8325 if (dce8_average_bandwidth(wm) <=
8326 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8327 return true;
8328 else
8329 return false;
8330}
8331
8332/**
8333 * dce8_average_bandwidth_vs_available_bandwidth - check
8334 * average and available bandwidth
8335 *
8336 * @wm: watermark calculation data
8337 *
8338 * Check if the display average bandwidth fits in the display
8339 * available bandwidth (CIK).
8340 * Used for display watermark bandwidth calculations
8341 * Returns true if the display fits, false if not.
8342 */
8343static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8344{
8345 if (dce8_average_bandwidth(wm) <=
8346 (dce8_available_bandwidth(wm) / wm->num_heads))
8347 return true;
8348 else
8349 return false;
8350}
8351
8352/**
8353 * dce8_check_latency_hiding - check latency hiding
8354 *
8355 * @wm: watermark calculation data
8356 *
8357 * Check latency hiding (CIK).
8358 * Used for display watermark bandwidth calculations
8359 * Returns true if the display fits, false if not.
8360 */
8361static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8362{
8363 u32 lb_partitions = wm->lb_size / wm->src_width;
8364 u32 line_time = wm->active_time + wm->blank_time;
8365 u32 latency_tolerant_lines;
8366 u32 latency_hiding;
8367 fixed20_12 a;
8368
8369 a.full = dfixed_const(1);
8370 if (wm->vsc.full > a.full)
8371 latency_tolerant_lines = 1;
8372 else {
8373 if (lb_partitions <= (wm->vtaps + 1))
8374 latency_tolerant_lines = 1;
8375 else
8376 latency_tolerant_lines = 2;
8377 }
8378
8379 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8380
8381 if (dce8_latency_watermark(wm) <= latency_hiding)
8382 return true;
8383 else
8384 return false;
8385}
8386
8387/**
8388 * dce8_program_watermarks - program display watermarks
8389 *
8390 * @rdev: radeon_device pointer
8391 * @radeon_crtc: the selected display controller
8392 * @lb_size: line buffer size
8393 * @num_heads: number of display controllers in use
8394 *
8395 * Calculate and program the display watermarks for the
8396 * selected display controller (CIK).
8397 */
8398static void dce8_program_watermarks(struct radeon_device *rdev,
8399 struct radeon_crtc *radeon_crtc,
8400 u32 lb_size, u32 num_heads)
8401{
8402 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8403 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8404 u32 pixel_period;
8405 u32 line_time = 0;
8406 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8407 u32 tmp, wm_mask;
8408
8409 if (radeon_crtc->base.enabled && num_heads && mode) {
8410 pixel_period = 1000000 / (u32)mode->clock;
8411 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8412
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8413 /* watermark for high clocks */
8414 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8415 rdev->pm.dpm_enabled) {
8416 wm_high.yclk =
8417 radeon_dpm_get_mclk(rdev, false) * 10;
8418 wm_high.sclk =
8419 radeon_dpm_get_sclk(rdev, false) * 10;
8420 } else {
8421 wm_high.yclk = rdev->pm.current_mclk * 10;
8422 wm_high.sclk = rdev->pm.current_sclk * 10;
8423 }
8424
8425 wm_high.disp_clk = mode->clock;
8426 wm_high.src_width = mode->crtc_hdisplay;
8427 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8428 wm_high.blank_time = line_time - wm_high.active_time;
8429 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8430 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8431 wm_high.interlaced = true;
8432 wm_high.vsc = radeon_crtc->vsc;
8433 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8434 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8435 wm_high.vtaps = 2;
8436 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8437 wm_high.lb_size = lb_size;
8438 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8439 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8440
8441 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8442 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8443
8444 /* possibly force display priority to high */
8445 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8446 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8447 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8448 !dce8_check_latency_hiding(&wm_high) ||
8449 (rdev->disp_priority == 2)) {
8450 DRM_DEBUG_KMS("force priority to high\n");
8451 }
8452
8453 /* watermark for low clocks */
8454 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8455 rdev->pm.dpm_enabled) {
8456 wm_low.yclk =
8457 radeon_dpm_get_mclk(rdev, true) * 10;
8458 wm_low.sclk =
8459 radeon_dpm_get_sclk(rdev, true) * 10;
8460 } else {
8461 wm_low.yclk = rdev->pm.current_mclk * 10;
8462 wm_low.sclk = rdev->pm.current_sclk * 10;
8463 }
8464
8465 wm_low.disp_clk = mode->clock;
8466 wm_low.src_width = mode->crtc_hdisplay;
8467 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8468 wm_low.blank_time = line_time - wm_low.active_time;
8469 wm_low.interlaced = false;
8470 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8471 wm_low.interlaced = true;
8472 wm_low.vsc = radeon_crtc->vsc;
8473 wm_low.vtaps = 1;
8474 if (radeon_crtc->rmx_type != RMX_OFF)
8475 wm_low.vtaps = 2;
8476 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8477 wm_low.lb_size = lb_size;
8478 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8479 wm_low.num_heads = num_heads;
8480
8481 /* set for low clocks */
8482 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8483
8484 /* possibly force display priority to high */
8485 /* should really do this at mode validation time... */
8486 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8487 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8488 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8489 (rdev->disp_priority == 2)) {
8490 DRM_DEBUG_KMS("force priority to high\n");
8491 }
8492 }
8493
8494 /* select wm A */
8495 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8496 tmp = wm_mask;
8497 tmp &= ~LATENCY_WATERMARK_MASK(3);
8498 tmp |= LATENCY_WATERMARK_MASK(1);
8499 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8500 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8501 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8502 LATENCY_HIGH_WATERMARK(line_time)));
8503 /* select wm B */
8504 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8505 tmp &= ~LATENCY_WATERMARK_MASK(3);
8506 tmp |= LATENCY_WATERMARK_MASK(2);
8507 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8508 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8509 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8510 LATENCY_HIGH_WATERMARK(line_time)));
8511 /* restore original selection */
8512 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8513
8514 /* save values for DPM */
8515 radeon_crtc->line_time = line_time;
8516 radeon_crtc->wm_high = latency_watermark_a;
8517 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8518}
8519
8520/**
8521 * dce8_bandwidth_update - program display watermarks
8522 *
8523 * @rdev: radeon_device pointer
8524 *
8525 * Calculate and program the display watermarks and line
8526 * buffer allocation (CIK).
8527 */
8528void dce8_bandwidth_update(struct radeon_device *rdev)
8529{
8530 struct drm_display_mode *mode = NULL;
8531 u32 num_heads = 0, lb_size;
8532 int i;
8533
8534 radeon_update_display_priority(rdev);
8535
8536 for (i = 0; i < rdev->num_crtc; i++) {
8537 if (rdev->mode_info.crtcs[i]->base.enabled)
8538 num_heads++;
8539 }
8540 for (i = 0; i < rdev->num_crtc; i++) {
8541 mode = &rdev->mode_info.crtcs[i]->base.mode;
8542 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8543 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8544 }
8545}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8546
8547/**
8548 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8549 *
8550 * @rdev: radeon_device pointer
8551 *
8552 * Fetches a GPU clock counter snapshot (SI).
8553 * Returns the 64 bit clock counter snapshot.
8554 */
8555uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8556{
8557 uint64_t clock;
8558
8559 mutex_lock(&rdev->gpu_clock_mutex);
8560 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8561 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8562 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8563 mutex_unlock(&rdev->gpu_clock_mutex);
8564 return clock;
8565}
8566
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8567static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8568 u32 cntl_reg, u32 status_reg)
8569{
8570 int r, i;
8571 struct atom_clock_dividers dividers;
8572 uint32_t tmp;
8573
8574 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8575 clock, false, &dividers);
8576 if (r)
8577 return r;
8578
8579 tmp = RREG32_SMC(cntl_reg);
8580 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8581 tmp |= dividers.post_divider;
8582 WREG32_SMC(cntl_reg, tmp);
8583
8584 for (i = 0; i < 100; i++) {
8585 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8586 break;
8587 mdelay(10);
8588 }
8589 if (i == 100)
8590 return -ETIMEDOUT;
8591
8592 return 0;
8593}
8594
8595int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8596{
8597 int r = 0;
8598
8599 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8600 if (r)
8601 return r;
8602
8603 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8604 return r;
8605}
8606
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8607static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8608{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8609 struct pci_dev *root = rdev->pdev->bus->self;
8610 int bridge_pos, gpu_pos;
8611 u32 speed_cntl, mask, current_data_rate;
8612 int ret, i;
8613 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8614
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8615 if (radeon_pcie_gen2 == 0)
8616 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8617
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8618 if (rdev->flags & RADEON_IS_IGP)
8619 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8620
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8621 if (!(rdev->flags & RADEON_IS_PCIE))
8622 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8623
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8624 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8625 if (ret != 0)
8626 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8627
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8628 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8629 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8630
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8631 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8632 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8633 LC_CURRENT_DATA_RATE_SHIFT;
8634 if (mask & DRM_PCIE_SPEED_80) {
8635 if (current_data_rate == 2) {
8636 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8637 return;
8638 }
8639 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8640 } else if (mask & DRM_PCIE_SPEED_50) {
8641 if (current_data_rate == 1) {
8642 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8643 return;
8644 }
8645 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8646 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8647
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8648 bridge_pos = pci_pcie_cap(root);
8649 if (!bridge_pos)
8650 return;
8651
8652 gpu_pos = pci_pcie_cap(rdev->pdev);
8653 if (!gpu_pos)
8654 return;
8655
8656 if (mask & DRM_PCIE_SPEED_80) {
8657 /* re-try equalization if gen3 is not already enabled */
8658 if (current_data_rate != 2) {
8659 u16 bridge_cfg, gpu_cfg;
8660 u16 bridge_cfg2, gpu_cfg2;
8661 u32 max_lw, current_lw, tmp;
8662
8663 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8664 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8665
8666 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8667 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8668
8669 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8670 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8671
8672 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8673 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8674 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8675
8676 if (current_lw < max_lw) {
8677 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8678 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8679 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8680 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8681 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8682 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8683 }
8684 }
8685
8686 for (i = 0; i < 10; i++) {
8687 /* check status */
8688 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8689 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8690 break;
8691
8692 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8693 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8694
8695 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8696 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8697
8698 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8699 tmp |= LC_SET_QUIESCE;
8700 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8701
8702 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8703 tmp |= LC_REDO_EQ;
8704 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8705
8706 mdelay(100);
8707
8708 /* linkctl */
8709 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8710 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8711 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8712 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8713
8714 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8715 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8716 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8717 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8718
8719 /* linkctl2 */
8720 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8721 tmp16 &= ~((1 << 4) | (7 << 9));
8722 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8723 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8724
8725 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8726 tmp16 &= ~((1 << 4) | (7 << 9));
8727 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8728 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8729
8730 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8731 tmp &= ~LC_SET_QUIESCE;
8732 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8733 }
8734 }
8735 }
8736
8737 /* set the link speed */
8738 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8739 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8740 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8741
8742 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8743 tmp16 &= ~0xf;
8744 if (mask & DRM_PCIE_SPEED_80)
8745 tmp16 |= 3; /* gen3 */
8746 else if (mask & DRM_PCIE_SPEED_50)
8747 tmp16 |= 2; /* gen2 */
8748 else
8749 tmp16 |= 1; /* gen1 */
8750 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8751
8752 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8753 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8754 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8755
8756 for (i = 0; i < rdev->usec_timeout; i++) {
8757 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8758 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8759 break;
8760 udelay(1);
8761 }
8762}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]8763
8764static void cik_program_aspm(struct radeon_device *rdev)
8765{
8766 u32 data, orig;
8767 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8768 bool disable_clkreq = false;
8769
8770 if (radeon_aspm == 0)
8771 return;
8772
8773 /* XXX double check IGPs */
8774 if (rdev->flags & RADEON_IS_IGP)
8775 return;
8776
8777 if (!(rdev->flags & RADEON_IS_PCIE))
8778 return;
8779
8780 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8781 data &= ~LC_XMIT_N_FTS_MASK;
8782 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8783 if (orig != data)
8784 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8785
8786 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8787 data |= LC_GO_TO_RECOVERY;
8788 if (orig != data)
8789 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8790
8791 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8792 data |= P_IGNORE_EDB_ERR;
8793 if (orig != data)
8794 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8795
8796 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8797 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8798 data |= LC_PMI_TO_L1_DIS;
8799 if (!disable_l0s)
8800 data |= LC_L0S_INACTIVITY(7);
8801
8802 if (!disable_l1) {
8803 data |= LC_L1_INACTIVITY(7);
8804 data &= ~LC_PMI_TO_L1_DIS;
8805 if (orig != data)
8806 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8807
8808 if (!disable_plloff_in_l1) {
8809 bool clk_req_support;
8810
8811 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8812 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8813 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8814 if (orig != data)
8815 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8816
8817 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8818 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8819 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8820 if (orig != data)
8821 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8822
8823 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8824 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8825 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8826 if (orig != data)
8827 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8828
8829 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8830 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8831 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8832 if (orig != data)
8833 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8834
8835 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8836 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8837 data |= LC_DYN_LANES_PWR_STATE(3);
8838 if (orig != data)
8839 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8840
8841 if (!disable_clkreq) {
8842 struct pci_dev *root = rdev->pdev->bus->self;
8843 u32 lnkcap;
8844
8845 clk_req_support = false;
8846 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8847 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8848 clk_req_support = true;
8849 } else {
8850 clk_req_support = false;
8851 }
8852
8853 if (clk_req_support) {
8854 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8855 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8856 if (orig != data)
8857 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8858
8859 orig = data = RREG32_SMC(THM_CLK_CNTL);
8860 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8861 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8862 if (orig != data)
8863 WREG32_SMC(THM_CLK_CNTL, data);
8864
8865 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8866 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8867 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8868 if (orig != data)
8869 WREG32_SMC(MISC_CLK_CTRL, data);
8870
8871 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8872 data &= ~BCLK_AS_XCLK;
8873 if (orig != data)
8874 WREG32_SMC(CG_CLKPIN_CNTL, data);
8875
8876 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8877 data &= ~FORCE_BIF_REFCLK_EN;
8878 if (orig != data)
8879 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8880
8881 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8882 data &= ~MPLL_CLKOUT_SEL_MASK;
8883 data |= MPLL_CLKOUT_SEL(4);
8884 if (orig != data)
8885 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8886 }
8887 }
8888 } else {
8889 if (orig != data)
8890 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8891 }
8892
8893 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8894 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8895 if (orig != data)
8896 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8897
8898 if (!disable_l0s) {
8899 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8900 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8901 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8902 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8903 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8904 data &= ~LC_L0S_INACTIVITY_MASK;
8905 if (orig != data)
8906 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8907 }
8908 }
8909 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8910}