blob: 389f5a981358b9cbab6093034f31fee0ff27952e [file] [log] [blame]
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090026#include <linux/slab.h>
David Howells760285e2012-10-02 18:01:07 +010027#include <drm/drmP.h>
Alex Deucherbcc1c2a2010-01-12 17:54:34 -050028#include "radeon.h"
Daniel Vettere6990372010-03-11 21:19:17 +000029#include "radeon_asic.h"
David Howells760285e2012-10-02 18:01:07 +010030#include <drm/radeon_drm.h>
Alex Deucher0fcdb612010-03-24 13:20:41 -040031#include "evergreend.h"
Alex Deucherbcc1c2a2010-01-12 17:54:34 -050032#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
Alex Deucher2281a372010-10-21 13:31:38 -040035#include "evergreen_blit_shaders.h"
Alex Deucher138e4e12013-01-11 15:33:13 -050036#include "radeon_ucode.h"
Alex Deucherfe251e22010-03-24 13:36:43 -040037
Alex Deucher4a159032012-08-15 17:13:53 -040038static const u32 crtc_offsets[6] =
39{
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46};
47
Alex Deucher2948f5e2013-04-12 13:52:52 -040048#include "clearstate_evergreen.h"
49
Alex Deucher1fd11772013-04-17 17:53:50 -040050static const u32 sumo_rlc_save_restore_register_list[] =
Alex Deucher2948f5e2013-04-12 13:52:52 -040051{
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133};
Alex Deucher2948f5e2013-04-12 13:52:52 -0400134
Alex Deucherbcc1c2a2010-01-12 17:54:34 -0500135static void evergreen_gpu_init(struct radeon_device *rdev);
136void evergreen_fini(struct radeon_device *rdev);
Ilija Hadzicb07759b2011-09-20 10:22:58 -0400137void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
Alex Deucherf52382d2013-02-15 11:02:50 -0500138void evergreen_program_aspm(struct radeon_device *rdev);
Alex Deucher1b370782011-11-17 20:13:28 -0500139extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
Alex Deucher54e2e492013-06-13 18:26:25 -0400141extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
Alex Deucher22c775c2013-07-23 09:41:05 -0400143void cik_init_cp_pg_table(struct radeon_device *rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -0500144
Alex Deucherd4788db2013-02-28 14:40:09 -0500145static const u32 evergreen_golden_registers[] =
146{
147 0x3f90, 0xffff0000, 0xff000000,
148 0x9148, 0xffff0000, 0xff000000,
149 0x3f94, 0xffff0000, 0xff000000,
150 0x914c, 0xffff0000, 0xff000000,
151 0x9b7c, 0xffffffff, 0x00000000,
152 0x8a14, 0xffffffff, 0x00000007,
153 0x8b10, 0xffffffff, 0x00000000,
154 0x960c, 0xffffffff, 0x54763210,
155 0x88c4, 0xffffffff, 0x000000c2,
156 0x88d4, 0xffffffff, 0x00000010,
157 0x8974, 0xffffffff, 0x00000000,
158 0xc78, 0x00000080, 0x00000080,
159 0x5eb4, 0xffffffff, 0x00000002,
160 0x5e78, 0xffffffff, 0x001000f0,
161 0x6104, 0x01000300, 0x00000000,
162 0x5bc0, 0x00300000, 0x00000000,
163 0x7030, 0xffffffff, 0x00000011,
164 0x7c30, 0xffffffff, 0x00000011,
165 0x10830, 0xffffffff, 0x00000011,
166 0x11430, 0xffffffff, 0x00000011,
167 0x12030, 0xffffffff, 0x00000011,
168 0x12c30, 0xffffffff, 0x00000011,
169 0xd02c, 0xffffffff, 0x08421000,
170 0x240c, 0xffffffff, 0x00000380,
171 0x8b24, 0xffffffff, 0x00ff0fff,
172 0x28a4c, 0x06000000, 0x06000000,
173 0x10c, 0x00000001, 0x00000001,
174 0x8d00, 0xffffffff, 0x100e4848,
175 0x8d04, 0xffffffff, 0x00164745,
176 0x8c00, 0xffffffff, 0xe4000003,
177 0x8c04, 0xffffffff, 0x40600060,
178 0x8c08, 0xffffffff, 0x001c001c,
179 0x8cf0, 0xffffffff, 0x08e00620,
180 0x8c20, 0xffffffff, 0x00800080,
181 0x8c24, 0xffffffff, 0x00800080,
182 0x8c18, 0xffffffff, 0x20202078,
183 0x8c1c, 0xffffffff, 0x00001010,
184 0x28350, 0xffffffff, 0x00000000,
185 0xa008, 0xffffffff, 0x00010000,
186 0x5cc, 0xffffffff, 0x00000001,
187 0x9508, 0xffffffff, 0x00000002,
188 0x913c, 0x0000000f, 0x0000000a
189};
190
191static const u32 evergreen_golden_registers2[] =
192{
193 0x2f4c, 0xffffffff, 0x00000000,
194 0x54f4, 0xffffffff, 0x00000000,
195 0x54f0, 0xffffffff, 0x00000000,
196 0x5498, 0xffffffff, 0x00000000,
197 0x549c, 0xffffffff, 0x00000000,
198 0x5494, 0xffffffff, 0x00000000,
199 0x53cc, 0xffffffff, 0x00000000,
200 0x53c8, 0xffffffff, 0x00000000,
201 0x53c4, 0xffffffff, 0x00000000,
202 0x53c0, 0xffffffff, 0x00000000,
203 0x53bc, 0xffffffff, 0x00000000,
204 0x53b8, 0xffffffff, 0x00000000,
205 0x53b4, 0xffffffff, 0x00000000,
206 0x53b0, 0xffffffff, 0x00000000
207};
208
209static const u32 cypress_mgcg_init[] =
210{
211 0x802c, 0xffffffff, 0xc0000000,
212 0x5448, 0xffffffff, 0x00000100,
213 0x55e4, 0xffffffff, 0x00000100,
214 0x160c, 0xffffffff, 0x00000100,
215 0x5644, 0xffffffff, 0x00000100,
216 0xc164, 0xffffffff, 0x00000100,
217 0x8a18, 0xffffffff, 0x00000100,
218 0x897c, 0xffffffff, 0x06000100,
219 0x8b28, 0xffffffff, 0x00000100,
220 0x9144, 0xffffffff, 0x00000100,
221 0x9a60, 0xffffffff, 0x00000100,
222 0x9868, 0xffffffff, 0x00000100,
223 0x8d58, 0xffffffff, 0x00000100,
224 0x9510, 0xffffffff, 0x00000100,
225 0x949c, 0xffffffff, 0x00000100,
226 0x9654, 0xffffffff, 0x00000100,
227 0x9030, 0xffffffff, 0x00000100,
228 0x9034, 0xffffffff, 0x00000100,
229 0x9038, 0xffffffff, 0x00000100,
230 0x903c, 0xffffffff, 0x00000100,
231 0x9040, 0xffffffff, 0x00000100,
232 0xa200, 0xffffffff, 0x00000100,
233 0xa204, 0xffffffff, 0x00000100,
234 0xa208, 0xffffffff, 0x00000100,
235 0xa20c, 0xffffffff, 0x00000100,
236 0x971c, 0xffffffff, 0x00000100,
237 0x977c, 0xffffffff, 0x00000100,
238 0x3f80, 0xffffffff, 0x00000100,
239 0xa210, 0xffffffff, 0x00000100,
240 0xa214, 0xffffffff, 0x00000100,
241 0x4d8, 0xffffffff, 0x00000100,
242 0x9784, 0xffffffff, 0x00000100,
243 0x9698, 0xffffffff, 0x00000100,
244 0x4d4, 0xffffffff, 0x00000200,
245 0x30cc, 0xffffffff, 0x00000100,
246 0xd0c0, 0xffffffff, 0xff000100,
247 0x802c, 0xffffffff, 0x40000000,
248 0x915c, 0xffffffff, 0x00010000,
249 0x9160, 0xffffffff, 0x00030002,
250 0x9178, 0xffffffff, 0x00070000,
251 0x917c, 0xffffffff, 0x00030002,
252 0x9180, 0xffffffff, 0x00050004,
253 0x918c, 0xffffffff, 0x00010006,
254 0x9190, 0xffffffff, 0x00090008,
255 0x9194, 0xffffffff, 0x00070000,
256 0x9198, 0xffffffff, 0x00030002,
257 0x919c, 0xffffffff, 0x00050004,
258 0x91a8, 0xffffffff, 0x00010006,
259 0x91ac, 0xffffffff, 0x00090008,
260 0x91b0, 0xffffffff, 0x00070000,
261 0x91b4, 0xffffffff, 0x00030002,
262 0x91b8, 0xffffffff, 0x00050004,
263 0x91c4, 0xffffffff, 0x00010006,
264 0x91c8, 0xffffffff, 0x00090008,
265 0x91cc, 0xffffffff, 0x00070000,
266 0x91d0, 0xffffffff, 0x00030002,
267 0x91d4, 0xffffffff, 0x00050004,
268 0x91e0, 0xffffffff, 0x00010006,
269 0x91e4, 0xffffffff, 0x00090008,
270 0x91e8, 0xffffffff, 0x00000000,
271 0x91ec, 0xffffffff, 0x00070000,
272 0x91f0, 0xffffffff, 0x00030002,
273 0x91f4, 0xffffffff, 0x00050004,
274 0x9200, 0xffffffff, 0x00010006,
275 0x9204, 0xffffffff, 0x00090008,
276 0x9208, 0xffffffff, 0x00070000,
277 0x920c, 0xffffffff, 0x00030002,
278 0x9210, 0xffffffff, 0x00050004,
279 0x921c, 0xffffffff, 0x00010006,
280 0x9220, 0xffffffff, 0x00090008,
281 0x9224, 0xffffffff, 0x00070000,
282 0x9228, 0xffffffff, 0x00030002,
283 0x922c, 0xffffffff, 0x00050004,
284 0x9238, 0xffffffff, 0x00010006,
285 0x923c, 0xffffffff, 0x00090008,
286 0x9240, 0xffffffff, 0x00070000,
287 0x9244, 0xffffffff, 0x00030002,
288 0x9248, 0xffffffff, 0x00050004,
289 0x9254, 0xffffffff, 0x00010006,
290 0x9258, 0xffffffff, 0x00090008,
291 0x925c, 0xffffffff, 0x00070000,
292 0x9260, 0xffffffff, 0x00030002,
293 0x9264, 0xffffffff, 0x00050004,
294 0x9270, 0xffffffff, 0x00010006,
295 0x9274, 0xffffffff, 0x00090008,
296 0x9278, 0xffffffff, 0x00070000,
297 0x927c, 0xffffffff, 0x00030002,
298 0x9280, 0xffffffff, 0x00050004,
299 0x928c, 0xffffffff, 0x00010006,
300 0x9290, 0xffffffff, 0x00090008,
301 0x9294, 0xffffffff, 0x00000000,
302 0x929c, 0xffffffff, 0x00000001,
303 0x802c, 0xffffffff, 0x40010000,
304 0x915c, 0xffffffff, 0x00010000,
305 0x9160, 0xffffffff, 0x00030002,
306 0x9178, 0xffffffff, 0x00070000,
307 0x917c, 0xffffffff, 0x00030002,
308 0x9180, 0xffffffff, 0x00050004,
309 0x918c, 0xffffffff, 0x00010006,
310 0x9190, 0xffffffff, 0x00090008,
311 0x9194, 0xffffffff, 0x00070000,
312 0x9198, 0xffffffff, 0x00030002,
313 0x919c, 0xffffffff, 0x00050004,
314 0x91a8, 0xffffffff, 0x00010006,
315 0x91ac, 0xffffffff, 0x00090008,
316 0x91b0, 0xffffffff, 0x00070000,
317 0x91b4, 0xffffffff, 0x00030002,
318 0x91b8, 0xffffffff, 0x00050004,
319 0x91c4, 0xffffffff, 0x00010006,
320 0x91c8, 0xffffffff, 0x00090008,
321 0x91cc, 0xffffffff, 0x00070000,
322 0x91d0, 0xffffffff, 0x00030002,
323 0x91d4, 0xffffffff, 0x00050004,
324 0x91e0, 0xffffffff, 0x00010006,
325 0x91e4, 0xffffffff, 0x00090008,
326 0x91e8, 0xffffffff, 0x00000000,
327 0x91ec, 0xffffffff, 0x00070000,
328 0x91f0, 0xffffffff, 0x00030002,
329 0x91f4, 0xffffffff, 0x00050004,
330 0x9200, 0xffffffff, 0x00010006,
331 0x9204, 0xffffffff, 0x00090008,
332 0x9208, 0xffffffff, 0x00070000,
333 0x920c, 0xffffffff, 0x00030002,
334 0x9210, 0xffffffff, 0x00050004,
335 0x921c, 0xffffffff, 0x00010006,
336 0x9220, 0xffffffff, 0x00090008,
337 0x9224, 0xffffffff, 0x00070000,
338 0x9228, 0xffffffff, 0x00030002,
339 0x922c, 0xffffffff, 0x00050004,
340 0x9238, 0xffffffff, 0x00010006,
341 0x923c, 0xffffffff, 0x00090008,
342 0x9240, 0xffffffff, 0x00070000,
343 0x9244, 0xffffffff, 0x00030002,
344 0x9248, 0xffffffff, 0x00050004,
345 0x9254, 0xffffffff, 0x00010006,
346 0x9258, 0xffffffff, 0x00090008,
347 0x925c, 0xffffffff, 0x00070000,
348 0x9260, 0xffffffff, 0x00030002,
349 0x9264, 0xffffffff, 0x00050004,
350 0x9270, 0xffffffff, 0x00010006,
351 0x9274, 0xffffffff, 0x00090008,
352 0x9278, 0xffffffff, 0x00070000,
353 0x927c, 0xffffffff, 0x00030002,
354 0x9280, 0xffffffff, 0x00050004,
355 0x928c, 0xffffffff, 0x00010006,
356 0x9290, 0xffffffff, 0x00090008,
357 0x9294, 0xffffffff, 0x00000000,
358 0x929c, 0xffffffff, 0x00000001,
359 0x802c, 0xffffffff, 0xc0000000
360};
361
362static const u32 redwood_mgcg_init[] =
363{
364 0x802c, 0xffffffff, 0xc0000000,
365 0x5448, 0xffffffff, 0x00000100,
366 0x55e4, 0xffffffff, 0x00000100,
367 0x160c, 0xffffffff, 0x00000100,
368 0x5644, 0xffffffff, 0x00000100,
369 0xc164, 0xffffffff, 0x00000100,
370 0x8a18, 0xffffffff, 0x00000100,
371 0x897c, 0xffffffff, 0x06000100,
372 0x8b28, 0xffffffff, 0x00000100,
373 0x9144, 0xffffffff, 0x00000100,
374 0x9a60, 0xffffffff, 0x00000100,
375 0x9868, 0xffffffff, 0x00000100,
376 0x8d58, 0xffffffff, 0x00000100,
377 0x9510, 0xffffffff, 0x00000100,
378 0x949c, 0xffffffff, 0x00000100,
379 0x9654, 0xffffffff, 0x00000100,
380 0x9030, 0xffffffff, 0x00000100,
381 0x9034, 0xffffffff, 0x00000100,
382 0x9038, 0xffffffff, 0x00000100,
383 0x903c, 0xffffffff, 0x00000100,
384 0x9040, 0xffffffff, 0x00000100,
385 0xa200, 0xffffffff, 0x00000100,
386 0xa204, 0xffffffff, 0x00000100,
387 0xa208, 0xffffffff, 0x00000100,
388 0xa20c, 0xffffffff, 0x00000100,
389 0x971c, 0xffffffff, 0x00000100,
390 0x977c, 0xffffffff, 0x00000100,
391 0x3f80, 0xffffffff, 0x00000100,
392 0xa210, 0xffffffff, 0x00000100,
393 0xa214, 0xffffffff, 0x00000100,
394 0x4d8, 0xffffffff, 0x00000100,
395 0x9784, 0xffffffff, 0x00000100,
396 0x9698, 0xffffffff, 0x00000100,
397 0x4d4, 0xffffffff, 0x00000200,
398 0x30cc, 0xffffffff, 0x00000100,
399 0xd0c0, 0xffffffff, 0xff000100,
400 0x802c, 0xffffffff, 0x40000000,
401 0x915c, 0xffffffff, 0x00010000,
402 0x9160, 0xffffffff, 0x00030002,
403 0x9178, 0xffffffff, 0x00070000,
404 0x917c, 0xffffffff, 0x00030002,
405 0x9180, 0xffffffff, 0x00050004,
406 0x918c, 0xffffffff, 0x00010006,
407 0x9190, 0xffffffff, 0x00090008,
408 0x9194, 0xffffffff, 0x00070000,
409 0x9198, 0xffffffff, 0x00030002,
410 0x919c, 0xffffffff, 0x00050004,
411 0x91a8, 0xffffffff, 0x00010006,
412 0x91ac, 0xffffffff, 0x00090008,
413 0x91b0, 0xffffffff, 0x00070000,
414 0x91b4, 0xffffffff, 0x00030002,
415 0x91b8, 0xffffffff, 0x00050004,
416 0x91c4, 0xffffffff, 0x00010006,
417 0x91c8, 0xffffffff, 0x00090008,
418 0x91cc, 0xffffffff, 0x00070000,
419 0x91d0, 0xffffffff, 0x00030002,
420 0x91d4, 0xffffffff, 0x00050004,
421 0x91e0, 0xffffffff, 0x00010006,
422 0x91e4, 0xffffffff, 0x00090008,
423 0x91e8, 0xffffffff, 0x00000000,
424 0x91ec, 0xffffffff, 0x00070000,
425 0x91f0, 0xffffffff, 0x00030002,
426 0x91f4, 0xffffffff, 0x00050004,
427 0x9200, 0xffffffff, 0x00010006,
428 0x9204, 0xffffffff, 0x00090008,
429 0x9294, 0xffffffff, 0x00000000,
430 0x929c, 0xffffffff, 0x00000001,
431 0x802c, 0xffffffff, 0xc0000000
432};
433
434static const u32 cedar_golden_registers[] =
435{
436 0x3f90, 0xffff0000, 0xff000000,
437 0x9148, 0xffff0000, 0xff000000,
438 0x3f94, 0xffff0000, 0xff000000,
439 0x914c, 0xffff0000, 0xff000000,
440 0x9b7c, 0xffffffff, 0x00000000,
441 0x8a14, 0xffffffff, 0x00000007,
442 0x8b10, 0xffffffff, 0x00000000,
443 0x960c, 0xffffffff, 0x54763210,
444 0x88c4, 0xffffffff, 0x000000c2,
445 0x88d4, 0xffffffff, 0x00000000,
446 0x8974, 0xffffffff, 0x00000000,
447 0xc78, 0x00000080, 0x00000080,
448 0x5eb4, 0xffffffff, 0x00000002,
449 0x5e78, 0xffffffff, 0x001000f0,
450 0x6104, 0x01000300, 0x00000000,
451 0x5bc0, 0x00300000, 0x00000000,
452 0x7030, 0xffffffff, 0x00000011,
453 0x7c30, 0xffffffff, 0x00000011,
454 0x10830, 0xffffffff, 0x00000011,
455 0x11430, 0xffffffff, 0x00000011,
456 0xd02c, 0xffffffff, 0x08421000,
457 0x240c, 0xffffffff, 0x00000380,
458 0x8b24, 0xffffffff, 0x00ff0fff,
459 0x28a4c, 0x06000000, 0x06000000,
460 0x10c, 0x00000001, 0x00000001,
461 0x8d00, 0xffffffff, 0x100e4848,
462 0x8d04, 0xffffffff, 0x00164745,
463 0x8c00, 0xffffffff, 0xe4000003,
464 0x8c04, 0xffffffff, 0x40600060,
465 0x8c08, 0xffffffff, 0x001c001c,
466 0x8cf0, 0xffffffff, 0x08e00410,
467 0x8c20, 0xffffffff, 0x00800080,
468 0x8c24, 0xffffffff, 0x00800080,
469 0x8c18, 0xffffffff, 0x20202078,
470 0x8c1c, 0xffffffff, 0x00001010,
471 0x28350, 0xffffffff, 0x00000000,
472 0xa008, 0xffffffff, 0x00010000,
473 0x5cc, 0xffffffff, 0x00000001,
474 0x9508, 0xffffffff, 0x00000002
475};
476
477static const u32 cedar_mgcg_init[] =
478{
479 0x802c, 0xffffffff, 0xc0000000,
480 0x5448, 0xffffffff, 0x00000100,
481 0x55e4, 0xffffffff, 0x00000100,
482 0x160c, 0xffffffff, 0x00000100,
483 0x5644, 0xffffffff, 0x00000100,
484 0xc164, 0xffffffff, 0x00000100,
485 0x8a18, 0xffffffff, 0x00000100,
486 0x897c, 0xffffffff, 0x06000100,
487 0x8b28, 0xffffffff, 0x00000100,
488 0x9144, 0xffffffff, 0x00000100,
489 0x9a60, 0xffffffff, 0x00000100,
490 0x9868, 0xffffffff, 0x00000100,
491 0x8d58, 0xffffffff, 0x00000100,
492 0x9510, 0xffffffff, 0x00000100,
493 0x949c, 0xffffffff, 0x00000100,
494 0x9654, 0xffffffff, 0x00000100,
495 0x9030, 0xffffffff, 0x00000100,
496 0x9034, 0xffffffff, 0x00000100,
497 0x9038, 0xffffffff, 0x00000100,
498 0x903c, 0xffffffff, 0x00000100,
499 0x9040, 0xffffffff, 0x00000100,
500 0xa200, 0xffffffff, 0x00000100,
501 0xa204, 0xffffffff, 0x00000100,
502 0xa208, 0xffffffff, 0x00000100,
503 0xa20c, 0xffffffff, 0x00000100,
504 0x971c, 0xffffffff, 0x00000100,
505 0x977c, 0xffffffff, 0x00000100,
506 0x3f80, 0xffffffff, 0x00000100,
507 0xa210, 0xffffffff, 0x00000100,
508 0xa214, 0xffffffff, 0x00000100,
509 0x4d8, 0xffffffff, 0x00000100,
510 0x9784, 0xffffffff, 0x00000100,
511 0x9698, 0xffffffff, 0x00000100,
512 0x4d4, 0xffffffff, 0x00000200,
513 0x30cc, 0xffffffff, 0x00000100,
514 0xd0c0, 0xffffffff, 0xff000100,
515 0x802c, 0xffffffff, 0x40000000,
516 0x915c, 0xffffffff, 0x00010000,
517 0x9178, 0xffffffff, 0x00050000,
518 0x917c, 0xffffffff, 0x00030002,
519 0x918c, 0xffffffff, 0x00010004,
520 0x9190, 0xffffffff, 0x00070006,
521 0x9194, 0xffffffff, 0x00050000,
522 0x9198, 0xffffffff, 0x00030002,
523 0x91a8, 0xffffffff, 0x00010004,
524 0x91ac, 0xffffffff, 0x00070006,
525 0x91e8, 0xffffffff, 0x00000000,
526 0x9294, 0xffffffff, 0x00000000,
527 0x929c, 0xffffffff, 0x00000001,
528 0x802c, 0xffffffff, 0xc0000000
529};
530
531static const u32 juniper_mgcg_init[] =
532{
533 0x802c, 0xffffffff, 0xc0000000,
534 0x5448, 0xffffffff, 0x00000100,
535 0x55e4, 0xffffffff, 0x00000100,
536 0x160c, 0xffffffff, 0x00000100,
537 0x5644, 0xffffffff, 0x00000100,
538 0xc164, 0xffffffff, 0x00000100,
539 0x8a18, 0xffffffff, 0x00000100,
540 0x897c, 0xffffffff, 0x06000100,
541 0x8b28, 0xffffffff, 0x00000100,
542 0x9144, 0xffffffff, 0x00000100,
543 0x9a60, 0xffffffff, 0x00000100,
544 0x9868, 0xffffffff, 0x00000100,
545 0x8d58, 0xffffffff, 0x00000100,
546 0x9510, 0xffffffff, 0x00000100,
547 0x949c, 0xffffffff, 0x00000100,
548 0x9654, 0xffffffff, 0x00000100,
549 0x9030, 0xffffffff, 0x00000100,
550 0x9034, 0xffffffff, 0x00000100,
551 0x9038, 0xffffffff, 0x00000100,
552 0x903c, 0xffffffff, 0x00000100,
553 0x9040, 0xffffffff, 0x00000100,
554 0xa200, 0xffffffff, 0x00000100,
555 0xa204, 0xffffffff, 0x00000100,
556 0xa208, 0xffffffff, 0x00000100,
557 0xa20c, 0xffffffff, 0x00000100,
558 0x971c, 0xffffffff, 0x00000100,
559 0xd0c0, 0xffffffff, 0xff000100,
560 0x802c, 0xffffffff, 0x40000000,
561 0x915c, 0xffffffff, 0x00010000,
562 0x9160, 0xffffffff, 0x00030002,
563 0x9178, 0xffffffff, 0x00070000,
564 0x917c, 0xffffffff, 0x00030002,
565 0x9180, 0xffffffff, 0x00050004,
566 0x918c, 0xffffffff, 0x00010006,
567 0x9190, 0xffffffff, 0x00090008,
568 0x9194, 0xffffffff, 0x00070000,
569 0x9198, 0xffffffff, 0x00030002,
570 0x919c, 0xffffffff, 0x00050004,
571 0x91a8, 0xffffffff, 0x00010006,
572 0x91ac, 0xffffffff, 0x00090008,
573 0x91b0, 0xffffffff, 0x00070000,
574 0x91b4, 0xffffffff, 0x00030002,
575 0x91b8, 0xffffffff, 0x00050004,
576 0x91c4, 0xffffffff, 0x00010006,
577 0x91c8, 0xffffffff, 0x00090008,
578 0x91cc, 0xffffffff, 0x00070000,
579 0x91d0, 0xffffffff, 0x00030002,
580 0x91d4, 0xffffffff, 0x00050004,
581 0x91e0, 0xffffffff, 0x00010006,
582 0x91e4, 0xffffffff, 0x00090008,
583 0x91e8, 0xffffffff, 0x00000000,
584 0x91ec, 0xffffffff, 0x00070000,
585 0x91f0, 0xffffffff, 0x00030002,
586 0x91f4, 0xffffffff, 0x00050004,
587 0x9200, 0xffffffff, 0x00010006,
588 0x9204, 0xffffffff, 0x00090008,
589 0x9208, 0xffffffff, 0x00070000,
590 0x920c, 0xffffffff, 0x00030002,
591 0x9210, 0xffffffff, 0x00050004,
592 0x921c, 0xffffffff, 0x00010006,
593 0x9220, 0xffffffff, 0x00090008,
594 0x9224, 0xffffffff, 0x00070000,
595 0x9228, 0xffffffff, 0x00030002,
596 0x922c, 0xffffffff, 0x00050004,
597 0x9238, 0xffffffff, 0x00010006,
598 0x923c, 0xffffffff, 0x00090008,
599 0x9240, 0xffffffff, 0x00070000,
600 0x9244, 0xffffffff, 0x00030002,
601 0x9248, 0xffffffff, 0x00050004,
602 0x9254, 0xffffffff, 0x00010006,
603 0x9258, 0xffffffff, 0x00090008,
604 0x925c, 0xffffffff, 0x00070000,
605 0x9260, 0xffffffff, 0x00030002,
606 0x9264, 0xffffffff, 0x00050004,
607 0x9270, 0xffffffff, 0x00010006,
608 0x9274, 0xffffffff, 0x00090008,
609 0x9278, 0xffffffff, 0x00070000,
610 0x927c, 0xffffffff, 0x00030002,
611 0x9280, 0xffffffff, 0x00050004,
612 0x928c, 0xffffffff, 0x00010006,
613 0x9290, 0xffffffff, 0x00090008,
614 0x9294, 0xffffffff, 0x00000000,
615 0x929c, 0xffffffff, 0x00000001,
616 0x802c, 0xffffffff, 0xc0000000,
617 0x977c, 0xffffffff, 0x00000100,
618 0x3f80, 0xffffffff, 0x00000100,
619 0xa210, 0xffffffff, 0x00000100,
620 0xa214, 0xffffffff, 0x00000100,
621 0x4d8, 0xffffffff, 0x00000100,
622 0x9784, 0xffffffff, 0x00000100,
623 0x9698, 0xffffffff, 0x00000100,
624 0x4d4, 0xffffffff, 0x00000200,
625 0x30cc, 0xffffffff, 0x00000100,
626 0x802c, 0xffffffff, 0xc0000000
627};
628
629static const u32 supersumo_golden_registers[] =
630{
631 0x5eb4, 0xffffffff, 0x00000002,
632 0x5cc, 0xffffffff, 0x00000001,
633 0x7030, 0xffffffff, 0x00000011,
634 0x7c30, 0xffffffff, 0x00000011,
635 0x6104, 0x01000300, 0x00000000,
636 0x5bc0, 0x00300000, 0x00000000,
637 0x8c04, 0xffffffff, 0x40600060,
638 0x8c08, 0xffffffff, 0x001c001c,
639 0x8c20, 0xffffffff, 0x00800080,
640 0x8c24, 0xffffffff, 0x00800080,
641 0x8c18, 0xffffffff, 0x20202078,
642 0x8c1c, 0xffffffff, 0x00001010,
643 0x918c, 0xffffffff, 0x00010006,
644 0x91a8, 0xffffffff, 0x00010006,
645 0x91c4, 0xffffffff, 0x00010006,
646 0x91e0, 0xffffffff, 0x00010006,
647 0x9200, 0xffffffff, 0x00010006,
648 0x9150, 0xffffffff, 0x6e944040,
649 0x917c, 0xffffffff, 0x00030002,
650 0x9180, 0xffffffff, 0x00050004,
651 0x9198, 0xffffffff, 0x00030002,
652 0x919c, 0xffffffff, 0x00050004,
653 0x91b4, 0xffffffff, 0x00030002,
654 0x91b8, 0xffffffff, 0x00050004,
655 0x91d0, 0xffffffff, 0x00030002,
656 0x91d4, 0xffffffff, 0x00050004,
657 0x91f0, 0xffffffff, 0x00030002,
658 0x91f4, 0xffffffff, 0x00050004,
659 0x915c, 0xffffffff, 0x00010000,
660 0x9160, 0xffffffff, 0x00030002,
661 0x3f90, 0xffff0000, 0xff000000,
662 0x9178, 0xffffffff, 0x00070000,
663 0x9194, 0xffffffff, 0x00070000,
664 0x91b0, 0xffffffff, 0x00070000,
665 0x91cc, 0xffffffff, 0x00070000,
666 0x91ec, 0xffffffff, 0x00070000,
667 0x9148, 0xffff0000, 0xff000000,
668 0x9190, 0xffffffff, 0x00090008,
669 0x91ac, 0xffffffff, 0x00090008,
670 0x91c8, 0xffffffff, 0x00090008,
671 0x91e4, 0xffffffff, 0x00090008,
672 0x9204, 0xffffffff, 0x00090008,
673 0x3f94, 0xffff0000, 0xff000000,
674 0x914c, 0xffff0000, 0xff000000,
675 0x929c, 0xffffffff, 0x00000001,
676 0x8a18, 0xffffffff, 0x00000100,
677 0x8b28, 0xffffffff, 0x00000100,
678 0x9144, 0xffffffff, 0x00000100,
679 0x5644, 0xffffffff, 0x00000100,
680 0x9b7c, 0xffffffff, 0x00000000,
681 0x8030, 0xffffffff, 0x0000100a,
682 0x8a14, 0xffffffff, 0x00000007,
683 0x8b24, 0xffffffff, 0x00ff0fff,
684 0x8b10, 0xffffffff, 0x00000000,
685 0x28a4c, 0x06000000, 0x06000000,
686 0x4d8, 0xffffffff, 0x00000100,
687 0x913c, 0xffff000f, 0x0100000a,
688 0x960c, 0xffffffff, 0x54763210,
689 0x88c4, 0xffffffff, 0x000000c2,
690 0x88d4, 0xffffffff, 0x00000010,
691 0x8974, 0xffffffff, 0x00000000,
692 0xc78, 0x00000080, 0x00000080,
693 0x5e78, 0xffffffff, 0x001000f0,
694 0xd02c, 0xffffffff, 0x08421000,
695 0xa008, 0xffffffff, 0x00010000,
696 0x8d00, 0xffffffff, 0x100e4848,
697 0x8d04, 0xffffffff, 0x00164745,
698 0x8c00, 0xffffffff, 0xe4000003,
699 0x8cf0, 0x1fffffff, 0x08e00620,
700 0x28350, 0xffffffff, 0x00000000,
701 0x9508, 0xffffffff, 0x00000002
702};
703
704static const u32 sumo_golden_registers[] =
705{
706 0x900c, 0x00ffffff, 0x0017071f,
707 0x8c18, 0xffffffff, 0x10101060,
708 0x8c1c, 0xffffffff, 0x00001010,
709 0x8c30, 0x0000000f, 0x00000005,
710 0x9688, 0x0000000f, 0x00000007
711};
712
713static const u32 wrestler_golden_registers[] =
714{
715 0x5eb4, 0xffffffff, 0x00000002,
716 0x5cc, 0xffffffff, 0x00000001,
717 0x7030, 0xffffffff, 0x00000011,
718 0x7c30, 0xffffffff, 0x00000011,
719 0x6104, 0x01000300, 0x00000000,
720 0x5bc0, 0x00300000, 0x00000000,
721 0x918c, 0xffffffff, 0x00010006,
722 0x91a8, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9198, 0xffffffff, 0x00030002,
726 0x915c, 0xffffffff, 0x00010000,
727 0x3f90, 0xffff0000, 0xff000000,
728 0x9178, 0xffffffff, 0x00070000,
729 0x9194, 0xffffffff, 0x00070000,
730 0x9148, 0xffff0000, 0xff000000,
731 0x9190, 0xffffffff, 0x00090008,
732 0x91ac, 0xffffffff, 0x00090008,
733 0x3f94, 0xffff0000, 0xff000000,
734 0x914c, 0xffff0000, 0xff000000,
735 0x929c, 0xffffffff, 0x00000001,
736 0x8a18, 0xffffffff, 0x00000100,
737 0x8b28, 0xffffffff, 0x00000100,
738 0x9144, 0xffffffff, 0x00000100,
739 0x9b7c, 0xffffffff, 0x00000000,
740 0x8030, 0xffffffff, 0x0000100a,
741 0x8a14, 0xffffffff, 0x00000001,
742 0x8b24, 0xffffffff, 0x00ff0fff,
743 0x8b10, 0xffffffff, 0x00000000,
744 0x28a4c, 0x06000000, 0x06000000,
745 0x4d8, 0xffffffff, 0x00000100,
746 0x913c, 0xffff000f, 0x0100000a,
747 0x960c, 0xffffffff, 0x54763210,
748 0x88c4, 0xffffffff, 0x000000c2,
749 0x88d4, 0xffffffff, 0x00000010,
750 0x8974, 0xffffffff, 0x00000000,
751 0xc78, 0x00000080, 0x00000080,
752 0x5e78, 0xffffffff, 0x001000f0,
753 0xd02c, 0xffffffff, 0x08421000,
754 0xa008, 0xffffffff, 0x00010000,
755 0x8d00, 0xffffffff, 0x100e4848,
756 0x8d04, 0xffffffff, 0x00164745,
757 0x8c00, 0xffffffff, 0xe4000003,
758 0x8cf0, 0x1fffffff, 0x08e00410,
759 0x28350, 0xffffffff, 0x00000000,
760 0x9508, 0xffffffff, 0x00000002,
761 0x900c, 0xffffffff, 0x0017071f,
762 0x8c18, 0xffffffff, 0x10101060,
763 0x8c1c, 0xffffffff, 0x00001010
764};
765
766static const u32 barts_golden_registers[] =
767{
768 0x5eb4, 0xffffffff, 0x00000002,
769 0x5e78, 0x8f311ff1, 0x001000f0,
770 0x3f90, 0xffff0000, 0xff000000,
771 0x9148, 0xffff0000, 0xff000000,
772 0x3f94, 0xffff0000, 0xff000000,
773 0x914c, 0xffff0000, 0xff000000,
774 0xc78, 0x00000080, 0x00000080,
775 0xbd4, 0x70073777, 0x00010001,
776 0xd02c, 0xbfffff1f, 0x08421000,
777 0xd0b8, 0x03773777, 0x02011003,
778 0x5bc0, 0x00200000, 0x50100000,
779 0x98f8, 0x33773777, 0x02011003,
780 0x98fc, 0xffffffff, 0x76543210,
781 0x7030, 0x31000311, 0x00000011,
782 0x2f48, 0x00000007, 0x02011003,
783 0x6b28, 0x00000010, 0x00000012,
784 0x7728, 0x00000010, 0x00000012,
785 0x10328, 0x00000010, 0x00000012,
786 0x10f28, 0x00000010, 0x00000012,
787 0x11b28, 0x00000010, 0x00000012,
788 0x12728, 0x00000010, 0x00000012,
789 0x240c, 0x000007ff, 0x00000380,
790 0x8a14, 0xf000001f, 0x00000007,
791 0x8b24, 0x3fff3fff, 0x00ff0fff,
792 0x8b10, 0x0000ff0f, 0x00000000,
793 0x28a4c, 0x07ffffff, 0x06000000,
794 0x10c, 0x00000001, 0x00010003,
795 0xa02c, 0xffffffff, 0x0000009b,
796 0x913c, 0x0000000f, 0x0100000a,
797 0x8d00, 0xffff7f7f, 0x100e4848,
798 0x8d04, 0x00ffffff, 0x00164745,
799 0x8c00, 0xfffc0003, 0xe4000003,
800 0x8c04, 0xf8ff00ff, 0x40600060,
801 0x8c08, 0x00ff00ff, 0x001c001c,
802 0x8cf0, 0x1fff1fff, 0x08e00620,
803 0x8c20, 0x0fff0fff, 0x00800080,
804 0x8c24, 0x0fff0fff, 0x00800080,
805 0x8c18, 0xffffffff, 0x20202078,
806 0x8c1c, 0x0000ffff, 0x00001010,
807 0x28350, 0x00000f01, 0x00000000,
808 0x9508, 0x3700001f, 0x00000002,
809 0x960c, 0xffffffff, 0x54763210,
810 0x88c4, 0x001f3ae3, 0x000000c2,
811 0x88d4, 0x0000001f, 0x00000010,
812 0x8974, 0xffffffff, 0x00000000
813};
814
815static const u32 turks_golden_registers[] =
816{
817 0x5eb4, 0xffffffff, 0x00000002,
818 0x5e78, 0x8f311ff1, 0x001000f0,
819 0x8c8, 0x00003000, 0x00001070,
820 0x8cc, 0x000fffff, 0x00040035,
821 0x3f90, 0xffff0000, 0xfff00000,
822 0x9148, 0xffff0000, 0xfff00000,
823 0x3f94, 0xffff0000, 0xfff00000,
824 0x914c, 0xffff0000, 0xfff00000,
825 0xc78, 0x00000080, 0x00000080,
826 0xbd4, 0x00073007, 0x00010002,
827 0xd02c, 0xbfffff1f, 0x08421000,
828 0xd0b8, 0x03773777, 0x02010002,
829 0x5bc0, 0x00200000, 0x50100000,
830 0x98f8, 0x33773777, 0x00010002,
831 0x98fc, 0xffffffff, 0x33221100,
832 0x7030, 0x31000311, 0x00000011,
833 0x2f48, 0x33773777, 0x00010002,
834 0x6b28, 0x00000010, 0x00000012,
835 0x7728, 0x00000010, 0x00000012,
836 0x10328, 0x00000010, 0x00000012,
837 0x10f28, 0x00000010, 0x00000012,
838 0x11b28, 0x00000010, 0x00000012,
839 0x12728, 0x00000010, 0x00000012,
840 0x240c, 0x000007ff, 0x00000380,
841 0x8a14, 0xf000001f, 0x00000007,
842 0x8b24, 0x3fff3fff, 0x00ff0fff,
843 0x8b10, 0x0000ff0f, 0x00000000,
844 0x28a4c, 0x07ffffff, 0x06000000,
845 0x10c, 0x00000001, 0x00010003,
846 0xa02c, 0xffffffff, 0x0000009b,
847 0x913c, 0x0000000f, 0x0100000a,
848 0x8d00, 0xffff7f7f, 0x100e4848,
849 0x8d04, 0x00ffffff, 0x00164745,
850 0x8c00, 0xfffc0003, 0xe4000003,
851 0x8c04, 0xf8ff00ff, 0x40600060,
852 0x8c08, 0x00ff00ff, 0x001c001c,
853 0x8cf0, 0x1fff1fff, 0x08e00410,
854 0x8c20, 0x0fff0fff, 0x00800080,
855 0x8c24, 0x0fff0fff, 0x00800080,
856 0x8c18, 0xffffffff, 0x20202078,
857 0x8c1c, 0x0000ffff, 0x00001010,
858 0x28350, 0x00000f01, 0x00000000,
859 0x9508, 0x3700001f, 0x00000002,
860 0x960c, 0xffffffff, 0x54763210,
861 0x88c4, 0x001f3ae3, 0x000000c2,
862 0x88d4, 0x0000001f, 0x00000010,
863 0x8974, 0xffffffff, 0x00000000
864};
865
866static const u32 caicos_golden_registers[] =
867{
868 0x5eb4, 0xffffffff, 0x00000002,
869 0x5e78, 0x8f311ff1, 0x001000f0,
870 0x8c8, 0x00003420, 0x00001450,
871 0x8cc, 0x000fffff, 0x00040035,
872 0x3f90, 0xffff0000, 0xfffc0000,
873 0x9148, 0xffff0000, 0xfffc0000,
874 0x3f94, 0xffff0000, 0xfffc0000,
875 0x914c, 0xffff0000, 0xfffc0000,
876 0xc78, 0x00000080, 0x00000080,
877 0xbd4, 0x00073007, 0x00010001,
878 0xd02c, 0xbfffff1f, 0x08421000,
879 0xd0b8, 0x03773777, 0x02010001,
880 0x5bc0, 0x00200000, 0x50100000,
881 0x98f8, 0x33773777, 0x02010001,
882 0x98fc, 0xffffffff, 0x33221100,
883 0x7030, 0x31000311, 0x00000011,
884 0x2f48, 0x33773777, 0x02010001,
885 0x6b28, 0x00000010, 0x00000012,
886 0x7728, 0x00000010, 0x00000012,
887 0x10328, 0x00000010, 0x00000012,
888 0x10f28, 0x00000010, 0x00000012,
889 0x11b28, 0x00000010, 0x00000012,
890 0x12728, 0x00000010, 0x00000012,
891 0x240c, 0x000007ff, 0x00000380,
892 0x8a14, 0xf000001f, 0x00000001,
893 0x8b24, 0x3fff3fff, 0x00ff0fff,
894 0x8b10, 0x0000ff0f, 0x00000000,
895 0x28a4c, 0x07ffffff, 0x06000000,
896 0x10c, 0x00000001, 0x00010003,
897 0xa02c, 0xffffffff, 0x0000009b,
898 0x913c, 0x0000000f, 0x0100000a,
899 0x8d00, 0xffff7f7f, 0x100e4848,
900 0x8d04, 0x00ffffff, 0x00164745,
901 0x8c00, 0xfffc0003, 0xe4000003,
902 0x8c04, 0xf8ff00ff, 0x40600060,
903 0x8c08, 0x00ff00ff, 0x001c001c,
904 0x8cf0, 0x1fff1fff, 0x08e00410,
905 0x8c20, 0x0fff0fff, 0x00800080,
906 0x8c24, 0x0fff0fff, 0x00800080,
907 0x8c18, 0xffffffff, 0x20202078,
908 0x8c1c, 0x0000ffff, 0x00001010,
909 0x28350, 0x00000f01, 0x00000000,
910 0x9508, 0x3700001f, 0x00000002,
911 0x960c, 0xffffffff, 0x54763210,
912 0x88c4, 0x001f3ae3, 0x000000c2,
913 0x88d4, 0x0000001f, 0x00000010,
914 0x8974, 0xffffffff, 0x00000000
915};
916
917static void evergreen_init_golden_registers(struct radeon_device *rdev)
918{
919 switch (rdev->family) {
920 case CHIP_CYPRESS:
921 case CHIP_HEMLOCK:
922 radeon_program_register_sequence(rdev,
923 evergreen_golden_registers,
924 (const u32)ARRAY_SIZE(evergreen_golden_registers));
925 radeon_program_register_sequence(rdev,
926 evergreen_golden_registers2,
927 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
928 radeon_program_register_sequence(rdev,
929 cypress_mgcg_init,
930 (const u32)ARRAY_SIZE(cypress_mgcg_init));
931 break;
932 case CHIP_JUNIPER:
933 radeon_program_register_sequence(rdev,
934 evergreen_golden_registers,
935 (const u32)ARRAY_SIZE(evergreen_golden_registers));
936 radeon_program_register_sequence(rdev,
937 evergreen_golden_registers2,
938 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
939 radeon_program_register_sequence(rdev,
940 juniper_mgcg_init,
941 (const u32)ARRAY_SIZE(juniper_mgcg_init));
942 break;
943 case CHIP_REDWOOD:
944 radeon_program_register_sequence(rdev,
945 evergreen_golden_registers,
946 (const u32)ARRAY_SIZE(evergreen_golden_registers));
947 radeon_program_register_sequence(rdev,
948 evergreen_golden_registers2,
949 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
950 radeon_program_register_sequence(rdev,
951 redwood_mgcg_init,
952 (const u32)ARRAY_SIZE(redwood_mgcg_init));
953 break;
954 case CHIP_CEDAR:
955 radeon_program_register_sequence(rdev,
956 cedar_golden_registers,
957 (const u32)ARRAY_SIZE(cedar_golden_registers));
958 radeon_program_register_sequence(rdev,
959 evergreen_golden_registers2,
960 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
961 radeon_program_register_sequence(rdev,
962 cedar_mgcg_init,
963 (const u32)ARRAY_SIZE(cedar_mgcg_init));
964 break;
965 case CHIP_PALM:
966 radeon_program_register_sequence(rdev,
967 wrestler_golden_registers,
968 (const u32)ARRAY_SIZE(wrestler_golden_registers));
969 break;
970 case CHIP_SUMO:
971 radeon_program_register_sequence(rdev,
972 supersumo_golden_registers,
973 (const u32)ARRAY_SIZE(supersumo_golden_registers));
974 break;
975 case CHIP_SUMO2:
976 radeon_program_register_sequence(rdev,
977 supersumo_golden_registers,
978 (const u32)ARRAY_SIZE(supersumo_golden_registers));
979 radeon_program_register_sequence(rdev,
980 sumo_golden_registers,
981 (const u32)ARRAY_SIZE(sumo_golden_registers));
982 break;
983 case CHIP_BARTS:
984 radeon_program_register_sequence(rdev,
985 barts_golden_registers,
986 (const u32)ARRAY_SIZE(barts_golden_registers));
987 break;
988 case CHIP_TURKS:
989 radeon_program_register_sequence(rdev,
990 turks_golden_registers,
991 (const u32)ARRAY_SIZE(turks_golden_registers));
992 break;
993 case CHIP_CAICOS:
994 radeon_program_register_sequence(rdev,
995 caicos_golden_registers,
996 (const u32)ARRAY_SIZE(caicos_golden_registers));
997 break;
998 default:
999 break;
1000 }
1001}
1002
Jerome Glisse285484e2011-12-16 17:03:42 -05001003void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1004 unsigned *bankh, unsigned *mtaspect,
1005 unsigned *tile_split)
1006{
1007 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1008 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1009 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1010 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1011 switch (*bankw) {
1012 default:
1013 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1014 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1015 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1016 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1017 }
1018 switch (*bankh) {
1019 default:
1020 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1021 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1022 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1023 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1024 }
1025 switch (*mtaspect) {
1026 default:
1027 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1028 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1029 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1030 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1031 }
1032}
1033
Alex Deucher23d33ba2013-04-08 12:41:32 +02001034static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1035 u32 cntl_reg, u32 status_reg)
1036{
1037 int r, i;
1038 struct atom_clock_dividers dividers;
1039
1040 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1041 clock, false, &dividers);
1042 if (r)
1043 return r;
1044
1045 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1046
1047 for (i = 0; i < 100; i++) {
1048 if (RREG32(status_reg) & DCLK_STATUS)
1049 break;
1050 mdelay(10);
1051 }
1052 if (i == 100)
1053 return -ETIMEDOUT;
1054
1055 return 0;
1056}
1057
1058int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1059{
1060 int r = 0;
1061 u32 cg_scratch = RREG32(CG_SCRATCH1);
1062
1063 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1064 if (r)
1065 goto done;
1066 cg_scratch &= 0xffff0000;
1067 cg_scratch |= vclk / 100; /* Mhz */
1068
1069 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1070 if (r)
1071 goto done;
1072 cg_scratch &= 0x0000ffff;
1073 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1074
1075done:
1076 WREG32(CG_SCRATCH1, cg_scratch);
1077
1078 return r;
1079}
1080
Alex Deuchera8b49252013-04-08 12:41:33 +02001081int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1082{
1083 /* start off with something large */
Christian Königfacd1122013-04-29 11:55:02 +02001084 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
Alex Deuchera8b49252013-04-08 12:41:33 +02001085 int r;
1086
Christian König4ed10832013-04-18 15:25:58 +02001087 /* bypass vclk and dclk with bclk */
1088 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1089 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1090 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1091
1092 /* put PLL in bypass mode */
1093 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1094
1095 if (!vclk || !dclk) {
1096 /* keep the Bypass mode, put PLL to sleep */
1097 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1098 return 0;
1099 }
1100
Christian Königfacd1122013-04-29 11:55:02 +02001101 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1102 16384, 0x03FFFFFF, 0, 128, 5,
1103 &fb_div, &vclk_div, &dclk_div);
1104 if (r)
1105 return r;
Alex Deuchera8b49252013-04-08 12:41:33 +02001106
1107 /* set VCO_MODE to 1 */
1108 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1109
1110 /* toggle UPLL_SLEEP to 1 then back to 0 */
1111 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1112 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1113
1114 /* deassert UPLL_RESET */
1115 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1116
1117 mdelay(1);
1118
Christian Königfacd1122013-04-29 11:55:02 +02001119 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Alex Deuchera8b49252013-04-08 12:41:33 +02001120 if (r)
1121 return r;
1122
1123 /* assert UPLL_RESET again */
1124 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1125
1126 /* disable spread spectrum. */
1127 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1128
1129 /* set feedback divider */
Christian Königfacd1122013-04-29 11:55:02 +02001130 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
Alex Deuchera8b49252013-04-08 12:41:33 +02001131
1132 /* set ref divider to 0 */
1133 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1134
Christian Königfacd1122013-04-29 11:55:02 +02001135 if (fb_div < 307200)
Alex Deuchera8b49252013-04-08 12:41:33 +02001136 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1137 else
1138 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1139
1140 /* set PDIV_A and PDIV_B */
1141 WREG32_P(CG_UPLL_FUNC_CNTL_2,
Christian Königfacd1122013-04-29 11:55:02 +02001142 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
Alex Deuchera8b49252013-04-08 12:41:33 +02001143 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1144
1145 /* give the PLL some time to settle */
1146 mdelay(15);
1147
1148 /* deassert PLL_RESET */
1149 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1150
1151 mdelay(15);
1152
1153 /* switch from bypass mode to normal mode */
1154 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1155
Christian Königfacd1122013-04-29 11:55:02 +02001156 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Alex Deuchera8b49252013-04-08 12:41:33 +02001157 if (r)
1158 return r;
1159
1160 /* switch VCLK and DCLK selection */
1161 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1162 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1163 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1164
1165 mdelay(100);
1166
1167 return 0;
1168}
1169
Alex Deucherd054ac12011-09-01 17:46:15 +00001170void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1171{
1172 u16 ctl, v;
Jiang Liu32195ae2012-07-24 17:20:30 +08001173 int err;
Alex Deucherd054ac12011-09-01 17:46:15 +00001174
Jiang Liu32195ae2012-07-24 17:20:30 +08001175 err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
Alex Deucherd054ac12011-09-01 17:46:15 +00001176 if (err)
1177 return;
1178
1179 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1180
1181 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1182 * to avoid hangs or perfomance issues
1183 */
1184 if ((v == 0) || (v == 6) || (v == 7)) {
1185 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1186 ctl |= (2 << 12);
Jiang Liu32195ae2012-07-24 17:20:30 +08001187 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
Alex Deucherd054ac12011-09-01 17:46:15 +00001188 }
1189}
1190
Alex Deucher10257a62013-04-09 18:49:59 -04001191static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1192{
1193 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1194 return true;
1195 else
1196 return false;
1197}
1198
1199static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1200{
1201 u32 pos1, pos2;
1202
1203 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1204 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1205
1206 if (pos1 != pos2)
1207 return true;
1208 else
1209 return false;
1210}
1211
Alex Deucher377edc82012-07-17 14:02:42 -04001212/**
1213 * dce4_wait_for_vblank - vblank wait asic callback.
1214 *
1215 * @rdev: radeon_device pointer
1216 * @crtc: crtc to wait for vblank on
1217 *
1218 * Wait for vblank on the requested crtc (evergreen+).
1219 */
Alex Deucher3ae19b72012-02-23 17:53:37 -05001220void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1221{
Alex Deucher10257a62013-04-09 18:49:59 -04001222 unsigned i = 0;
Alex Deucher3ae19b72012-02-23 17:53:37 -05001223
Alex Deucher4a159032012-08-15 17:13:53 -04001224 if (crtc >= rdev->num_crtc)
1225 return;
1226
Alex Deucher10257a62013-04-09 18:49:59 -04001227 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1228 return;
1229
1230 /* depending on when we hit vblank, we may be close to active; if so,
1231 * wait for another frame.
1232 */
1233 while (dce4_is_in_vblank(rdev, crtc)) {
1234 if (i++ % 100 == 0) {
1235 if (!dce4_is_counter_moving(rdev, crtc))
Alex Deucher3ae19b72012-02-23 17:53:37 -05001236 break;
Alex Deucher3ae19b72012-02-23 17:53:37 -05001237 }
Alex Deucher10257a62013-04-09 18:49:59 -04001238 }
1239
1240 while (!dce4_is_in_vblank(rdev, crtc)) {
1241 if (i++ % 100 == 0) {
1242 if (!dce4_is_counter_moving(rdev, crtc))
Alex Deucher3ae19b72012-02-23 17:53:37 -05001243 break;
Alex Deucher3ae19b72012-02-23 17:53:37 -05001244 }
1245 }
1246}
1247
Alex Deucher377edc82012-07-17 14:02:42 -04001248/**
1249 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1250 *
1251 * @rdev: radeon_device pointer
1252 * @crtc: crtc to prepare for pageflip on
1253 *
1254 * Pre-pageflip callback (evergreen+).
1255 * Enables the pageflip irq (vblank irq).
1256 */
Alex Deucher6f34be52010-11-21 10:59:01 -05001257void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1258{
Alex Deucher6f34be52010-11-21 10:59:01 -05001259 /* enable the pflip int */
1260 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1261}
1262
Alex Deucher377edc82012-07-17 14:02:42 -04001263/**
1264 * evergreen_post_page_flip - pos-pageflip callback.
1265 *
1266 * @rdev: radeon_device pointer
1267 * @crtc: crtc to cleanup pageflip on
1268 *
1269 * Post-pageflip callback (evergreen+).
1270 * Disables the pageflip irq (vblank irq).
1271 */
Alex Deucher6f34be52010-11-21 10:59:01 -05001272void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1273{
1274 /* disable the pflip int */
1275 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1276}
1277
Alex Deucher377edc82012-07-17 14:02:42 -04001278/**
1279 * evergreen_page_flip - pageflip callback.
1280 *
1281 * @rdev: radeon_device pointer
1282 * @crtc_id: crtc to cleanup pageflip on
1283 * @crtc_base: new address of the crtc (GPU MC address)
1284 *
1285 * Does the actual pageflip (evergreen+).
1286 * During vblank we take the crtc lock and wait for the update_pending
1287 * bit to go high, when it does, we release the lock, and allow the
1288 * double buffered update to take place.
1289 * Returns the current update pending status.
1290 */
Alex Deucher6f34be52010-11-21 10:59:01 -05001291u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1292{
1293 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1294 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
Alex Deucherf6496472011-11-28 14:49:26 -05001295 int i;
Alex Deucher6f34be52010-11-21 10:59:01 -05001296
1297 /* Lock the graphics update lock */
1298 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1299 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1300
1301 /* update the scanout addresses */
1302 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1303 upper_32_bits(crtc_base));
1304 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1305 (u32)crtc_base);
1306
1307 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1308 upper_32_bits(crtc_base));
1309 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1310 (u32)crtc_base);
1311
1312 /* Wait for update_pending to go high. */
Alex Deucherf6496472011-11-28 14:49:26 -05001313 for (i = 0; i < rdev->usec_timeout; i++) {
1314 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1315 break;
1316 udelay(1);
1317 }
Alex Deucher6f34be52010-11-21 10:59:01 -05001318 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1319
1320 /* Unlock the lock, so double-buffering can take place inside vblank */
1321 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1322 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1323
1324 /* Return current update_pending status: */
1325 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1326}
1327
Alex Deucher21a81222010-07-02 12:58:16 -04001328/* get temperature in millidegrees */
Alex Deucher20d391d2011-02-01 16:12:34 -05001329int evergreen_get_temp(struct radeon_device *rdev)
Alex Deucher21a81222010-07-02 12:58:16 -04001330{
Alex Deucher1c88d742011-06-14 19:15:53 +00001331 u32 temp, toffset;
1332 int actual_temp = 0;
Alex Deucher21a81222010-07-02 12:58:16 -04001333
Alex Deucher67b3f822011-05-25 18:45:37 -04001334 if (rdev->family == CHIP_JUNIPER) {
1335 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1336 TOFFSET_SHIFT;
1337 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1338 TS0_ADC_DOUT_SHIFT;
Alex Deucher21a81222010-07-02 12:58:16 -04001339
Alex Deucher67b3f822011-05-25 18:45:37 -04001340 if (toffset & 0x100)
1341 actual_temp = temp / 2 - (0x200 - toffset);
1342 else
1343 actual_temp = temp / 2 + toffset;
1344
1345 actual_temp = actual_temp * 1000;
1346
1347 } else {
1348 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1349 ASIC_T_SHIFT;
1350
1351 if (temp & 0x400)
1352 actual_temp = -256;
1353 else if (temp & 0x200)
1354 actual_temp = 255;
1355 else if (temp & 0x100) {
1356 actual_temp = temp & 0x1ff;
1357 actual_temp |= ~0x1ff;
1358 } else
1359 actual_temp = temp & 0xff;
1360
1361 actual_temp = (actual_temp * 1000) / 2;
1362 }
1363
1364 return actual_temp;
Alex Deucher21a81222010-07-02 12:58:16 -04001365}
1366
Alex Deucher20d391d2011-02-01 16:12:34 -05001367int sumo_get_temp(struct radeon_device *rdev)
Alex Deuchere33df252010-11-22 17:56:32 -05001368{
1369 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
Alex Deucher20d391d2011-02-01 16:12:34 -05001370 int actual_temp = temp - 49;
Alex Deuchere33df252010-11-22 17:56:32 -05001371
1372 return actual_temp * 1000;
1373}
1374
Alex Deucher377edc82012-07-17 14:02:42 -04001375/**
1376 * sumo_pm_init_profile - Initialize power profiles callback.
1377 *
1378 * @rdev: radeon_device pointer
1379 *
1380 * Initialize the power states used in profile mode
1381 * (sumo, trinity, SI).
1382 * Used for profile mode only.
1383 */
Alex Deuchera4c9e2e2011-11-04 10:09:41 -04001384void sumo_pm_init_profile(struct radeon_device *rdev)
1385{
1386 int idx;
1387
1388 /* default */
1389 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1390 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1391 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1392 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1393
1394 /* low,mid sh/mh */
1395 if (rdev->flags & RADEON_IS_MOBILITY)
1396 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1397 else
1398 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1399
1400 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1401 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1402 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1403 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1404
1405 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1406 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1407 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1408 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1409
1410 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1411 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1412 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1413 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1414
1415 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1416 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1417 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1418 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1419
1420 /* high sh/mh */
1421 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1422 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1423 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1424 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1425 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1426 rdev->pm.power_state[idx].num_clock_modes - 1;
1427
1428 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1429 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1430 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1431 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1432 rdev->pm.power_state[idx].num_clock_modes - 1;
1433}
1434
Alex Deucher377edc82012-07-17 14:02:42 -04001435/**
Alex Deucher27810fb2012-10-01 19:25:11 -04001436 * btc_pm_init_profile - Initialize power profiles callback.
1437 *
1438 * @rdev: radeon_device pointer
1439 *
1440 * Initialize the power states used in profile mode
1441 * (BTC, cayman).
1442 * Used for profile mode only.
1443 */
1444void btc_pm_init_profile(struct radeon_device *rdev)
1445{
1446 int idx;
1447
1448 /* default */
1449 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1450 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1451 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1452 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1453 /* starting with BTC, there is one state that is used for both
1454 * MH and SH. Difference is that we always use the high clock index for
1455 * mclk.
1456 */
1457 if (rdev->flags & RADEON_IS_MOBILITY)
1458 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1459 else
1460 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1461 /* low sh */
1462 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1463 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1464 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1465 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1466 /* mid sh */
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1471 /* high sh */
1472 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1476 /* low mh */
1477 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1481 /* mid mh */
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1486 /* high mh */
1487 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1488 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1489 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1491}
1492
1493/**
Alex Deucher377edc82012-07-17 14:02:42 -04001494 * evergreen_pm_misc - set additional pm hw parameters callback.
1495 *
1496 * @rdev: radeon_device pointer
1497 *
1498 * Set non-clock parameters associated with a power state
1499 * (voltage, etc.) (evergreen+).
1500 */
Alex Deucher49e02b72010-04-23 17:57:27 -04001501void evergreen_pm_misc(struct radeon_device *rdev)
1502{
Rafał Miłeckia081a9d2010-06-07 18:20:25 -04001503 int req_ps_idx = rdev->pm.requested_power_state_index;
1504 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1505 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1506 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
Alex Deucher49e02b72010-04-23 17:57:27 -04001507
Alex Deucher2feea492011-04-12 14:49:24 -04001508 if (voltage->type == VOLTAGE_SW) {
Alex Deucherc6cf7772013-07-05 13:14:30 -04001509 /* 0xff0x are flags rather then an actual voltage */
1510 if ((voltage->voltage & 0xff00) == 0xff00)
Alex Deuchera377e182011-06-20 13:00:31 -04001511 return;
Alex Deucher2feea492011-04-12 14:49:24 -04001512 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
Alex Deucher8a83ec52011-04-12 14:49:23 -04001513 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
Alex Deucher4d601732010-06-07 18:15:18 -04001514 rdev->pm.current_vddc = voltage->voltage;
Alex Deucher2feea492011-04-12 14:49:24 -04001515 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1516 }
Alex Deucher7ae764b2013-02-11 08:44:48 -05001517
1518 /* starting with BTC, there is one state that is used for both
1519 * MH and SH. Difference is that we always use the high clock index for
1520 * mclk and vddci.
1521 */
1522 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1523 (rdev->family >= CHIP_BARTS) &&
1524 rdev->pm.active_crtc_count &&
1525 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1526 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1527 voltage = &rdev->pm.power_state[req_ps_idx].
1528 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1529
Alex Deucherc6cf7772013-07-05 13:14:30 -04001530 /* 0xff0x are flags rather then an actual voltage */
1531 if ((voltage->vddci & 0xff00) == 0xff00)
Alex Deuchera377e182011-06-20 13:00:31 -04001532 return;
Alex Deucher2feea492011-04-12 14:49:24 -04001533 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1534 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1535 rdev->pm.current_vddci = voltage->vddci;
1536 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
Alex Deucher4d601732010-06-07 18:15:18 -04001537 }
1538 }
Alex Deucher49e02b72010-04-23 17:57:27 -04001539}
1540
Alex Deucher377edc82012-07-17 14:02:42 -04001541/**
1542 * evergreen_pm_prepare - pre-power state change callback.
1543 *
1544 * @rdev: radeon_device pointer
1545 *
1546 * Prepare for a power state change (evergreen+).
1547 */
Alex Deucher49e02b72010-04-23 17:57:27 -04001548void evergreen_pm_prepare(struct radeon_device *rdev)
1549{
1550 struct drm_device *ddev = rdev->ddev;
1551 struct drm_crtc *crtc;
1552 struct radeon_crtc *radeon_crtc;
1553 u32 tmp;
1554
1555 /* disable any active CRTCs */
1556 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1557 radeon_crtc = to_radeon_crtc(crtc);
1558 if (radeon_crtc->enabled) {
1559 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1560 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1561 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1562 }
1563 }
1564}
1565
Alex Deucher377edc82012-07-17 14:02:42 -04001566/**
1567 * evergreen_pm_finish - post-power state change callback.
1568 *
1569 * @rdev: radeon_device pointer
1570 *
1571 * Clean up after a power state change (evergreen+).
1572 */
Alex Deucher49e02b72010-04-23 17:57:27 -04001573void evergreen_pm_finish(struct radeon_device *rdev)
1574{
1575 struct drm_device *ddev = rdev->ddev;
1576 struct drm_crtc *crtc;
1577 struct radeon_crtc *radeon_crtc;
1578 u32 tmp;
1579
1580 /* enable any active CRTCs */
1581 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1582 radeon_crtc = to_radeon_crtc(crtc);
1583 if (radeon_crtc->enabled) {
1584 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1585 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1586 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1587 }
1588 }
1589}
1590
Alex Deucher377edc82012-07-17 14:02:42 -04001591/**
1592 * evergreen_hpd_sense - hpd sense callback.
1593 *
1594 * @rdev: radeon_device pointer
1595 * @hpd: hpd (hotplug detect) pin
1596 *
1597 * Checks if a digital monitor is connected (evergreen+).
1598 * Returns true if connected, false if not connected.
1599 */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001600bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1601{
1602 bool connected = false;
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001603
1604 switch (hpd) {
1605 case RADEON_HPD_1:
1606 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1607 connected = true;
1608 break;
1609 case RADEON_HPD_2:
1610 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1611 connected = true;
1612 break;
1613 case RADEON_HPD_3:
1614 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1615 connected = true;
1616 break;
1617 case RADEON_HPD_4:
1618 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1619 connected = true;
1620 break;
1621 case RADEON_HPD_5:
1622 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1623 connected = true;
1624 break;
1625 case RADEON_HPD_6:
1626 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1627 connected = true;
1628 break;
1629 default:
1630 break;
1631 }
1632
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001633 return connected;
1634}
1635
Alex Deucher377edc82012-07-17 14:02:42 -04001636/**
1637 * evergreen_hpd_set_polarity - hpd set polarity callback.
1638 *
1639 * @rdev: radeon_device pointer
1640 * @hpd: hpd (hotplug detect) pin
1641 *
1642 * Set the polarity of the hpd pin (evergreen+).
1643 */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001644void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1645 enum radeon_hpd_id hpd)
1646{
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001647 u32 tmp;
1648 bool connected = evergreen_hpd_sense(rdev, hpd);
1649
1650 switch (hpd) {
1651 case RADEON_HPD_1:
1652 tmp = RREG32(DC_HPD1_INT_CONTROL);
1653 if (connected)
1654 tmp &= ~DC_HPDx_INT_POLARITY;
1655 else
1656 tmp |= DC_HPDx_INT_POLARITY;
1657 WREG32(DC_HPD1_INT_CONTROL, tmp);
1658 break;
1659 case RADEON_HPD_2:
1660 tmp = RREG32(DC_HPD2_INT_CONTROL);
1661 if (connected)
1662 tmp &= ~DC_HPDx_INT_POLARITY;
1663 else
1664 tmp |= DC_HPDx_INT_POLARITY;
1665 WREG32(DC_HPD2_INT_CONTROL, tmp);
1666 break;
1667 case RADEON_HPD_3:
1668 tmp = RREG32(DC_HPD3_INT_CONTROL);
1669 if (connected)
1670 tmp &= ~DC_HPDx_INT_POLARITY;
1671 else
1672 tmp |= DC_HPDx_INT_POLARITY;
1673 WREG32(DC_HPD3_INT_CONTROL, tmp);
1674 break;
1675 case RADEON_HPD_4:
1676 tmp = RREG32(DC_HPD4_INT_CONTROL);
1677 if (connected)
1678 tmp &= ~DC_HPDx_INT_POLARITY;
1679 else
1680 tmp |= DC_HPDx_INT_POLARITY;
1681 WREG32(DC_HPD4_INT_CONTROL, tmp);
1682 break;
1683 case RADEON_HPD_5:
1684 tmp = RREG32(DC_HPD5_INT_CONTROL);
1685 if (connected)
1686 tmp &= ~DC_HPDx_INT_POLARITY;
1687 else
1688 tmp |= DC_HPDx_INT_POLARITY;
1689 WREG32(DC_HPD5_INT_CONTROL, tmp);
1690 break;
1691 case RADEON_HPD_6:
1692 tmp = RREG32(DC_HPD6_INT_CONTROL);
1693 if (connected)
1694 tmp &= ~DC_HPDx_INT_POLARITY;
1695 else
1696 tmp |= DC_HPDx_INT_POLARITY;
1697 WREG32(DC_HPD6_INT_CONTROL, tmp);
1698 break;
1699 default:
1700 break;
1701 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001702}
1703
Alex Deucher377edc82012-07-17 14:02:42 -04001704/**
1705 * evergreen_hpd_init - hpd setup callback.
1706 *
1707 * @rdev: radeon_device pointer
1708 *
1709 * Setup the hpd pins used by the card (evergreen+).
1710 * Enable the pin, set the polarity, and enable the hpd interrupts.
1711 */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001712void evergreen_hpd_init(struct radeon_device *rdev)
1713{
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001714 struct drm_device *dev = rdev->ddev;
1715 struct drm_connector *connector;
Christian Koenigfb982572012-05-17 01:33:30 +02001716 unsigned enabled = 0;
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001717 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1718 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001719
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001720 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1721 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher2e97be72013-04-11 12:45:34 -04001722
1723 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1724 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1725 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1726 * aux dp channel on imac and help (but not completely fix)
1727 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1728 * also avoid interrupt storms during dpms.
1729 */
1730 continue;
1731 }
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001732 switch (radeon_connector->hpd.hpd) {
1733 case RADEON_HPD_1:
1734 WREG32(DC_HPD1_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001735 break;
1736 case RADEON_HPD_2:
1737 WREG32(DC_HPD2_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001738 break;
1739 case RADEON_HPD_3:
1740 WREG32(DC_HPD3_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001741 break;
1742 case RADEON_HPD_4:
1743 WREG32(DC_HPD4_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001744 break;
1745 case RADEON_HPD_5:
1746 WREG32(DC_HPD5_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001747 break;
1748 case RADEON_HPD_6:
1749 WREG32(DC_HPD6_CONTROL, tmp);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001750 break;
1751 default:
1752 break;
1753 }
Alex Deucher64912e92011-11-03 11:21:39 -04001754 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
Christian Koenigfb982572012-05-17 01:33:30 +02001755 enabled |= 1 << radeon_connector->hpd.hpd;
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001756 }
Christian Koenigfb982572012-05-17 01:33:30 +02001757 radeon_irq_kms_enable_hpd(rdev, enabled);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001758}
1759
Alex Deucher377edc82012-07-17 14:02:42 -04001760/**
1761 * evergreen_hpd_fini - hpd tear down callback.
1762 *
1763 * @rdev: radeon_device pointer
1764 *
1765 * Tear down the hpd pins used by the card (evergreen+).
1766 * Disable the hpd interrupts.
1767 */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05001768void evergreen_hpd_fini(struct radeon_device *rdev)
1769{
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001770 struct drm_device *dev = rdev->ddev;
1771 struct drm_connector *connector;
Christian Koenigfb982572012-05-17 01:33:30 +02001772 unsigned disabled = 0;
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001773
1774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1775 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1776 switch (radeon_connector->hpd.hpd) {
1777 case RADEON_HPD_1:
1778 WREG32(DC_HPD1_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001779 break;
1780 case RADEON_HPD_2:
1781 WREG32(DC_HPD2_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001782 break;
1783 case RADEON_HPD_3:
1784 WREG32(DC_HPD3_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001785 break;
1786 case RADEON_HPD_4:
1787 WREG32(DC_HPD4_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001788 break;
1789 case RADEON_HPD_5:
1790 WREG32(DC_HPD5_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001791 break;
1792 case RADEON_HPD_6:
1793 WREG32(DC_HPD6_CONTROL, 0);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001794 break;
1795 default:
1796 break;
1797 }
Christian Koenigfb982572012-05-17 01:33:30 +02001798 disabled |= 1 << radeon_connector->hpd.hpd;
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001799 }
Christian Koenigfb982572012-05-17 01:33:30 +02001800 radeon_irq_kms_disable_hpd(rdev, disabled);
Alex Deucher0ca2ab52010-02-26 13:57:45 -05001801}
1802
Alex Deucherf9d9c362010-10-22 02:51:05 -04001803/* watermark setup */
1804
1805static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806 struct radeon_crtc *radeon_crtc,
1807 struct drm_display_mode *mode,
1808 struct drm_display_mode *other_mode)
1809{
Alex Deucher12dfc842011-04-14 19:07:34 -04001810 u32 tmp;
Alex Deucherf9d9c362010-10-22 02:51:05 -04001811 /*
1812 * Line Buffer Setup
1813 * There are 3 line buffers, each one shared by 2 display controllers.
1814 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1815 * the display controllers. The paritioning is done via one of four
1816 * preset allocations specified in bits 2:0:
1817 * first display controller
1818 * 0 - first half of lb (3840 * 2)
1819 * 1 - first 3/4 of lb (5760 * 2)
Alex Deucher12dfc842011-04-14 19:07:34 -04001820 * 2 - whole lb (7680 * 2), other crtc must be disabled
Alex Deucherf9d9c362010-10-22 02:51:05 -04001821 * 3 - first 1/4 of lb (1920 * 2)
1822 * second display controller
1823 * 4 - second half of lb (3840 * 2)
1824 * 5 - second 3/4 of lb (5760 * 2)
Alex Deucher12dfc842011-04-14 19:07:34 -04001825 * 6 - whole lb (7680 * 2), other crtc must be disabled
Alex Deucherf9d9c362010-10-22 02:51:05 -04001826 * 7 - last 1/4 of lb (1920 * 2)
1827 */
Alex Deucher12dfc842011-04-14 19:07:34 -04001828 /* this can get tricky if we have two large displays on a paired group
1829 * of crtcs. Ideally for multiple large displays we'd assign them to
1830 * non-linked crtcs for maximum line buffer allocation.
1831 */
1832 if (radeon_crtc->base.enabled && mode) {
1833 if (other_mode)
Alex Deucherf9d9c362010-10-22 02:51:05 -04001834 tmp = 0; /* 1/2 */
Alex Deucher12dfc842011-04-14 19:07:34 -04001835 else
1836 tmp = 2; /* whole */
1837 } else
1838 tmp = 0;
Alex Deucherf9d9c362010-10-22 02:51:05 -04001839
1840 /* second controller of the pair uses second half of the lb */
1841 if (radeon_crtc->crtc_id % 2)
1842 tmp += 4;
1843 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1844
Alex Deucher12dfc842011-04-14 19:07:34 -04001845 if (radeon_crtc->base.enabled && mode) {
1846 switch (tmp) {
1847 case 0:
1848 case 4:
1849 default:
1850 if (ASIC_IS_DCE5(rdev))
1851 return 4096 * 2;
1852 else
1853 return 3840 * 2;
1854 case 1:
1855 case 5:
1856 if (ASIC_IS_DCE5(rdev))
1857 return 6144 * 2;
1858 else
1859 return 5760 * 2;
1860 case 2:
1861 case 6:
1862 if (ASIC_IS_DCE5(rdev))
1863 return 8192 * 2;
1864 else
1865 return 7680 * 2;
1866 case 3:
1867 case 7:
1868 if (ASIC_IS_DCE5(rdev))
1869 return 2048 * 2;
1870 else
1871 return 1920 * 2;
1872 }
Alex Deucherf9d9c362010-10-22 02:51:05 -04001873 }
Alex Deucher12dfc842011-04-14 19:07:34 -04001874
1875 /* controller not enabled, so no lb used */
1876 return 0;
Alex Deucherf9d9c362010-10-22 02:51:05 -04001877}
1878
Alex Deucherca7db222012-03-20 17:18:30 -04001879u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
Alex Deucherf9d9c362010-10-22 02:51:05 -04001880{
1881 u32 tmp = RREG32(MC_SHARED_CHMAP);
1882
1883 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1884 case 0:
1885 default:
1886 return 1;
1887 case 1:
1888 return 2;
1889 case 2:
1890 return 4;
1891 case 3:
1892 return 8;
1893 }
1894}
1895
1896struct evergreen_wm_params {
1897 u32 dram_channels; /* number of dram channels */
1898 u32 yclk; /* bandwidth per dram data pin in kHz */
1899 u32 sclk; /* engine clock in kHz */
1900 u32 disp_clk; /* display clock in kHz */
1901 u32 src_width; /* viewport width */
1902 u32 active_time; /* active display time in ns */
1903 u32 blank_time; /* blank time in ns */
1904 bool interlaced; /* mode is interlaced */
1905 fixed20_12 vsc; /* vertical scale ratio */
1906 u32 num_heads; /* number of active crtcs */
1907 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1908 u32 lb_size; /* line buffer allocated to pipe */
1909 u32 vtaps; /* vertical scaler taps */
1910};
1911
1912static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1913{
1914 /* Calculate DRAM Bandwidth and the part allocated to display. */
1915 fixed20_12 dram_efficiency; /* 0.7 */
1916 fixed20_12 yclk, dram_channels, bandwidth;
1917 fixed20_12 a;
1918
1919 a.full = dfixed_const(1000);
1920 yclk.full = dfixed_const(wm->yclk);
1921 yclk.full = dfixed_div(yclk, a);
1922 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1923 a.full = dfixed_const(10);
1924 dram_efficiency.full = dfixed_const(7);
1925 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1926 bandwidth.full = dfixed_mul(dram_channels, yclk);
1927 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1928
1929 return dfixed_trunc(bandwidth);
1930}
1931
1932static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1933{
1934 /* Calculate DRAM Bandwidth and the part allocated to display. */
1935 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1936 fixed20_12 yclk, dram_channels, bandwidth;
1937 fixed20_12 a;
1938
1939 a.full = dfixed_const(1000);
1940 yclk.full = dfixed_const(wm->yclk);
1941 yclk.full = dfixed_div(yclk, a);
1942 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1943 a.full = dfixed_const(10);
1944 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1945 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1946 bandwidth.full = dfixed_mul(dram_channels, yclk);
1947 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1948
1949 return dfixed_trunc(bandwidth);
1950}
1951
1952static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1953{
1954 /* Calculate the display Data return Bandwidth */
1955 fixed20_12 return_efficiency; /* 0.8 */
1956 fixed20_12 sclk, bandwidth;
1957 fixed20_12 a;
1958
1959 a.full = dfixed_const(1000);
1960 sclk.full = dfixed_const(wm->sclk);
1961 sclk.full = dfixed_div(sclk, a);
1962 a.full = dfixed_const(10);
1963 return_efficiency.full = dfixed_const(8);
1964 return_efficiency.full = dfixed_div(return_efficiency, a);
1965 a.full = dfixed_const(32);
1966 bandwidth.full = dfixed_mul(a, sclk);
1967 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1968
1969 return dfixed_trunc(bandwidth);
1970}
1971
1972static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1973{
1974 /* Calculate the DMIF Request Bandwidth */
1975 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1976 fixed20_12 disp_clk, bandwidth;
1977 fixed20_12 a;
1978
1979 a.full = dfixed_const(1000);
1980 disp_clk.full = dfixed_const(wm->disp_clk);
1981 disp_clk.full = dfixed_div(disp_clk, a);
1982 a.full = dfixed_const(10);
1983 disp_clk_request_efficiency.full = dfixed_const(8);
1984 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1985 a.full = dfixed_const(32);
1986 bandwidth.full = dfixed_mul(a, disp_clk);
1987 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1988
1989 return dfixed_trunc(bandwidth);
1990}
1991
1992static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1993{
1994 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1995 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1996 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1997 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1998
1999 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2000}
2001
2002static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2003{
2004 /* Calculate the display mode Average Bandwidth
2005 * DisplayMode should contain the source and destination dimensions,
2006 * timing, etc.
2007 */
2008 fixed20_12 bpp;
2009 fixed20_12 line_time;
2010 fixed20_12 src_width;
2011 fixed20_12 bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2016 line_time.full = dfixed_div(line_time, a);
2017 bpp.full = dfixed_const(wm->bytes_per_pixel);
2018 src_width.full = dfixed_const(wm->src_width);
2019 bandwidth.full = dfixed_mul(src_width, bpp);
2020 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2021 bandwidth.full = dfixed_div(bandwidth, line_time);
2022
2023 return dfixed_trunc(bandwidth);
2024}
2025
2026static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2027{
2028 /* First calcualte the latency in ns */
2029 u32 mc_latency = 2000; /* 2000 ns. */
2030 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2031 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2032 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2033 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2034 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2035 (wm->num_heads * cursor_line_pair_return_time);
2036 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2037 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2038 fixed20_12 a, b, c;
2039
2040 if (wm->num_heads == 0)
2041 return 0;
2042
2043 a.full = dfixed_const(2);
2044 b.full = dfixed_const(1);
2045 if ((wm->vsc.full > a.full) ||
2046 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2047 (wm->vtaps >= 5) ||
2048 ((wm->vsc.full >= a.full) && wm->interlaced))
2049 max_src_lines_per_dst_line = 4;
2050 else
2051 max_src_lines_per_dst_line = 2;
2052
2053 a.full = dfixed_const(available_bandwidth);
2054 b.full = dfixed_const(wm->num_heads);
2055 a.full = dfixed_div(a, b);
2056
2057 b.full = dfixed_const(1000);
2058 c.full = dfixed_const(wm->disp_clk);
2059 b.full = dfixed_div(c, b);
2060 c.full = dfixed_const(wm->bytes_per_pixel);
2061 b.full = dfixed_mul(b, c);
2062
2063 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2064
2065 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2066 b.full = dfixed_const(1000);
2067 c.full = dfixed_const(lb_fill_bw);
2068 b.full = dfixed_div(c, b);
2069 a.full = dfixed_div(a, b);
2070 line_fill_time = dfixed_trunc(a);
2071
2072 if (line_fill_time < wm->active_time)
2073 return latency;
2074 else
2075 return latency + (line_fill_time - wm->active_time);
2076
2077}
2078
2079static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2080{
2081 if (evergreen_average_bandwidth(wm) <=
2082 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2083 return true;
2084 else
2085 return false;
2086};
2087
2088static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2089{
2090 if (evergreen_average_bandwidth(wm) <=
2091 (evergreen_available_bandwidth(wm) / wm->num_heads))
2092 return true;
2093 else
2094 return false;
2095};
2096
2097static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2098{
2099 u32 lb_partitions = wm->lb_size / wm->src_width;
2100 u32 line_time = wm->active_time + wm->blank_time;
2101 u32 latency_tolerant_lines;
2102 u32 latency_hiding;
2103 fixed20_12 a;
2104
2105 a.full = dfixed_const(1);
2106 if (wm->vsc.full > a.full)
2107 latency_tolerant_lines = 1;
2108 else {
2109 if (lb_partitions <= (wm->vtaps + 1))
2110 latency_tolerant_lines = 1;
2111 else
2112 latency_tolerant_lines = 2;
2113 }
2114
2115 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2116
2117 if (evergreen_latency_watermark(wm) <= latency_hiding)
2118 return true;
2119 else
2120 return false;
2121}
2122
2123static void evergreen_program_watermarks(struct radeon_device *rdev,
2124 struct radeon_crtc *radeon_crtc,
2125 u32 lb_size, u32 num_heads)
2126{
2127 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002128 struct evergreen_wm_params wm_low, wm_high;
2129 u32 dram_channels;
Alex Deucherf9d9c362010-10-22 02:51:05 -04002130 u32 pixel_period;
2131 u32 line_time = 0;
2132 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2133 u32 priority_a_mark = 0, priority_b_mark = 0;
2134 u32 priority_a_cnt = PRIORITY_OFF;
2135 u32 priority_b_cnt = PRIORITY_OFF;
2136 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2137 u32 tmp, arb_control3;
2138 fixed20_12 a, b, c;
2139
2140 if (radeon_crtc->base.enabled && num_heads && mode) {
2141 pixel_period = 1000000 / (u32)mode->clock;
2142 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2143 priority_a_cnt = 0;
2144 priority_b_cnt = 0;
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002145 dram_channels = evergreen_get_number_of_dram_channels(rdev);
Alex Deucherf9d9c362010-10-22 02:51:05 -04002146
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002147 /* watermark for high clocks */
2148 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2149 wm_high.yclk =
2150 radeon_dpm_get_mclk(rdev, false) * 10;
2151 wm_high.sclk =
2152 radeon_dpm_get_sclk(rdev, false) * 10;
2153 } else {
2154 wm_high.yclk = rdev->pm.current_mclk * 10;
2155 wm_high.sclk = rdev->pm.current_sclk * 10;
2156 }
2157
2158 wm_high.disp_clk = mode->clock;
2159 wm_high.src_width = mode->crtc_hdisplay;
2160 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2161 wm_high.blank_time = line_time - wm_high.active_time;
2162 wm_high.interlaced = false;
Alex Deucherf9d9c362010-10-22 02:51:05 -04002163 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002164 wm_high.interlaced = true;
2165 wm_high.vsc = radeon_crtc->vsc;
2166 wm_high.vtaps = 1;
Alex Deucherf9d9c362010-10-22 02:51:05 -04002167 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002168 wm_high.vtaps = 2;
2169 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2170 wm_high.lb_size = lb_size;
2171 wm_high.dram_channels = dram_channels;
2172 wm_high.num_heads = num_heads;
2173
2174 /* watermark for low clocks */
2175 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2176 wm_low.yclk =
2177 radeon_dpm_get_mclk(rdev, true) * 10;
2178 wm_low.sclk =
2179 radeon_dpm_get_sclk(rdev, true) * 10;
2180 } else {
2181 wm_low.yclk = rdev->pm.current_mclk * 10;
2182 wm_low.sclk = rdev->pm.current_sclk * 10;
2183 }
2184
2185 wm_low.disp_clk = mode->clock;
2186 wm_low.src_width = mode->crtc_hdisplay;
2187 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2188 wm_low.blank_time = line_time - wm_low.active_time;
2189 wm_low.interlaced = false;
2190 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2191 wm_low.interlaced = true;
2192 wm_low.vsc = radeon_crtc->vsc;
2193 wm_low.vtaps = 1;
2194 if (radeon_crtc->rmx_type != RMX_OFF)
2195 wm_low.vtaps = 2;
2196 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197 wm_low.lb_size = lb_size;
2198 wm_low.dram_channels = dram_channels;
2199 wm_low.num_heads = num_heads;
Alex Deucherf9d9c362010-10-22 02:51:05 -04002200
2201 /* set for high clocks */
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002202 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
Alex Deucherf9d9c362010-10-22 02:51:05 -04002203 /* set for low clocks */
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002204 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
Alex Deucherf9d9c362010-10-22 02:51:05 -04002205
2206 /* possibly force display priority to high */
2207 /* should really do this at mode validation time... */
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002208 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2209 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2210 !evergreen_check_latency_hiding(&wm_high) ||
Alex Deucherf9d9c362010-10-22 02:51:05 -04002211 (rdev->disp_priority == 2)) {
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002212 DRM_DEBUG_KMS("force priority a to high\n");
Alex Deucherf9d9c362010-10-22 02:51:05 -04002213 priority_a_cnt |= PRIORITY_ALWAYS_ON;
Alex Deuchercf0cfdd2012-03-13 16:25:11 -04002214 }
2215 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2216 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2217 !evergreen_check_latency_hiding(&wm_low) ||
2218 (rdev->disp_priority == 2)) {
2219 DRM_DEBUG_KMS("force priority b to high\n");
Alex Deucherf9d9c362010-10-22 02:51:05 -04002220 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2221 }
2222
2223 a.full = dfixed_const(1000);
2224 b.full = dfixed_const(mode->clock);
2225 b.full = dfixed_div(b, a);
2226 c.full = dfixed_const(latency_watermark_a);
2227 c.full = dfixed_mul(c, b);
2228 c.full = dfixed_mul(c, radeon_crtc->hsc);
2229 c.full = dfixed_div(c, a);
2230 a.full = dfixed_const(16);
2231 c.full = dfixed_div(c, a);
2232 priority_a_mark = dfixed_trunc(c);
2233 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2234
2235 a.full = dfixed_const(1000);
2236 b.full = dfixed_const(mode->clock);
2237 b.full = dfixed_div(b, a);
2238 c.full = dfixed_const(latency_watermark_b);
2239 c.full = dfixed_mul(c, b);
2240 c.full = dfixed_mul(c, radeon_crtc->hsc);
2241 c.full = dfixed_div(c, a);
2242 a.full = dfixed_const(16);
2243 c.full = dfixed_div(c, a);
2244 priority_b_mark = dfixed_trunc(c);
2245 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2246 }
2247
2248 /* select wm A */
2249 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2250 tmp = arb_control3;
2251 tmp &= ~LATENCY_WATERMARK_MASK(3);
2252 tmp |= LATENCY_WATERMARK_MASK(1);
2253 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2254 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2255 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2256 LATENCY_HIGH_WATERMARK(line_time)));
2257 /* select wm B */
2258 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2259 tmp &= ~LATENCY_WATERMARK_MASK(3);
2260 tmp |= LATENCY_WATERMARK_MASK(2);
2261 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2262 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2263 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2264 LATENCY_HIGH_WATERMARK(line_time)));
2265 /* restore original selection */
2266 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2267
2268 /* write the priority marks */
2269 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2270 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2271
Alex Deucher7178d2a2013-03-21 10:38:49 -04002272 /* save values for DPM */
2273 radeon_crtc->line_time = line_time;
2274 radeon_crtc->wm_high = latency_watermark_a;
2275 radeon_crtc->wm_low = latency_watermark_b;
Alex Deucherf9d9c362010-10-22 02:51:05 -04002276}
2277
Alex Deucher377edc82012-07-17 14:02:42 -04002278/**
2279 * evergreen_bandwidth_update - update display watermarks callback.
2280 *
2281 * @rdev: radeon_device pointer
2282 *
2283 * Update the display watermarks based on the requested mode(s)
2284 * (evergreen+).
2285 */
Alex Deucher0ca2ab52010-02-26 13:57:45 -05002286void evergreen_bandwidth_update(struct radeon_device *rdev)
2287{
Alex Deucherf9d9c362010-10-22 02:51:05 -04002288 struct drm_display_mode *mode0 = NULL;
2289 struct drm_display_mode *mode1 = NULL;
2290 u32 num_heads = 0, lb_size;
2291 int i;
2292
2293 radeon_update_display_priority(rdev);
2294
2295 for (i = 0; i < rdev->num_crtc; i++) {
2296 if (rdev->mode_info.crtcs[i]->base.enabled)
2297 num_heads++;
2298 }
2299 for (i = 0; i < rdev->num_crtc; i += 2) {
2300 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2301 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2302 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2303 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2304 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2305 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2306 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002307}
2308
Alex Deucher377edc82012-07-17 14:02:42 -04002309/**
2310 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2311 *
2312 * @rdev: radeon_device pointer
2313 *
2314 * Wait for the MC (memory controller) to be idle.
2315 * (evergreen+).
2316 * Returns 0 if the MC is idle, -1 if not.
2317 */
Alex Deucherb9952a82011-03-02 20:07:33 -05002318int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002319{
2320 unsigned i;
2321 u32 tmp;
2322
2323 for (i = 0; i < rdev->usec_timeout; i++) {
2324 /* read MC_STATUS */
2325 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2326 if (!tmp)
2327 return 0;
2328 udelay(1);
2329 }
2330 return -1;
2331}
2332
2333/*
2334 * GART
2335 */
Alex Deucher0fcdb612010-03-24 13:20:41 -04002336void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2337{
2338 unsigned i;
2339 u32 tmp;
2340
Alex Deucher6f2f48a2010-12-15 11:01:56 -05002341 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2342
Alex Deucher0fcdb612010-03-24 13:20:41 -04002343 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2344 for (i = 0; i < rdev->usec_timeout; i++) {
2345 /* read MC_STATUS */
2346 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2347 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2348 if (tmp == 2) {
2349 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2350 return;
2351 }
2352 if (tmp) {
2353 return;
2354 }
2355 udelay(1);
2356 }
2357}
2358
Lauri Kasanen1109ca02012-08-31 13:43:50 -04002359static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002360{
2361 u32 tmp;
Alex Deucher0fcdb612010-03-24 13:20:41 -04002362 int r;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002363
Jerome Glissec9a1be92011-11-03 11:16:49 -04002364 if (rdev->gart.robj == NULL) {
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002365 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2366 return -EINVAL;
2367 }
2368 r = radeon_gart_table_vram_pin(rdev);
2369 if (r)
2370 return r;
Dave Airlie82568562010-02-05 16:00:07 +10002371 radeon_gart_restore(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002372 /* Setup L2 cache */
2373 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2374 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2375 EFFECTIVE_L2_QUEUE_SIZE(7));
2376 WREG32(VM_L2_CNTL2, 0);
2377 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2378 /* Setup TLB control */
2379 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2380 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2381 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2382 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
Alex Deucher8aeb96f2011-05-03 19:28:02 -04002383 if (rdev->flags & RADEON_IS_IGP) {
2384 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2385 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2386 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2387 } else {
2388 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2389 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2390 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
Alex Deucher0b8c30b2012-05-31 18:54:43 -04002391 if ((rdev->family == CHIP_JUNIPER) ||
2392 (rdev->family == CHIP_CYPRESS) ||
2393 (rdev->family == CHIP_HEMLOCK) ||
2394 (rdev->family == CHIP_BARTS))
2395 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
Alex Deucher8aeb96f2011-05-03 19:28:02 -04002396 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002397 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2398 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2399 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2400 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2401 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2402 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2403 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2404 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2405 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2406 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2407 (u32)(rdev->dummy_page.addr >> 12));
Alex Deucher0fcdb612010-03-24 13:20:41 -04002408 WREG32(VM_CONTEXT1_CNTL, 0);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002409
Alex Deucher0fcdb612010-03-24 13:20:41 -04002410 evergreen_pcie_gart_tlb_flush(rdev);
Tormod Voldenfcf4de52011-08-31 21:54:07 +00002411 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2412 (unsigned)(rdev->mc.gtt_size >> 20),
2413 (unsigned long long)rdev->gart.table_addr);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002414 rdev->gart.ready = true;
2415 return 0;
2416}
2417
Lauri Kasanen1109ca02012-08-31 13:43:50 -04002418static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002419{
2420 u32 tmp;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002421
2422 /* Disable all tables */
Alex Deucher0fcdb612010-03-24 13:20:41 -04002423 WREG32(VM_CONTEXT0_CNTL, 0);
2424 WREG32(VM_CONTEXT1_CNTL, 0);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002425
2426 /* Setup L2 cache */
2427 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2428 EFFECTIVE_L2_QUEUE_SIZE(7));
2429 WREG32(VM_L2_CNTL2, 0);
2430 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2431 /* Setup TLB control */
2432 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2433 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2434 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2435 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2439 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
Jerome Glissec9a1be92011-11-03 11:16:49 -04002440 radeon_gart_table_vram_unpin(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002441}
2442
Lauri Kasanen1109ca02012-08-31 13:43:50 -04002443static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002444{
2445 evergreen_pcie_gart_disable(rdev);
2446 radeon_gart_table_vram_free(rdev);
2447 radeon_gart_fini(rdev);
2448}
2449
2450
Lauri Kasanen1109ca02012-08-31 13:43:50 -04002451static void evergreen_agp_enable(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002452{
2453 u32 tmp;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002454
2455 /* Setup L2 cache */
2456 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2457 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2458 EFFECTIVE_L2_QUEUE_SIZE(7));
2459 WREG32(VM_L2_CNTL2, 0);
2460 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2461 /* Setup TLB control */
2462 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2463 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2464 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2465 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2466 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
Alex Deucher0fcdb612010-03-24 13:20:41 -04002473 WREG32(VM_CONTEXT0_CNTL, 0);
2474 WREG32(VM_CONTEXT1_CNTL, 0);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002475}
2476
Alex Deucherb9952a82011-03-02 20:07:33 -05002477void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002478{
Alex Deucher62444b72012-08-15 17:18:42 -04002479 u32 crtc_enabled, tmp, frame_count, blackout;
2480 int i, j;
2481
Alex Deucher51535502012-08-30 14:34:30 -04002482 if (!ASIC_IS_NODCE(rdev)) {
2483 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2484 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002485
Alex Deucher51535502012-08-30 14:34:30 -04002486 /* disable VGA render */
2487 WREG32(VGA_RENDER_CONTROL, 0);
2488 }
Alex Deucher62444b72012-08-15 17:18:42 -04002489 /* blank the display controllers */
2490 for (i = 0; i < rdev->num_crtc; i++) {
2491 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2492 if (crtc_enabled) {
2493 save->crtc_enabled[i] = true;
2494 if (ASIC_IS_DCE6(rdev)) {
2495 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2496 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2497 radeon_wait_for_vblank(rdev, i);
Alex Deucherabf14572013-04-10 19:08:14 -04002498 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
Alex Deucher62444b72012-08-15 17:18:42 -04002499 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2500 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2501 }
2502 } else {
2503 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2504 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2505 radeon_wait_for_vblank(rdev, i);
Alex Deucherabf14572013-04-10 19:08:14 -04002506 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
Alex Deucher62444b72012-08-15 17:18:42 -04002507 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2508 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
Alex Deucherabf14572013-04-10 19:08:14 -04002509 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
Alex Deucher62444b72012-08-15 17:18:42 -04002510 }
2511 }
2512 /* wait for the next frame */
2513 frame_count = radeon_get_vblank_counter(rdev, i);
2514 for (j = 0; j < rdev->usec_timeout; j++) {
2515 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2516 break;
2517 udelay(1);
2518 }
Alex Deucherabf14572013-04-10 19:08:14 -04002519
2520 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2521 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2523 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2524 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2525 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2526 save->crtc_enabled[i] = false;
2527 /* ***** */
Alex Deucher804cc4a02012-11-19 09:11:27 -05002528 } else {
2529 save->crtc_enabled[i] = false;
Alex Deucher62444b72012-08-15 17:18:42 -04002530 }
Alex Deucher18007402010-11-22 17:56:28 -05002531 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002532
Alex Deucher62444b72012-08-15 17:18:42 -04002533 radeon_mc_wait_for_idle(rdev);
2534
2535 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2536 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2537 /* Block CPU access */
2538 WREG32(BIF_FB_EN, 0);
2539 /* blackout the MC */
2540 blackout &= ~BLACKOUT_MODE_MASK;
2541 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
Alex Deucherb7eff392011-07-08 11:44:56 -04002542 }
Alex Deuchered39fad2013-01-31 09:00:52 -05002543 /* wait for the MC to settle */
2544 udelay(100);
Alex Deucher968c0162013-04-10 09:58:42 -04002545
2546 /* lock double buffered regs */
2547 for (i = 0; i < rdev->num_crtc; i++) {
2548 if (save->crtc_enabled[i]) {
2549 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2550 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2551 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2552 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2553 }
2554 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2555 if (!(tmp & 1)) {
2556 tmp |= 1;
2557 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2558 }
2559 }
2560 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002561}
2562
Alex Deucherb9952a82011-03-02 20:07:33 -05002563void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002564{
Alex Deucher62444b72012-08-15 17:18:42 -04002565 u32 tmp, frame_count;
2566 int i, j;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002567
Alex Deucher62444b72012-08-15 17:18:42 -04002568 /* update crtc base addresses */
2569 for (i = 0; i < rdev->num_crtc; i++) {
2570 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
Alex Deucher18007402010-11-22 17:56:28 -05002571 upper_32_bits(rdev->mc.vram_start));
Alex Deucher62444b72012-08-15 17:18:42 -04002572 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
Alex Deucher18007402010-11-22 17:56:28 -05002573 upper_32_bits(rdev->mc.vram_start));
Alex Deucher62444b72012-08-15 17:18:42 -04002574 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
Alex Deucher18007402010-11-22 17:56:28 -05002575 (u32)rdev->mc.vram_start);
Alex Deucher62444b72012-08-15 17:18:42 -04002576 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
Alex Deucher18007402010-11-22 17:56:28 -05002577 (u32)rdev->mc.vram_start);
Alex Deucherb7eff392011-07-08 11:44:56 -04002578 }
Alex Deucher51535502012-08-30 14:34:30 -04002579
2580 if (!ASIC_IS_NODCE(rdev)) {
2581 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2582 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2583 }
Alex Deucher62444b72012-08-15 17:18:42 -04002584
Alex Deucher968c0162013-04-10 09:58:42 -04002585 /* unlock regs and wait for update */
2586 for (i = 0; i < rdev->num_crtc; i++) {
2587 if (save->crtc_enabled[i]) {
2588 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2589 if ((tmp & 0x3) != 0) {
2590 tmp &= ~0x3;
2591 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2592 }
2593 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2594 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2595 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2596 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2597 }
2598 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2599 if (tmp & 1) {
2600 tmp &= ~1;
2601 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2602 }
2603 for (j = 0; j < rdev->usec_timeout; j++) {
2604 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2605 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2606 break;
2607 udelay(1);
2608 }
2609 }
2610 }
2611
Alex Deucher62444b72012-08-15 17:18:42 -04002612 /* unblackout the MC */
2613 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2614 tmp &= ~BLACKOUT_MODE_MASK;
2615 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2616 /* allow CPU access */
2617 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2618
2619 for (i = 0; i < rdev->num_crtc; i++) {
Alex Deucher695ddeb2012-11-05 16:34:58 +00002620 if (save->crtc_enabled[i]) {
Alex Deucher62444b72012-08-15 17:18:42 -04002621 if (ASIC_IS_DCE6(rdev)) {
2622 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2623 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
Christopher Staitebb5888202013-01-26 11:10:58 -05002624 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
Alex Deucher62444b72012-08-15 17:18:42 -04002625 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
Christopher Staitebb5888202013-01-26 11:10:58 -05002626 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
Alex Deucher62444b72012-08-15 17:18:42 -04002627 } else {
2628 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2629 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
Christopher Staitebb5888202013-01-26 11:10:58 -05002630 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
Alex Deucher62444b72012-08-15 17:18:42 -04002631 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
Christopher Staitebb5888202013-01-26 11:10:58 -05002632 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
Alex Deucher62444b72012-08-15 17:18:42 -04002633 }
2634 /* wait for the next frame */
2635 frame_count = radeon_get_vblank_counter(rdev, i);
2636 for (j = 0; j < rdev->usec_timeout; j++) {
2637 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2638 break;
2639 udelay(1);
2640 }
2641 }
2642 }
Alex Deucher51535502012-08-30 14:34:30 -04002643 if (!ASIC_IS_NODCE(rdev)) {
2644 /* Unlock vga access */
2645 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2646 mdelay(1);
2647 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2648 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002649}
2650
Alex Deucher755d8192011-03-02 20:07:34 -05002651void evergreen_mc_program(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002652{
2653 struct evergreen_mc_save save;
2654 u32 tmp;
2655 int i, j;
2656
2657 /* Initialize HDP */
2658 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2659 WREG32((0x2c14 + j), 0x00000000);
2660 WREG32((0x2c18 + j), 0x00000000);
2661 WREG32((0x2c1c + j), 0x00000000);
2662 WREG32((0x2c20 + j), 0x00000000);
2663 WREG32((0x2c24 + j), 0x00000000);
2664 }
2665 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2666
2667 evergreen_mc_stop(rdev, &save);
2668 if (evergreen_mc_wait_for_idle(rdev)) {
2669 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2670 }
2671 /* Lockout access through VGA aperture*/
2672 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2673 /* Update configuration */
2674 if (rdev->flags & RADEON_IS_AGP) {
2675 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2676 /* VRAM before AGP */
2677 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2678 rdev->mc.vram_start >> 12);
2679 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2680 rdev->mc.gtt_end >> 12);
2681 } else {
2682 /* VRAM after AGP */
2683 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2684 rdev->mc.gtt_start >> 12);
2685 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2686 rdev->mc.vram_end >> 12);
2687 }
2688 } else {
2689 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2690 rdev->mc.vram_start >> 12);
2691 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2692 rdev->mc.vram_end >> 12);
2693 }
Alex Deucher3b9832f2011-11-10 08:59:39 -05002694 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
Alex Deucher05b3ef62012-03-20 17:18:37 -04002695 /* llano/ontario only */
2696 if ((rdev->family == CHIP_PALM) ||
2697 (rdev->family == CHIP_SUMO) ||
2698 (rdev->family == CHIP_SUMO2)) {
Alex Deucherb4183e32010-12-15 11:04:10 -05002699 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2700 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2701 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2702 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2703 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002704 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2705 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2706 WREG32(MC_VM_FB_LOCATION, tmp);
2707 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
Alex Deucherc46cb4d2011-01-06 19:12:37 -05002708 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
Jerome Glisse46fcd2b2010-06-03 19:34:48 +02002709 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002710 if (rdev->flags & RADEON_IS_AGP) {
2711 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2712 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2713 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2714 } else {
2715 WREG32(MC_VM_AGP_BASE, 0);
2716 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2717 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2718 }
2719 if (evergreen_mc_wait_for_idle(rdev)) {
2720 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2721 }
2722 evergreen_mc_resume(rdev, &save);
2723 /* we need to own VRAM, so turn off the VGA renderer here
2724 * to stop it overwriting our objects */
2725 rv515_vga_render_disable(rdev);
2726}
2727
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002728/*
2729 * CP.
2730 */
Alex Deucher12920592011-02-02 12:37:40 -05002731void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2732{
Christian König876dc9f2012-05-08 14:24:01 +02002733 struct radeon_ring *ring = &rdev->ring[ib->ring];
Alex Deucher89d35802012-07-17 14:02:31 -04002734 u32 next_rptr;
Christian König7b1f2482011-09-23 15:11:23 +02002735
Alex Deucher12920592011-02-02 12:37:40 -05002736 /* set to DX10/11 mode */
Christian Könige32eb502011-10-23 12:56:27 +02002737 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2738 radeon_ring_write(ring, 1);
Christian König45df6802012-07-06 16:22:55 +02002739
2740 if (ring->rptr_save_reg) {
Alex Deucher89d35802012-07-17 14:02:31 -04002741 next_rptr = ring->wptr + 3 + 4;
Christian König45df6802012-07-06 16:22:55 +02002742 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2743 radeon_ring_write(ring, ((ring->rptr_save_reg -
2744 PACKET3_SET_CONFIG_REG_START) >> 2));
2745 radeon_ring_write(ring, next_rptr);
Alex Deucher89d35802012-07-17 14:02:31 -04002746 } else if (rdev->wb.enabled) {
2747 next_rptr = ring->wptr + 5 + 4;
2748 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2749 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2750 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2751 radeon_ring_write(ring, next_rptr);
2752 radeon_ring_write(ring, 0);
Christian König45df6802012-07-06 16:22:55 +02002753 }
2754
Christian Könige32eb502011-10-23 12:56:27 +02002755 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2756 radeon_ring_write(ring,
Alex Deucher0f234f5f2011-02-13 19:06:33 -05002757#ifdef __BIG_ENDIAN
2758 (2 << 0) |
2759#endif
2760 (ib->gpu_addr & 0xFFFFFFFC));
Christian Könige32eb502011-10-23 12:56:27 +02002761 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2762 radeon_ring_write(ring, ib->length_dw);
Alex Deucher12920592011-02-02 12:37:40 -05002763}
2764
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002765
2766static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2767{
Alex Deucherfe251e22010-03-24 13:36:43 -04002768 const __be32 *fw_data;
2769 int i;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002770
Alex Deucherfe251e22010-03-24 13:36:43 -04002771 if (!rdev->me_fw || !rdev->pfp_fw)
2772 return -EINVAL;
2773
2774 r700_cp_stop(rdev);
Alex Deucher0f234f5f2011-02-13 19:06:33 -05002775 WREG32(CP_RB_CNTL,
2776#ifdef __BIG_ENDIAN
2777 BUF_SWAP_32BIT |
2778#endif
2779 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
Alex Deucherfe251e22010-03-24 13:36:43 -04002780
2781 fw_data = (const __be32 *)rdev->pfp_fw->data;
2782 WREG32(CP_PFP_UCODE_ADDR, 0);
2783 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2784 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2785 WREG32(CP_PFP_UCODE_ADDR, 0);
2786
2787 fw_data = (const __be32 *)rdev->me_fw->data;
2788 WREG32(CP_ME_RAM_WADDR, 0);
2789 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2790 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2791
2792 WREG32(CP_PFP_UCODE_ADDR, 0);
2793 WREG32(CP_ME_RAM_WADDR, 0);
2794 WREG32(CP_ME_RAM_RADDR, 0);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002795 return 0;
2796}
2797
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002798static int evergreen_cp_start(struct radeon_device *rdev)
2799{
Christian Könige32eb502011-10-23 12:56:27 +02002800 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Alex Deucher2281a372010-10-21 13:31:38 -04002801 int r, i;
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002802 uint32_t cp_me;
2803
Christian Könige32eb502011-10-23 12:56:27 +02002804 r = radeon_ring_lock(rdev, ring, 7);
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002805 if (r) {
2806 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2807 return r;
2808 }
Christian Könige32eb502011-10-23 12:56:27 +02002809 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2810 radeon_ring_write(ring, 0x1);
2811 radeon_ring_write(ring, 0x0);
2812 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2813 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2814 radeon_ring_write(ring, 0);
2815 radeon_ring_write(ring, 0);
2816 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002817
2818 cp_me = 0xff;
2819 WREG32(CP_ME_CNTL, cp_me);
2820
Christian Könige32eb502011-10-23 12:56:27 +02002821 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002822 if (r) {
2823 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2824 return r;
2825 }
Alex Deucher2281a372010-10-21 13:31:38 -04002826
2827 /* setup clear context state */
Christian Könige32eb502011-10-23 12:56:27 +02002828 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2829 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deucher2281a372010-10-21 13:31:38 -04002830
2831 for (i = 0; i < evergreen_default_size; i++)
Christian Könige32eb502011-10-23 12:56:27 +02002832 radeon_ring_write(ring, evergreen_default_state[i]);
Alex Deucher2281a372010-10-21 13:31:38 -04002833
Christian Könige32eb502011-10-23 12:56:27 +02002834 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2835 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deucher2281a372010-10-21 13:31:38 -04002836
2837 /* set clear context state */
Christian Könige32eb502011-10-23 12:56:27 +02002838 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2839 radeon_ring_write(ring, 0);
Alex Deucher2281a372010-10-21 13:31:38 -04002840
2841 /* SQ_VTX_BASE_VTX_LOC */
Christian Könige32eb502011-10-23 12:56:27 +02002842 radeon_ring_write(ring, 0xc0026f00);
2843 radeon_ring_write(ring, 0x00000000);
2844 radeon_ring_write(ring, 0x00000000);
2845 radeon_ring_write(ring, 0x00000000);
Alex Deucher2281a372010-10-21 13:31:38 -04002846
2847 /* Clear consts */
Christian Könige32eb502011-10-23 12:56:27 +02002848 radeon_ring_write(ring, 0xc0036f00);
2849 radeon_ring_write(ring, 0x00000bc4);
2850 radeon_ring_write(ring, 0xffffffff);
2851 radeon_ring_write(ring, 0xffffffff);
2852 radeon_ring_write(ring, 0xffffffff);
Alex Deucher2281a372010-10-21 13:31:38 -04002853
Christian Könige32eb502011-10-23 12:56:27 +02002854 radeon_ring_write(ring, 0xc0026900);
2855 radeon_ring_write(ring, 0x00000316);
2856 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2857 radeon_ring_write(ring, 0x00000010); /* */
Alex Deucher18ff84d2011-02-02 12:37:41 -05002858
Christian Könige32eb502011-10-23 12:56:27 +02002859 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002860
2861 return 0;
2862}
2863
Lauri Kasanen1109ca02012-08-31 13:43:50 -04002864static int evergreen_cp_resume(struct radeon_device *rdev)
Alex Deucherfe251e22010-03-24 13:36:43 -04002865{
Christian Könige32eb502011-10-23 12:56:27 +02002866 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Alex Deucherfe251e22010-03-24 13:36:43 -04002867 u32 tmp;
2868 u32 rb_bufsz;
2869 int r;
2870
2871 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2872 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2873 SOFT_RESET_PA |
2874 SOFT_RESET_SH |
2875 SOFT_RESET_VGT |
Jerome Glissea49a50d2011-08-24 20:00:17 +00002876 SOFT_RESET_SPI |
Alex Deucherfe251e22010-03-24 13:36:43 -04002877 SOFT_RESET_SX));
2878 RREG32(GRBM_SOFT_RESET);
2879 mdelay(15);
2880 WREG32(GRBM_SOFT_RESET, 0);
2881 RREG32(GRBM_SOFT_RESET);
2882
2883 /* Set ring buffer size */
Christian Könige32eb502011-10-23 12:56:27 +02002884 rb_bufsz = drm_order(ring->ring_size / 8);
Alex Deucher724c80e2010-08-27 18:25:25 -04002885 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucherfe251e22010-03-24 13:36:43 -04002886#ifdef __BIG_ENDIAN
2887 tmp |= BUF_SWAP_32BIT;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04002888#endif
Alex Deucherfe251e22010-03-24 13:36:43 -04002889 WREG32(CP_RB_CNTL, tmp);
Christian König15d33322011-09-15 19:02:22 +02002890 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher11ef3f1f2012-01-20 14:47:43 -05002891 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucherfe251e22010-03-24 13:36:43 -04002892
2893 /* Set the write pointer delay */
2894 WREG32(CP_RB_WPTR_DELAY, 0);
2895
2896 /* Initialize the ring buffer's read and write pointers */
2897 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2898 WREG32(CP_RB_RPTR_WR, 0);
Christian Könige32eb502011-10-23 12:56:27 +02002899 ring->wptr = 0;
2900 WREG32(CP_RB_WPTR, ring->wptr);
Alex Deucher724c80e2010-08-27 18:25:25 -04002901
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04002902 /* set the wb address whether it's enabled or not */
Alex Deucher0f234f5f2011-02-13 19:06:33 -05002903 WREG32(CP_RB_RPTR_ADDR,
Alex Deucher0f234f5f2011-02-13 19:06:33 -05002904 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
Alex Deucher724c80e2010-08-27 18:25:25 -04002905 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2906 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2907
2908 if (rdev->wb.enabled)
2909 WREG32(SCRATCH_UMSK, 0xff);
2910 else {
2911 tmp |= RB_NO_UPDATE;
2912 WREG32(SCRATCH_UMSK, 0);
2913 }
2914
Alex Deucherfe251e22010-03-24 13:36:43 -04002915 mdelay(1);
2916 WREG32(CP_RB_CNTL, tmp);
2917
Christian Könige32eb502011-10-23 12:56:27 +02002918 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
Alex Deucherfe251e22010-03-24 13:36:43 -04002919 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2920
Christian Könige32eb502011-10-23 12:56:27 +02002921 ring->rptr = RREG32(CP_RB_RPTR);
Alex Deucherfe251e22010-03-24 13:36:43 -04002922
Alex Deucher7e7b41d2010-09-02 21:32:32 -04002923 evergreen_cp_start(rdev);
Christian Könige32eb502011-10-23 12:56:27 +02002924 ring->ready = true;
Alex Deucherf7128122012-02-23 17:53:45 -05002925 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
Alex Deucherfe251e22010-03-24 13:36:43 -04002926 if (r) {
Christian Könige32eb502011-10-23 12:56:27 +02002927 ring->ready = false;
Alex Deucherfe251e22010-03-24 13:36:43 -04002928 return r;
2929 }
2930 return 0;
2931}
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002932
2933/*
2934 * Core functions
2935 */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05002936static void evergreen_gpu_init(struct radeon_device *rdev)
2937{
Alex Deucher416a2bd2012-05-31 19:00:25 -04002938 u32 gb_addr_config;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04002939 u32 mc_shared_chmap, mc_arb_ramcfg;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04002940 u32 sx_debug_1;
2941 u32 smx_dc_ctl0;
2942 u32 sq_config;
2943 u32 sq_lds_resource_mgmt;
2944 u32 sq_gpr_resource_mgmt_1;
2945 u32 sq_gpr_resource_mgmt_2;
2946 u32 sq_gpr_resource_mgmt_3;
2947 u32 sq_thread_resource_mgmt;
2948 u32 sq_thread_resource_mgmt_2;
2949 u32 sq_stack_resource_mgmt_1;
2950 u32 sq_stack_resource_mgmt_2;
2951 u32 sq_stack_resource_mgmt_3;
2952 u32 vgt_cache_invalidation;
Alex Deucherf25a5c62011-05-19 11:07:57 -04002953 u32 hdp_host_path_cntl, tmp;
Alex Deucher416a2bd2012-05-31 19:00:25 -04002954 u32 disabled_rb_mask;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04002955 int i, j, num_shader_engines, ps_thread_count;
2956
2957 switch (rdev->family) {
2958 case CHIP_CYPRESS:
2959 case CHIP_HEMLOCK:
2960 rdev->config.evergreen.num_ses = 2;
2961 rdev->config.evergreen.max_pipes = 4;
2962 rdev->config.evergreen.max_tile_pipes = 8;
2963 rdev->config.evergreen.max_simds = 10;
2964 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2965 rdev->config.evergreen.max_gprs = 256;
2966 rdev->config.evergreen.max_threads = 248;
2967 rdev->config.evergreen.max_gs_threads = 32;
2968 rdev->config.evergreen.max_stack_entries = 512;
2969 rdev->config.evergreen.sx_num_of_sets = 4;
2970 rdev->config.evergreen.sx_max_export_size = 256;
2971 rdev->config.evergreen.sx_max_export_pos_size = 64;
2972 rdev->config.evergreen.sx_max_export_smx_size = 192;
2973 rdev->config.evergreen.max_hw_contexts = 8;
2974 rdev->config.evergreen.sq_num_cf_insts = 2;
2975
2976 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2977 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04002979 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04002980 break;
2981 case CHIP_JUNIPER:
2982 rdev->config.evergreen.num_ses = 1;
2983 rdev->config.evergreen.max_pipes = 4;
2984 rdev->config.evergreen.max_tile_pipes = 4;
2985 rdev->config.evergreen.max_simds = 10;
2986 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2987 rdev->config.evergreen.max_gprs = 256;
2988 rdev->config.evergreen.max_threads = 248;
2989 rdev->config.evergreen.max_gs_threads = 32;
2990 rdev->config.evergreen.max_stack_entries = 512;
2991 rdev->config.evergreen.sx_num_of_sets = 4;
2992 rdev->config.evergreen.sx_max_export_size = 256;
2993 rdev->config.evergreen.sx_max_export_pos_size = 64;
2994 rdev->config.evergreen.sx_max_export_smx_size = 192;
2995 rdev->config.evergreen.max_hw_contexts = 8;
2996 rdev->config.evergreen.sq_num_cf_insts = 2;
2997
2998 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2999 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3000 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003001 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003002 break;
3003 case CHIP_REDWOOD:
3004 rdev->config.evergreen.num_ses = 1;
3005 rdev->config.evergreen.max_pipes = 4;
3006 rdev->config.evergreen.max_tile_pipes = 4;
3007 rdev->config.evergreen.max_simds = 5;
3008 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3009 rdev->config.evergreen.max_gprs = 256;
3010 rdev->config.evergreen.max_threads = 248;
3011 rdev->config.evergreen.max_gs_threads = 32;
3012 rdev->config.evergreen.max_stack_entries = 256;
3013 rdev->config.evergreen.sx_num_of_sets = 4;
3014 rdev->config.evergreen.sx_max_export_size = 256;
3015 rdev->config.evergreen.sx_max_export_pos_size = 64;
3016 rdev->config.evergreen.sx_max_export_smx_size = 192;
3017 rdev->config.evergreen.max_hw_contexts = 8;
3018 rdev->config.evergreen.sq_num_cf_insts = 2;
3019
3020 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3021 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3022 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003023 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003024 break;
3025 case CHIP_CEDAR:
3026 default:
3027 rdev->config.evergreen.num_ses = 1;
3028 rdev->config.evergreen.max_pipes = 2;
3029 rdev->config.evergreen.max_tile_pipes = 2;
3030 rdev->config.evergreen.max_simds = 2;
3031 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3032 rdev->config.evergreen.max_gprs = 256;
3033 rdev->config.evergreen.max_threads = 192;
3034 rdev->config.evergreen.max_gs_threads = 16;
3035 rdev->config.evergreen.max_stack_entries = 256;
3036 rdev->config.evergreen.sx_num_of_sets = 4;
3037 rdev->config.evergreen.sx_max_export_size = 128;
3038 rdev->config.evergreen.sx_max_export_pos_size = 32;
3039 rdev->config.evergreen.sx_max_export_smx_size = 96;
3040 rdev->config.evergreen.max_hw_contexts = 4;
3041 rdev->config.evergreen.sq_num_cf_insts = 1;
3042
3043 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3044 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3045 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003046 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003047 break;
Alex Deucherd5e455e2010-11-22 17:56:29 -05003048 case CHIP_PALM:
3049 rdev->config.evergreen.num_ses = 1;
3050 rdev->config.evergreen.max_pipes = 2;
3051 rdev->config.evergreen.max_tile_pipes = 2;
3052 rdev->config.evergreen.max_simds = 2;
3053 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054 rdev->config.evergreen.max_gprs = 256;
3055 rdev->config.evergreen.max_threads = 192;
3056 rdev->config.evergreen.max_gs_threads = 16;
3057 rdev->config.evergreen.max_stack_entries = 256;
3058 rdev->config.evergreen.sx_num_of_sets = 4;
3059 rdev->config.evergreen.sx_max_export_size = 128;
3060 rdev->config.evergreen.sx_max_export_pos_size = 32;
3061 rdev->config.evergreen.sx_max_export_smx_size = 96;
3062 rdev->config.evergreen.max_hw_contexts = 4;
3063 rdev->config.evergreen.sq_num_cf_insts = 1;
3064
3065 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003068 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
Alex Deucherd5e455e2010-11-22 17:56:29 -05003069 break;
Alex Deucherd5c5a722011-05-31 15:42:48 -04003070 case CHIP_SUMO:
3071 rdev->config.evergreen.num_ses = 1;
3072 rdev->config.evergreen.max_pipes = 4;
Jerome Glissebd25f072012-12-11 11:56:52 -05003073 rdev->config.evergreen.max_tile_pipes = 4;
Alex Deucherd5c5a722011-05-31 15:42:48 -04003074 if (rdev->pdev->device == 0x9648)
3075 rdev->config.evergreen.max_simds = 3;
3076 else if ((rdev->pdev->device == 0x9647) ||
3077 (rdev->pdev->device == 0x964a))
3078 rdev->config.evergreen.max_simds = 4;
3079 else
3080 rdev->config.evergreen.max_simds = 5;
3081 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3082 rdev->config.evergreen.max_gprs = 256;
3083 rdev->config.evergreen.max_threads = 248;
3084 rdev->config.evergreen.max_gs_threads = 32;
3085 rdev->config.evergreen.max_stack_entries = 256;
3086 rdev->config.evergreen.sx_num_of_sets = 4;
3087 rdev->config.evergreen.sx_max_export_size = 256;
3088 rdev->config.evergreen.sx_max_export_pos_size = 64;
3089 rdev->config.evergreen.sx_max_export_smx_size = 192;
3090 rdev->config.evergreen.max_hw_contexts = 8;
3091 rdev->config.evergreen.sq_num_cf_insts = 2;
3092
3093 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3094 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3095 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Jerome Glissebd25f072012-12-11 11:56:52 -05003096 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
Alex Deucherd5c5a722011-05-31 15:42:48 -04003097 break;
3098 case CHIP_SUMO2:
3099 rdev->config.evergreen.num_ses = 1;
3100 rdev->config.evergreen.max_pipes = 4;
3101 rdev->config.evergreen.max_tile_pipes = 4;
3102 rdev->config.evergreen.max_simds = 2;
3103 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3104 rdev->config.evergreen.max_gprs = 256;
3105 rdev->config.evergreen.max_threads = 248;
3106 rdev->config.evergreen.max_gs_threads = 32;
3107 rdev->config.evergreen.max_stack_entries = 512;
3108 rdev->config.evergreen.sx_num_of_sets = 4;
3109 rdev->config.evergreen.sx_max_export_size = 256;
3110 rdev->config.evergreen.sx_max_export_pos_size = 64;
3111 rdev->config.evergreen.sx_max_export_smx_size = 192;
3112 rdev->config.evergreen.max_hw_contexts = 8;
3113 rdev->config.evergreen.sq_num_cf_insts = 2;
3114
3115 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Jerome Glissebd25f072012-12-11 11:56:52 -05003118 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
Alex Deucherd5c5a722011-05-31 15:42:48 -04003119 break;
Alex Deucheradb68fa2011-01-06 21:19:24 -05003120 case CHIP_BARTS:
3121 rdev->config.evergreen.num_ses = 2;
3122 rdev->config.evergreen.max_pipes = 4;
3123 rdev->config.evergreen.max_tile_pipes = 8;
3124 rdev->config.evergreen.max_simds = 7;
3125 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3126 rdev->config.evergreen.max_gprs = 256;
3127 rdev->config.evergreen.max_threads = 248;
3128 rdev->config.evergreen.max_gs_threads = 32;
3129 rdev->config.evergreen.max_stack_entries = 512;
3130 rdev->config.evergreen.sx_num_of_sets = 4;
3131 rdev->config.evergreen.sx_max_export_size = 256;
3132 rdev->config.evergreen.sx_max_export_pos_size = 64;
3133 rdev->config.evergreen.sx_max_export_smx_size = 192;
3134 rdev->config.evergreen.max_hw_contexts = 8;
3135 rdev->config.evergreen.sq_num_cf_insts = 2;
3136
3137 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3138 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003140 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
Alex Deucheradb68fa2011-01-06 21:19:24 -05003141 break;
3142 case CHIP_TURKS:
3143 rdev->config.evergreen.num_ses = 1;
3144 rdev->config.evergreen.max_pipes = 4;
3145 rdev->config.evergreen.max_tile_pipes = 4;
3146 rdev->config.evergreen.max_simds = 6;
3147 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3148 rdev->config.evergreen.max_gprs = 256;
3149 rdev->config.evergreen.max_threads = 248;
3150 rdev->config.evergreen.max_gs_threads = 32;
3151 rdev->config.evergreen.max_stack_entries = 256;
3152 rdev->config.evergreen.sx_num_of_sets = 4;
3153 rdev->config.evergreen.sx_max_export_size = 256;
3154 rdev->config.evergreen.sx_max_export_pos_size = 64;
3155 rdev->config.evergreen.sx_max_export_smx_size = 192;
3156 rdev->config.evergreen.max_hw_contexts = 8;
3157 rdev->config.evergreen.sq_num_cf_insts = 2;
3158
3159 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3160 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3161 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003162 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
Alex Deucheradb68fa2011-01-06 21:19:24 -05003163 break;
3164 case CHIP_CAICOS:
3165 rdev->config.evergreen.num_ses = 1;
Jerome Glissebd25f072012-12-11 11:56:52 -05003166 rdev->config.evergreen.max_pipes = 2;
Alex Deucheradb68fa2011-01-06 21:19:24 -05003167 rdev->config.evergreen.max_tile_pipes = 2;
3168 rdev->config.evergreen.max_simds = 2;
3169 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3170 rdev->config.evergreen.max_gprs = 256;
3171 rdev->config.evergreen.max_threads = 192;
3172 rdev->config.evergreen.max_gs_threads = 16;
3173 rdev->config.evergreen.max_stack_entries = 256;
3174 rdev->config.evergreen.sx_num_of_sets = 4;
3175 rdev->config.evergreen.sx_max_export_size = 128;
3176 rdev->config.evergreen.sx_max_export_pos_size = 32;
3177 rdev->config.evergreen.sx_max_export_smx_size = 96;
3178 rdev->config.evergreen.max_hw_contexts = 4;
3179 rdev->config.evergreen.sq_num_cf_insts = 1;
3180
3181 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3182 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3183 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher416a2bd2012-05-31 19:00:25 -04003184 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
Alex Deucheradb68fa2011-01-06 21:19:24 -05003185 break;
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003186 }
3187
3188 /* Initialize HDP */
3189 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3190 WREG32((0x2c14 + j), 0x00000000);
3191 WREG32((0x2c18 + j), 0x00000000);
3192 WREG32((0x2c1c + j), 0x00000000);
3193 WREG32((0x2c20 + j), 0x00000000);
3194 WREG32((0x2c24 + j), 0x00000000);
3195 }
3196
3197 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3198
Alex Deucherd054ac12011-09-01 17:46:15 +00003199 evergreen_fix_pci_max_read_req_size(rdev);
3200
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003201 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
Alex Deucher05b3ef62012-03-20 17:18:37 -04003202 if ((rdev->family == CHIP_PALM) ||
3203 (rdev->family == CHIP_SUMO) ||
3204 (rdev->family == CHIP_SUMO2))
Alex Deucherd9282fc2011-05-11 03:15:24 -04003205 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3206 else
3207 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003208
Alex Deucher1aa52bd2010-11-17 12:11:03 -05003209 /* setup tiling info dword. gb_addr_config is not adequate since it does
3210 * not have bank info, so create a custom tiling dword.
3211 * bits 3:0 num_pipes
3212 * bits 7:4 num_banks
3213 * bits 11:8 group_size
3214 * bits 15:12 row_size
3215 */
3216 rdev->config.evergreen.tile_config = 0;
3217 switch (rdev->config.evergreen.max_tile_pipes) {
3218 case 1:
3219 default:
3220 rdev->config.evergreen.tile_config |= (0 << 0);
3221 break;
3222 case 2:
3223 rdev->config.evergreen.tile_config |= (1 << 0);
3224 break;
3225 case 4:
3226 rdev->config.evergreen.tile_config |= (2 << 0);
3227 break;
3228 case 8:
3229 rdev->config.evergreen.tile_config |= (3 << 0);
3230 break;
3231 }
Alex Deucherd698a342011-06-23 00:49:29 -04003232 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
Alex Deucher5bfa4872011-05-20 12:35:22 -04003233 if (rdev->flags & RADEON_IS_IGP)
Alex Deucherd698a342011-06-23 00:49:29 -04003234 rdev->config.evergreen.tile_config |= 1 << 4;
Alex Deucher29d65402012-05-31 18:53:36 -04003235 else {
Alex Deucherc8d15ed2012-07-31 11:01:10 -04003236 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3237 case 0: /* four banks */
Alex Deucher29d65402012-05-31 18:53:36 -04003238 rdev->config.evergreen.tile_config |= 0 << 4;
Alex Deucherc8d15ed2012-07-31 11:01:10 -04003239 break;
3240 case 1: /* eight banks */
3241 rdev->config.evergreen.tile_config |= 1 << 4;
3242 break;
3243 case 2: /* sixteen banks */
3244 default:
3245 rdev->config.evergreen.tile_config |= 2 << 4;
3246 break;
3247 }
Alex Deucher29d65402012-05-31 18:53:36 -04003248 }
Alex Deucher416a2bd2012-05-31 19:00:25 -04003249 rdev->config.evergreen.tile_config |= 0 << 8;
Alex Deucher1aa52bd2010-11-17 12:11:03 -05003250 rdev->config.evergreen.tile_config |=
3251 ((gb_addr_config & 0x30000000) >> 28) << 12;
3252
Alex Deucher416a2bd2012-05-31 19:00:25 -04003253 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3254
3255 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3256 u32 efuse_straps_4;
3257 u32 efuse_straps_3;
3258
Alex Deucherff82bbc2013-04-12 11:27:20 -04003259 efuse_straps_4 = RREG32_RCU(0x204);
3260 efuse_straps_3 = RREG32_RCU(0x203);
Alex Deucher416a2bd2012-05-31 19:00:25 -04003261 tmp = (((efuse_straps_4 & 0xf) << 4) |
3262 ((efuse_straps_3 & 0xf0000000) >> 28));
3263 } else {
3264 tmp = 0;
3265 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3266 u32 rb_disable_bitmap;
3267
3268 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3269 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3270 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3271 tmp <<= 4;
3272 tmp |= rb_disable_bitmap;
3273 }
3274 }
3275 /* enabled rb are just the one not disabled :) */
3276 disabled_rb_mask = tmp;
Alex Deuchercedb6552013-04-09 10:13:22 -04003277 tmp = 0;
3278 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3279 tmp |= (1 << i);
3280 /* if all the backends are disabled, fix it up here */
3281 if ((disabled_rb_mask & tmp) == tmp) {
3282 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3283 disabled_rb_mask &= ~(1 << i);
3284 }
Alex Deucher416a2bd2012-05-31 19:00:25 -04003285
3286 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3287 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3288
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003289 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3290 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3291 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
Alex Deucher233d1ad2012-12-04 15:25:59 -05003292 WREG32(DMA_TILING_CONFIG, gb_addr_config);
Christian König9a210592013-04-08 12:41:37 +02003293 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3294 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3295 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003296
Alex Deucherf7eb9732013-01-30 13:57:40 -05003297 if ((rdev->config.evergreen.max_backends == 1) &&
3298 (rdev->flags & RADEON_IS_IGP)) {
3299 if ((disabled_rb_mask & 3) == 1) {
3300 /* RB0 disabled, RB1 enabled */
3301 tmp = 0x11111111;
3302 } else {
3303 /* RB1 disabled, RB0 enabled */
3304 tmp = 0x00000000;
3305 }
3306 } else {
3307 tmp = gb_addr_config & NUM_PIPES_MASK;
3308 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3309 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3310 }
Alex Deucher416a2bd2012-05-31 19:00:25 -04003311 WREG32(GB_BACKEND_MAP, tmp);
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003312
3313 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3314 WREG32(CGTS_TCC_DISABLE, 0);
3315 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3316 WREG32(CGTS_USER_TCC_DISABLE, 0);
3317
3318 /* set HW defaults for 3D engine */
3319 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3320 ROQ_IB2_START(0x2b)));
3321
3322 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3323
3324 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3325 SYNC_GRADIENT |
3326 SYNC_WALKER |
3327 SYNC_ALIGNER));
3328
3329 sx_debug_1 = RREG32(SX_DEBUG_1);
3330 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3331 WREG32(SX_DEBUG_1, sx_debug_1);
3332
3333
3334 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3335 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3336 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3337 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3338
Alex Deucherb866d132012-06-14 22:06:36 +02003339 if (rdev->family <= CHIP_SUMO2)
3340 WREG32(SMX_SAR_CTL0, 0x00010000);
3341
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003342 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3343 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3344 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3345
3346 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3347 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3348 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3349
3350 WREG32(VGT_NUM_INSTANCES, 1);
3351 WREG32(SPI_CONFIG_CNTL, 0);
3352 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3353 WREG32(CP_PERFMON_CNTL, 0);
3354
3355 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3356 FETCH_FIFO_HIWATER(0x4) |
3357 DONE_FIFO_HIWATER(0xe0) |
3358 ALU_UPDATE_FIFO_HIWATER(0x8)));
3359
3360 sq_config = RREG32(SQ_CONFIG);
3361 sq_config &= ~(PS_PRIO(3) |
3362 VS_PRIO(3) |
3363 GS_PRIO(3) |
3364 ES_PRIO(3));
3365 sq_config |= (VC_ENABLE |
3366 EXPORT_SRC_C |
3367 PS_PRIO(0) |
3368 VS_PRIO(1) |
3369 GS_PRIO(2) |
3370 ES_PRIO(3));
3371
Alex Deucherd5e455e2010-11-22 17:56:29 -05003372 switch (rdev->family) {
3373 case CHIP_CEDAR:
3374 case CHIP_PALM:
Alex Deucherd5c5a722011-05-31 15:42:48 -04003375 case CHIP_SUMO:
3376 case CHIP_SUMO2:
Alex Deucheradb68fa2011-01-06 21:19:24 -05003377 case CHIP_CAICOS:
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003378 /* no vertex cache */
3379 sq_config &= ~VC_ENABLE;
Alex Deucherd5e455e2010-11-22 17:56:29 -05003380 break;
3381 default:
3382 break;
3383 }
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003384
3385 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3386
3387 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3388 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3389 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3390 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3391 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3392 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3393 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3394
Alex Deucherd5e455e2010-11-22 17:56:29 -05003395 switch (rdev->family) {
3396 case CHIP_CEDAR:
3397 case CHIP_PALM:
Alex Deucherd5c5a722011-05-31 15:42:48 -04003398 case CHIP_SUMO:
3399 case CHIP_SUMO2:
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003400 ps_thread_count = 96;
Alex Deucherd5e455e2010-11-22 17:56:29 -05003401 break;
3402 default:
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003403 ps_thread_count = 128;
Alex Deucherd5e455e2010-11-22 17:56:29 -05003404 break;
3405 }
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003406
3407 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
Alex Deucherf96b35c2010-06-16 12:24:07 -04003408 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3412 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003413
3414 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3419 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3420
3421 WREG32(SQ_CONFIG, sq_config);
3422 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3423 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3424 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3425 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3426 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3427 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3428 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3429 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3430 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3431 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3432
3433 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3434 FORCE_EOV_MAX_REZ_CNT(255)));
3435
Alex Deucherd5e455e2010-11-22 17:56:29 -05003436 switch (rdev->family) {
3437 case CHIP_CEDAR:
3438 case CHIP_PALM:
Alex Deucherd5c5a722011-05-31 15:42:48 -04003439 case CHIP_SUMO:
3440 case CHIP_SUMO2:
Alex Deucheradb68fa2011-01-06 21:19:24 -05003441 case CHIP_CAICOS:
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003442 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
Alex Deucherd5e455e2010-11-22 17:56:29 -05003443 break;
3444 default:
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003445 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
Alex Deucherd5e455e2010-11-22 17:56:29 -05003446 break;
3447 }
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003448 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3449 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3450
3451 WREG32(VGT_GS_VERTEX_REUSE, 16);
Alex Deucher12920592011-02-02 12:37:40 -05003452 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003453 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3454
Alex Deucher60a4a3e2010-06-29 17:03:35 -04003455 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3456 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3457
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003458 WREG32(CB_PERF_CTR0_SEL_0, 0);
3459 WREG32(CB_PERF_CTR0_SEL_1, 0);
3460 WREG32(CB_PERF_CTR1_SEL_0, 0);
3461 WREG32(CB_PERF_CTR1_SEL_1, 0);
3462 WREG32(CB_PERF_CTR2_SEL_0, 0);
3463 WREG32(CB_PERF_CTR2_SEL_1, 0);
3464 WREG32(CB_PERF_CTR3_SEL_0, 0);
3465 WREG32(CB_PERF_CTR3_SEL_1, 0);
3466
Alex Deucher60a4a3e2010-06-29 17:03:35 -04003467 /* clear render buffer base addresses */
3468 WREG32(CB_COLOR0_BASE, 0);
3469 WREG32(CB_COLOR1_BASE, 0);
3470 WREG32(CB_COLOR2_BASE, 0);
3471 WREG32(CB_COLOR3_BASE, 0);
3472 WREG32(CB_COLOR4_BASE, 0);
3473 WREG32(CB_COLOR5_BASE, 0);
3474 WREG32(CB_COLOR6_BASE, 0);
3475 WREG32(CB_COLOR7_BASE, 0);
3476 WREG32(CB_COLOR8_BASE, 0);
3477 WREG32(CB_COLOR9_BASE, 0);
3478 WREG32(CB_COLOR10_BASE, 0);
3479 WREG32(CB_COLOR11_BASE, 0);
3480
3481 /* set the shader const cache sizes to 0 */
3482 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3483 WREG32(i, 0);
3484 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3485 WREG32(i, 0);
3486
Alex Deucherf25a5c62011-05-19 11:07:57 -04003487 tmp = RREG32(HDP_MISC_CNTL);
3488 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3489 WREG32(HDP_MISC_CNTL, tmp);
3490
Alex Deucher32fcdbf2010-03-24 13:33:47 -04003491 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3492 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3493
3494 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3495
3496 udelay(50);
3497
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003498}
3499
3500int evergreen_mc_init(struct radeon_device *rdev)
3501{
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003502 u32 tmp;
3503 int chansize, numchan;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003504
3505 /* Get VRAM informations */
3506 rdev->mc.vram_is_ddr = true;
Alex Deucher05b3ef62012-03-20 17:18:37 -04003507 if ((rdev->family == CHIP_PALM) ||
3508 (rdev->family == CHIP_SUMO) ||
3509 (rdev->family == CHIP_SUMO2))
Alex Deucher82084412011-07-01 13:18:28 -04003510 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3511 else
3512 tmp = RREG32(MC_ARB_RAMCFG);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003513 if (tmp & CHANSIZE_OVERRIDE) {
3514 chansize = 16;
3515 } else if (tmp & CHANSIZE_MASK) {
3516 chansize = 64;
3517 } else {
3518 chansize = 32;
3519 }
3520 tmp = RREG32(MC_SHARED_CHMAP);
3521 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3522 case 0:
3523 default:
3524 numchan = 1;
3525 break;
3526 case 1:
3527 numchan = 2;
3528 break;
3529 case 2:
3530 numchan = 4;
3531 break;
3532 case 3:
3533 numchan = 8;
3534 break;
3535 }
3536 rdev->mc.vram_width = numchan * chansize;
3537 /* Could aper size report 0 ? */
Jordan Crouse01d73a62010-05-27 13:40:24 -06003538 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3539 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003540 /* Setup GPU memory space */
Alex Deucher05b3ef62012-03-20 17:18:37 -04003541 if ((rdev->family == CHIP_PALM) ||
3542 (rdev->family == CHIP_SUMO) ||
3543 (rdev->family == CHIP_SUMO2)) {
Alex Deucher6eb18f82010-11-22 17:56:27 -05003544 /* size in bytes on fusion */
3545 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3546 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3547 } else {
Alex Deucher05b3ef62012-03-20 17:18:37 -04003548 /* size in MB on evergreen/cayman/tn */
Niels Ole Salscheiderfc986032013-05-18 21:19:23 +02003549 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3550 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher6eb18f82010-11-22 17:56:27 -05003551 }
Jerome Glisse51e5fcd2010-02-19 14:33:54 +00003552 rdev->mc.visible_vram_size = rdev->mc.aper_size;
Alex Deucher0ef0c1f2010-11-22 17:56:26 -05003553 r700_vram_gtt_location(rdev, &rdev->mc);
Alex Deucherf47299c2010-03-16 20:54:38 -04003554 radeon_update_bandwidth_info(rdev);
3555
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003556 return 0;
3557}
Jerome Glissed594e462010-02-17 21:54:29 +00003558
Alex Deucher187e3592013-01-18 14:51:38 -05003559void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
Alex Deucher747943e2010-03-24 13:26:36 -04003560{
Jerome Glisse64c56e82013-01-02 17:30:35 -05003561 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
Alex Deucher747943e2010-03-24 13:26:36 -04003562 RREG32(GRBM_STATUS));
Jerome Glisse64c56e82013-01-02 17:30:35 -05003563 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
Alex Deucher747943e2010-03-24 13:26:36 -04003564 RREG32(GRBM_STATUS_SE0));
Jerome Glisse64c56e82013-01-02 17:30:35 -05003565 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
Alex Deucher747943e2010-03-24 13:26:36 -04003566 RREG32(GRBM_STATUS_SE1));
Jerome Glisse64c56e82013-01-02 17:30:35 -05003567 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
Alex Deucher747943e2010-03-24 13:26:36 -04003568 RREG32(SRBM_STATUS));
Alex Deuchera65a4362013-01-18 18:55:54 -05003569 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3570 RREG32(SRBM_STATUS2));
Jerome Glisse440a7cd2012-06-27 12:25:01 -04003571 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3572 RREG32(CP_STALLED_STAT1));
3573 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3574 RREG32(CP_STALLED_STAT2));
3575 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3576 RREG32(CP_BUSY_STAT));
3577 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3578 RREG32(CP_STAT));
Alex Deucher0ecebb92013-01-03 12:40:13 -05003579 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3580 RREG32(DMA_STATUS_REG));
Alex Deucher168757e2013-01-18 19:17:22 -05003581 if (rdev->family >= CHIP_CAYMAN) {
3582 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3583 RREG32(DMA_STATUS_REG + 0x800));
3584 }
Alex Deucher0ecebb92013-01-03 12:40:13 -05003585}
3586
Alex Deucher168757e2013-01-18 19:17:22 -05003587bool evergreen_is_display_hung(struct radeon_device *rdev)
Alex Deuchera65a4362013-01-18 18:55:54 -05003588{
3589 u32 crtc_hung = 0;
3590 u32 crtc_status[6];
3591 u32 i, j, tmp;
3592
3593 for (i = 0; i < rdev->num_crtc; i++) {
3594 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3595 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3596 crtc_hung |= (1 << i);
3597 }
3598 }
3599
3600 for (j = 0; j < 10; j++) {
3601 for (i = 0; i < rdev->num_crtc; i++) {
3602 if (crtc_hung & (1 << i)) {
3603 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3604 if (tmp != crtc_status[i])
3605 crtc_hung &= ~(1 << i);
3606 }
3607 }
3608 if (crtc_hung == 0)
3609 return false;
3610 udelay(100);
3611 }
3612
3613 return true;
3614}
3615
3616static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3617{
3618 u32 reset_mask = 0;
3619 u32 tmp;
3620
3621 /* GRBM_STATUS */
3622 tmp = RREG32(GRBM_STATUS);
3623 if (tmp & (PA_BUSY | SC_BUSY |
3624 SH_BUSY | SX_BUSY |
3625 TA_BUSY | VGT_BUSY |
3626 DB_BUSY | CB_BUSY |
3627 SPI_BUSY | VGT_BUSY_NO_DMA))
3628 reset_mask |= RADEON_RESET_GFX;
3629
3630 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3631 CP_BUSY | CP_COHERENCY_BUSY))
3632 reset_mask |= RADEON_RESET_CP;
3633
3634 if (tmp & GRBM_EE_BUSY)
3635 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3636
3637 /* DMA_STATUS_REG */
3638 tmp = RREG32(DMA_STATUS_REG);
3639 if (!(tmp & DMA_IDLE))
3640 reset_mask |= RADEON_RESET_DMA;
3641
3642 /* SRBM_STATUS2 */
3643 tmp = RREG32(SRBM_STATUS2);
3644 if (tmp & DMA_BUSY)
3645 reset_mask |= RADEON_RESET_DMA;
3646
3647 /* SRBM_STATUS */
3648 tmp = RREG32(SRBM_STATUS);
3649 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3650 reset_mask |= RADEON_RESET_RLC;
3651
3652 if (tmp & IH_BUSY)
3653 reset_mask |= RADEON_RESET_IH;
3654
3655 if (tmp & SEM_BUSY)
3656 reset_mask |= RADEON_RESET_SEM;
3657
3658 if (tmp & GRBM_RQ_PENDING)
3659 reset_mask |= RADEON_RESET_GRBM;
3660
3661 if (tmp & VMC_BUSY)
3662 reset_mask |= RADEON_RESET_VMC;
3663
3664 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3665 MCC_BUSY | MCD_BUSY))
3666 reset_mask |= RADEON_RESET_MC;
3667
3668 if (evergreen_is_display_hung(rdev))
3669 reset_mask |= RADEON_RESET_DISPLAY;
3670
3671 /* VM_L2_STATUS */
3672 tmp = RREG32(VM_L2_STATUS);
3673 if (tmp & L2_BUSY)
3674 reset_mask |= RADEON_RESET_VMC;
3675
Alex Deucherd808fc82013-02-28 10:03:08 -05003676 /* Skip MC reset as it's mostly likely not hung, just busy */
3677 if (reset_mask & RADEON_RESET_MC) {
3678 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3679 reset_mask &= ~RADEON_RESET_MC;
3680 }
3681
Alex Deuchera65a4362013-01-18 18:55:54 -05003682 return reset_mask;
3683}
3684
3685static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
Alex Deucher0ecebb92013-01-03 12:40:13 -05003686{
3687 struct evergreen_mc_save save;
Alex Deucherb7630472013-01-18 14:28:41 -05003688 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3689 u32 tmp;
Alex Deucher19fc42e2013-01-14 11:04:39 -05003690
Alex Deucher0ecebb92013-01-03 12:40:13 -05003691 if (reset_mask == 0)
Alex Deuchera65a4362013-01-18 18:55:54 -05003692 return;
Alex Deucher0ecebb92013-01-03 12:40:13 -05003693
3694 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3695
Alex Deucherb7630472013-01-18 14:28:41 -05003696 evergreen_print_gpu_status_regs(rdev);
3697
Alex Deucherb7630472013-01-18 14:28:41 -05003698 /* Disable CP parsing/prefetching */
3699 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3700
3701 if (reset_mask & RADEON_RESET_DMA) {
3702 /* Disable DMA */
3703 tmp = RREG32(DMA_RB_CNTL);
3704 tmp &= ~DMA_RB_ENABLE;
3705 WREG32(DMA_RB_CNTL, tmp);
3706 }
3707
Alex Deucherb21b6e72013-01-23 18:57:56 -05003708 udelay(50);
3709
3710 evergreen_mc_stop(rdev, &save);
3711 if (evergreen_mc_wait_for_idle(rdev)) {
3712 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3713 }
3714
Alex Deucherb7630472013-01-18 14:28:41 -05003715 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3716 grbm_soft_reset |= SOFT_RESET_DB |
3717 SOFT_RESET_CB |
3718 SOFT_RESET_PA |
3719 SOFT_RESET_SC |
3720 SOFT_RESET_SPI |
3721 SOFT_RESET_SX |
3722 SOFT_RESET_SH |
3723 SOFT_RESET_TC |
3724 SOFT_RESET_TA |
3725 SOFT_RESET_VC |
3726 SOFT_RESET_VGT;
3727 }
3728
3729 if (reset_mask & RADEON_RESET_CP) {
3730 grbm_soft_reset |= SOFT_RESET_CP |
3731 SOFT_RESET_VGT;
3732
3733 srbm_soft_reset |= SOFT_RESET_GRBM;
3734 }
Alex Deucher0ecebb92013-01-03 12:40:13 -05003735
3736 if (reset_mask & RADEON_RESET_DMA)
Alex Deucherb7630472013-01-18 14:28:41 -05003737 srbm_soft_reset |= SOFT_RESET_DMA;
3738
Alex Deuchera65a4362013-01-18 18:55:54 -05003739 if (reset_mask & RADEON_RESET_DISPLAY)
3740 srbm_soft_reset |= SOFT_RESET_DC;
3741
3742 if (reset_mask & RADEON_RESET_RLC)
3743 srbm_soft_reset |= SOFT_RESET_RLC;
3744
3745 if (reset_mask & RADEON_RESET_SEM)
3746 srbm_soft_reset |= SOFT_RESET_SEM;
3747
3748 if (reset_mask & RADEON_RESET_IH)
3749 srbm_soft_reset |= SOFT_RESET_IH;
3750
3751 if (reset_mask & RADEON_RESET_GRBM)
3752 srbm_soft_reset |= SOFT_RESET_GRBM;
3753
3754 if (reset_mask & RADEON_RESET_VMC)
3755 srbm_soft_reset |= SOFT_RESET_VMC;
3756
Alex Deucher24178ec2013-01-24 15:00:17 -05003757 if (!(rdev->flags & RADEON_IS_IGP)) {
3758 if (reset_mask & RADEON_RESET_MC)
3759 srbm_soft_reset |= SOFT_RESET_MC;
3760 }
Alex Deuchera65a4362013-01-18 18:55:54 -05003761
Alex Deucherb7630472013-01-18 14:28:41 -05003762 if (grbm_soft_reset) {
3763 tmp = RREG32(GRBM_SOFT_RESET);
3764 tmp |= grbm_soft_reset;
3765 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3766 WREG32(GRBM_SOFT_RESET, tmp);
3767 tmp = RREG32(GRBM_SOFT_RESET);
3768
3769 udelay(50);
3770
3771 tmp &= ~grbm_soft_reset;
3772 WREG32(GRBM_SOFT_RESET, tmp);
3773 tmp = RREG32(GRBM_SOFT_RESET);
3774 }
3775
3776 if (srbm_soft_reset) {
3777 tmp = RREG32(SRBM_SOFT_RESET);
3778 tmp |= srbm_soft_reset;
3779 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3780 WREG32(SRBM_SOFT_RESET, tmp);
3781 tmp = RREG32(SRBM_SOFT_RESET);
3782
3783 udelay(50);
3784
3785 tmp &= ~srbm_soft_reset;
3786 WREG32(SRBM_SOFT_RESET, tmp);
3787 tmp = RREG32(SRBM_SOFT_RESET);
3788 }
Alex Deucher0ecebb92013-01-03 12:40:13 -05003789
3790 /* Wait a little for things to settle down */
3791 udelay(50);
3792
Alex Deucher747943e2010-03-24 13:26:36 -04003793 evergreen_mc_resume(rdev, &save);
Alex Deucherb7630472013-01-18 14:28:41 -05003794 udelay(50);
Alex Deucher410a3412013-01-18 13:05:39 -05003795
Alex Deucherb7630472013-01-18 14:28:41 -05003796 evergreen_print_gpu_status_regs(rdev);
Alex Deucher747943e2010-03-24 13:26:36 -04003797}
3798
Jerome Glissea2d07b72010-03-09 14:45:11 +00003799int evergreen_asic_reset(struct radeon_device *rdev)
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003800{
Alex Deuchera65a4362013-01-18 18:55:54 -05003801 u32 reset_mask;
3802
3803 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805 if (reset_mask)
3806 r600_set_bios_scratch_engine_hung(rdev, true);
3807
3808 evergreen_gpu_soft_reset(rdev, reset_mask);
3809
3810 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3811
3812 if (!reset_mask)
3813 r600_set_bios_scratch_engine_hung(rdev, false);
3814
3815 return 0;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05003816}
3817
Alex Deucher123bc182013-01-24 11:37:19 -05003818/**
3819 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3820 *
3821 * @rdev: radeon_device pointer
3822 * @ring: radeon_ring structure holding ring information
3823 *
3824 * Check if the GFX engine is locked up.
3825 * Returns true if the engine appears to be locked up, false if not.
3826 */
3827bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3828{
3829 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3830
3831 if (!(reset_mask & (RADEON_RESET_GFX |
3832 RADEON_RESET_COMPUTE |
3833 RADEON_RESET_CP))) {
3834 radeon_ring_lockup_update(ring);
3835 return false;
3836 }
3837 /* force CP activities */
3838 radeon_ring_force_activity(rdev, ring);
3839 return radeon_ring_test_lockup(rdev, ring);
3840}
3841
3842/**
3843 * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3844 *
3845 * @rdev: radeon_device pointer
3846 * @ring: radeon_ring structure holding ring information
3847 *
3848 * Check if the async DMA engine is locked up.
3849 * Returns true if the engine appears to be locked up, false if not.
3850 */
3851bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3852{
3853 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3854
3855 if (!(reset_mask & RADEON_RESET_DMA)) {
3856 radeon_ring_lockup_update(ring);
3857 return false;
3858 }
3859 /* force ring activities */
3860 radeon_ring_force_activity(rdev, ring);
3861 return radeon_ring_test_lockup(rdev, ring);
3862}
3863
Alex Deucher2948f5e2013-04-12 13:52:52 -04003864/*
3865 * RLC
3866 */
3867#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3868#define RLC_CLEAR_STATE_END_MARKER 0x00000001
3869
3870void sumo_rlc_fini(struct radeon_device *rdev)
3871{
3872 int r;
3873
3874 /* save restore block */
3875 if (rdev->rlc.save_restore_obj) {
3876 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3877 if (unlikely(r != 0))
3878 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3879 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3880 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3881
3882 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3883 rdev->rlc.save_restore_obj = NULL;
3884 }
3885
3886 /* clear state block */
3887 if (rdev->rlc.clear_state_obj) {
3888 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3889 if (unlikely(r != 0))
3890 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3891 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3892 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3893
3894 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3895 rdev->rlc.clear_state_obj = NULL;
3896 }
Alex Deucher22c775c2013-07-23 09:41:05 -04003897
3898 /* clear state block */
3899 if (rdev->rlc.cp_table_obj) {
3900 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3901 if (unlikely(r != 0))
3902 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3903 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3904 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3905
3906 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3907 rdev->rlc.cp_table_obj = NULL;
3908 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04003909}
3910
Alex Deucher22c775c2013-07-23 09:41:05 -04003911#define CP_ME_TABLE_SIZE 96
3912
Alex Deucher2948f5e2013-04-12 13:52:52 -04003913int sumo_rlc_init(struct radeon_device *rdev)
3914{
Alex Deucher1fd11772013-04-17 17:53:50 -04003915 const u32 *src_ptr;
Alex Deucher2948f5e2013-04-12 13:52:52 -04003916 volatile u32 *dst_ptr;
3917 u32 dws, data, i, j, k, reg_num;
3918 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3919 u64 reg_list_mc_addr;
Alex Deucher1fd11772013-04-17 17:53:50 -04003920 const struct cs_section_def *cs_data;
Alex Deucher2948f5e2013-04-12 13:52:52 -04003921 int r;
3922
3923 src_ptr = rdev->rlc.reg_list;
3924 dws = rdev->rlc.reg_list_size;
3925 cs_data = rdev->rlc.cs_data;
3926
Alex Deucher10b7ca72013-04-17 17:22:05 -04003927 if (src_ptr) {
3928 /* save restore block */
3929 if (rdev->rlc.save_restore_obj == NULL) {
3930 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3931 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3932 if (r) {
3933 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3934 return r;
3935 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04003936 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04003937
Alex Deucher10b7ca72013-04-17 17:22:05 -04003938 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3939 if (unlikely(r != 0)) {
Alex Deucher2948f5e2013-04-12 13:52:52 -04003940 sumo_rlc_fini(rdev);
3941 return r;
3942 }
Alex Deucher10b7ca72013-04-17 17:22:05 -04003943 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3944 &rdev->rlc.save_restore_gpu_addr);
3945 if (r) {
3946 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3947 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3948 sumo_rlc_fini(rdev);
3949 return r;
Alex Deucher2948f5e2013-04-12 13:52:52 -04003950 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04003951
Alex Deucher10b7ca72013-04-17 17:22:05 -04003952 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3953 if (r) {
3954 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3955 sumo_rlc_fini(rdev);
3956 return r;
3957 }
3958 /* write the sr buffer */
3959 dst_ptr = rdev->rlc.sr_ptr;
Alex Deucher1fd11772013-04-17 17:53:50 -04003960 if (rdev->family >= CHIP_TAHITI) {
3961 /* SI */
3962 for (i = 0; i < dws; i++)
3963 dst_ptr[i] = src_ptr[i];
3964 } else {
3965 /* ON/LN/TN */
3966 /* format:
3967 * dw0: (reg2 << 16) | reg1
3968 * dw1: reg1 save space
3969 * dw2: reg2 save space
3970 */
3971 for (i = 0; i < dws; i++) {
3972 data = src_ptr[i] >> 2;
3973 i++;
3974 if (i < dws)
3975 data |= (src_ptr[i] >> 2) << 16;
3976 j = (((i - 1) * 3) / 2);
3977 dst_ptr[j] = data;
3978 }
3979 j = ((i * 3) / 2);
3980 dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
Alex Deucher10b7ca72013-04-17 17:22:05 -04003981 }
Alex Deucher10b7ca72013-04-17 17:22:05 -04003982 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3983 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3984 }
3985
3986 if (cs_data) {
3987 /* clear state block */
3988 reg_list_num = 0;
3989 dws = 0;
3990 for (i = 0; cs_data[i].section != NULL; i++) {
3991 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3992 reg_list_num++;
3993 dws += cs_data[i].section[j].reg_count;
3994 }
3995 }
3996 reg_list_blk_index = (3 * reg_list_num + 2);
3997 dws += reg_list_blk_index;
Alex Deucher22c775c2013-07-23 09:41:05 -04003998 rdev->rlc.clear_state_size = dws;
Alex Deucher10b7ca72013-04-17 17:22:05 -04003999
4000 if (rdev->rlc.clear_state_obj == NULL) {
Alex Deucher22c775c2013-07-23 09:41:05 -04004001 r = radeon_bo_create(rdev, rdev->rlc.clear_state_size * 4, PAGE_SIZE, true,
Alex Deucher10b7ca72013-04-17 17:22:05 -04004002 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4003 if (r) {
4004 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4005 sumo_rlc_fini(rdev);
4006 return r;
4007 }
4008 }
4009 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4010 if (unlikely(r != 0)) {
4011 sumo_rlc_fini(rdev);
4012 return r;
4013 }
4014 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4015 &rdev->rlc.clear_state_gpu_addr);
4016 if (r) {
4017 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4018 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4019 sumo_rlc_fini(rdev);
4020 return r;
4021 }
4022
4023 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4024 if (r) {
4025 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4026 sumo_rlc_fini(rdev);
4027 return r;
4028 }
4029 /* set up the cs buffer */
4030 dst_ptr = rdev->rlc.cs_ptr;
4031 reg_list_hdr_blk_index = 0;
4032 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4033 data = upper_32_bits(reg_list_mc_addr);
4034 dst_ptr[reg_list_hdr_blk_index] = data;
4035 reg_list_hdr_blk_index++;
4036 for (i = 0; cs_data[i].section != NULL; i++) {
4037 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4038 reg_num = cs_data[i].section[j].reg_count;
4039 data = reg_list_mc_addr & 0xffffffff;
4040 dst_ptr[reg_list_hdr_blk_index] = data;
4041 reg_list_hdr_blk_index++;
4042
4043 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4044 dst_ptr[reg_list_hdr_blk_index] = data;
4045 reg_list_hdr_blk_index++;
4046
4047 data = 0x08000000 | (reg_num * 4);
4048 dst_ptr[reg_list_hdr_blk_index] = data;
4049 reg_list_hdr_blk_index++;
4050
4051 for (k = 0; k < reg_num; k++) {
4052 data = cs_data[i].section[j].extent[k];
4053 dst_ptr[reg_list_blk_index + k] = data;
4054 }
4055 reg_list_mc_addr += reg_num * 4;
4056 reg_list_blk_index += reg_num;
4057 }
4058 }
4059 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4060
4061 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4062 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4063 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04004064
Alex Deucher22c775c2013-07-23 09:41:05 -04004065 if (rdev->rlc.cp_table_size) {
4066 if (rdev->rlc.cp_table_obj == NULL) {
4067 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4068 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4069 if (r) {
4070 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4071 sumo_rlc_fini(rdev);
4072 return r;
4073 }
4074 }
4075
4076 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4077 if (unlikely(r != 0)) {
4078 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4079 sumo_rlc_fini(rdev);
4080 return r;
4081 }
4082 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4083 &rdev->rlc.cp_table_gpu_addr);
4084 if (r) {
4085 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4086 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4087 sumo_rlc_fini(rdev);
4088 return r;
4089 }
4090 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4091 if (r) {
4092 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4093 sumo_rlc_fini(rdev);
4094 return r;
4095 }
4096
4097 cik_init_cp_pg_table(rdev);
4098
4099 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4100 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4101
4102 }
4103
Alex Deucher2948f5e2013-04-12 13:52:52 -04004104 return 0;
4105}
4106
4107static void evergreen_rlc_start(struct radeon_device *rdev)
4108{
Alex Deucher8ba10462013-02-15 16:26:33 -05004109 u32 mask = RLC_ENABLE;
4110
4111 if (rdev->flags & RADEON_IS_IGP) {
4112 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
Alex Deucher8ba10462013-02-15 16:26:33 -05004113 }
4114
4115 WREG32(RLC_CNTL, mask);
Alex Deucher2948f5e2013-04-12 13:52:52 -04004116}
4117
4118int evergreen_rlc_resume(struct radeon_device *rdev)
4119{
4120 u32 i;
4121 const __be32 *fw_data;
4122
4123 if (!rdev->rlc_fw)
4124 return -EINVAL;
4125
4126 r600_rlc_stop(rdev);
4127
4128 WREG32(RLC_HB_CNTL, 0);
4129
4130 if (rdev->flags & RADEON_IS_IGP) {
Alex Deucher8ba10462013-02-15 16:26:33 -05004131 if (rdev->family == CHIP_ARUBA) {
4132 u32 always_on_bitmap =
4133 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4134 /* find out the number of active simds */
4135 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4136 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4137 tmp = hweight32(~tmp);
4138 if (tmp == rdev->config.cayman.max_simds_per_se) {
4139 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4140 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4141 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4142 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4143 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4144 }
4145 } else {
4146 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4147 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4148 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04004149 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4150 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4151 } else {
4152 WREG32(RLC_HB_BASE, 0);
4153 WREG32(RLC_HB_RPTR, 0);
4154 WREG32(RLC_HB_WPTR, 0);
Alex Deucher8ba10462013-02-15 16:26:33 -05004155 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4156 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
Alex Deucher2948f5e2013-04-12 13:52:52 -04004157 }
Alex Deucher2948f5e2013-04-12 13:52:52 -04004158 WREG32(RLC_MC_CNTL, 0);
4159 WREG32(RLC_UCODE_CNTL, 0);
4160
4161 fw_data = (const __be32 *)rdev->rlc_fw->data;
4162 if (rdev->family >= CHIP_ARUBA) {
4163 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4164 WREG32(RLC_UCODE_ADDR, i);
4165 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4166 }
4167 } else if (rdev->family >= CHIP_CAYMAN) {
4168 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4169 WREG32(RLC_UCODE_ADDR, i);
4170 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4171 }
4172 } else {
4173 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4174 WREG32(RLC_UCODE_ADDR, i);
4175 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4176 }
4177 }
4178 WREG32(RLC_UCODE_ADDR, 0);
4179
4180 evergreen_rlc_start(rdev);
4181
4182 return 0;
4183}
4184
Alex Deucher45f9a392010-03-24 13:55:51 -04004185/* Interrupts */
4186
4187u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4188{
Alex Deucher46437052012-08-15 17:10:32 -04004189 if (crtc >= rdev->num_crtc)
Alex Deucher45f9a392010-03-24 13:55:51 -04004190 return 0;
Alex Deucher46437052012-08-15 17:10:32 -04004191 else
4192 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
Alex Deucher45f9a392010-03-24 13:55:51 -04004193}
4194
4195void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4196{
4197 u32 tmp;
4198
Alex Deucher1b370782011-11-17 20:13:28 -05004199 if (rdev->family >= CHIP_CAYMAN) {
4200 cayman_cp_int_cntl_setup(rdev, 0,
4201 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4202 cayman_cp_int_cntl_setup(rdev, 1, 0);
4203 cayman_cp_int_cntl_setup(rdev, 2, 0);
Alex Deucherf60cbd12012-12-04 15:27:33 -05004204 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4205 WREG32(CAYMAN_DMA1_CNTL, tmp);
Alex Deucher1b370782011-11-17 20:13:28 -05004206 } else
4207 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucher233d1ad2012-12-04 15:25:59 -05004208 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4209 WREG32(DMA_CNTL, tmp);
Alex Deucher45f9a392010-03-24 13:55:51 -04004210 WREG32(GRBM_INT_CNTL, 0);
4211 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4212 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
Alex Deucherb7eff392011-07-08 11:44:56 -04004213 if (rdev->num_crtc >= 4) {
Alex Deucher18007402010-11-22 17:56:28 -05004214 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4215 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
Alex Deucherb7eff392011-07-08 11:44:56 -04004216 }
4217 if (rdev->num_crtc >= 6) {
Alex Deucher18007402010-11-22 17:56:28 -05004218 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4219 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4220 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004221
4222 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4223 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
Alex Deucherb7eff392011-07-08 11:44:56 -04004224 if (rdev->num_crtc >= 4) {
Alex Deucher18007402010-11-22 17:56:28 -05004225 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4226 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
Alex Deucherb7eff392011-07-08 11:44:56 -04004227 }
4228 if (rdev->num_crtc >= 6) {
Alex Deucher18007402010-11-22 17:56:28 -05004229 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4230 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4231 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004232
Alex Deucher05b3ef62012-03-20 17:18:37 -04004233 /* only one DAC on DCE6 */
4234 if (!ASIC_IS_DCE6(rdev))
4235 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
Alex Deucher45f9a392010-03-24 13:55:51 -04004236 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4237
4238 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4239 WREG32(DC_HPD1_INT_CONTROL, tmp);
4240 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4241 WREG32(DC_HPD2_INT_CONTROL, tmp);
4242 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4243 WREG32(DC_HPD3_INT_CONTROL, tmp);
4244 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4245 WREG32(DC_HPD4_INT_CONTROL, tmp);
4246 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4247 WREG32(DC_HPD5_INT_CONTROL, tmp);
4248 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4249 WREG32(DC_HPD6_INT_CONTROL, tmp);
4250
4251}
4252
4253int evergreen_irq_set(struct radeon_device *rdev)
4254{
4255 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
Alex Deucher1b370782011-11-17 20:13:28 -05004256 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
Alex Deucher45f9a392010-03-24 13:55:51 -04004257 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4258 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
Alex Deucher2031f772010-04-22 12:52:11 -04004259 u32 grbm_int_cntl = 0;
Alex Deucher6f34be52010-11-21 10:59:01 -05004260 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
Alex Deucherf122c612012-03-30 08:59:57 -04004261 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
Alex Deucherf60cbd12012-12-04 15:27:33 -05004262 u32 dma_cntl, dma_cntl1 = 0;
Alex Deucherdc50ba72013-06-26 00:33:35 -04004263 u32 thermal_int = 0;
Alex Deucher45f9a392010-03-24 13:55:51 -04004264
4265 if (!rdev->irq.installed) {
Joe Perchesfce7d612010-10-30 21:08:30 +00004266 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
Alex Deucher45f9a392010-03-24 13:55:51 -04004267 return -EINVAL;
4268 }
4269 /* don't enable anything if the ih is disabled */
4270 if (!rdev->ih.enabled) {
4271 r600_disable_interrupts(rdev);
4272 /* force the active interrupt state to all disabled */
4273 evergreen_disable_interrupt_state(rdev);
4274 return 0;
4275 }
4276
4277 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4278 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4279 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4280 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4281 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4282 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
Alex Deucherd70229f2013-04-12 16:40:41 -04004283 if (rdev->family == CHIP_ARUBA)
4284 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4285 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4286 else
4287 thermal_int = RREG32(CG_THERMAL_INT) &
4288 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher45f9a392010-03-24 13:55:51 -04004289
Alex Deucherf122c612012-03-30 08:59:57 -04004290 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4291 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4292 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4293 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4294 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4295 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4296
Alex Deucher233d1ad2012-12-04 15:25:59 -05004297 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4298
Alex Deucher1b370782011-11-17 20:13:28 -05004299 if (rdev->family >= CHIP_CAYMAN) {
4300 /* enable CP interrupts on all rings */
Christian Koenig736fc372012-05-17 19:52:00 +02004301 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
Alex Deucher1b370782011-11-17 20:13:28 -05004302 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4303 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4304 }
Christian Koenig736fc372012-05-17 19:52:00 +02004305 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
Alex Deucher1b370782011-11-17 20:13:28 -05004306 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4307 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4308 }
Christian Koenig736fc372012-05-17 19:52:00 +02004309 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
Alex Deucher1b370782011-11-17 20:13:28 -05004310 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4311 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4312 }
4313 } else {
Christian Koenig736fc372012-05-17 19:52:00 +02004314 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
Alex Deucher1b370782011-11-17 20:13:28 -05004315 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4316 cp_int_cntl |= RB_INT_ENABLE;
4317 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4318 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004319 }
Alex Deucher1b370782011-11-17 20:13:28 -05004320
Alex Deucher233d1ad2012-12-04 15:25:59 -05004321 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4322 DRM_DEBUG("r600_irq_set: sw int dma\n");
4323 dma_cntl |= TRAP_ENABLE;
4324 }
4325
Alex Deucherf60cbd12012-12-04 15:27:33 -05004326 if (rdev->family >= CHIP_CAYMAN) {
4327 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4328 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4329 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4330 dma_cntl1 |= TRAP_ENABLE;
4331 }
4332 }
4333
Alex Deucherdc50ba72013-06-26 00:33:35 -04004334 if (rdev->irq.dpm_thermal) {
4335 DRM_DEBUG("dpm thermal\n");
4336 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4337 }
4338
Alex Deucher6f34be52010-11-21 10:59:01 -05004339 if (rdev->irq.crtc_vblank_int[0] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004340 atomic_read(&rdev->irq.pflip[0])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004341 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4342 crtc1 |= VBLANK_INT_MASK;
4343 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004344 if (rdev->irq.crtc_vblank_int[1] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004345 atomic_read(&rdev->irq.pflip[1])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004346 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4347 crtc2 |= VBLANK_INT_MASK;
4348 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004349 if (rdev->irq.crtc_vblank_int[2] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004350 atomic_read(&rdev->irq.pflip[2])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004351 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4352 crtc3 |= VBLANK_INT_MASK;
4353 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004354 if (rdev->irq.crtc_vblank_int[3] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004355 atomic_read(&rdev->irq.pflip[3])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004356 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4357 crtc4 |= VBLANK_INT_MASK;
4358 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004359 if (rdev->irq.crtc_vblank_int[4] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004360 atomic_read(&rdev->irq.pflip[4])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004361 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4362 crtc5 |= VBLANK_INT_MASK;
4363 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004364 if (rdev->irq.crtc_vblank_int[5] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004365 atomic_read(&rdev->irq.pflip[5])) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004366 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4367 crtc6 |= VBLANK_INT_MASK;
4368 }
4369 if (rdev->irq.hpd[0]) {
4370 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4371 hpd1 |= DC_HPDx_INT_EN;
4372 }
4373 if (rdev->irq.hpd[1]) {
4374 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4375 hpd2 |= DC_HPDx_INT_EN;
4376 }
4377 if (rdev->irq.hpd[2]) {
4378 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4379 hpd3 |= DC_HPDx_INT_EN;
4380 }
4381 if (rdev->irq.hpd[3]) {
4382 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4383 hpd4 |= DC_HPDx_INT_EN;
4384 }
4385 if (rdev->irq.hpd[4]) {
4386 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4387 hpd5 |= DC_HPDx_INT_EN;
4388 }
4389 if (rdev->irq.hpd[5]) {
4390 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4391 hpd6 |= DC_HPDx_INT_EN;
4392 }
Alex Deucherf122c612012-03-30 08:59:57 -04004393 if (rdev->irq.afmt[0]) {
4394 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4395 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4396 }
4397 if (rdev->irq.afmt[1]) {
4398 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4399 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4400 }
4401 if (rdev->irq.afmt[2]) {
4402 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4403 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4404 }
4405 if (rdev->irq.afmt[3]) {
4406 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4407 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4408 }
4409 if (rdev->irq.afmt[4]) {
4410 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4411 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4412 }
4413 if (rdev->irq.afmt[5]) {
4414 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4415 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4416 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004417
Alex Deucher1b370782011-11-17 20:13:28 -05004418 if (rdev->family >= CHIP_CAYMAN) {
4419 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4420 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4421 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4422 } else
4423 WREG32(CP_INT_CNTL, cp_int_cntl);
Alex Deucher233d1ad2012-12-04 15:25:59 -05004424
4425 WREG32(DMA_CNTL, dma_cntl);
4426
Alex Deucherf60cbd12012-12-04 15:27:33 -05004427 if (rdev->family >= CHIP_CAYMAN)
4428 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4429
Alex Deucher2031f772010-04-22 12:52:11 -04004430 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
Alex Deucher45f9a392010-03-24 13:55:51 -04004431
4432 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4433 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
Alex Deucherb7eff392011-07-08 11:44:56 -04004434 if (rdev->num_crtc >= 4) {
Alex Deucher18007402010-11-22 17:56:28 -05004435 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4436 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
Alex Deucherb7eff392011-07-08 11:44:56 -04004437 }
4438 if (rdev->num_crtc >= 6) {
Alex Deucher18007402010-11-22 17:56:28 -05004439 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4440 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4441 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004442
Alex Deucher6f34be52010-11-21 10:59:01 -05004443 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4444 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
Alex Deucherb7eff392011-07-08 11:44:56 -04004445 if (rdev->num_crtc >= 4) {
4446 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4447 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4448 }
4449 if (rdev->num_crtc >= 6) {
4450 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4451 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4452 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004453
Alex Deucher45f9a392010-03-24 13:55:51 -04004454 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4455 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4456 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4457 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4458 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4459 WREG32(DC_HPD6_INT_CONTROL, hpd6);
Alex Deucherd70229f2013-04-12 16:40:41 -04004460 if (rdev->family == CHIP_ARUBA)
4461 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4462 else
4463 WREG32(CG_THERMAL_INT, thermal_int);
Alex Deucher45f9a392010-03-24 13:55:51 -04004464
Alex Deucherf122c612012-03-30 08:59:57 -04004465 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4466 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4467 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4468 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4469 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4470 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4471
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05004472 return 0;
4473}
4474
Andi Kleencbdd4502011-10-13 16:08:46 -07004475static void evergreen_irq_ack(struct radeon_device *rdev)
Alex Deucher45f9a392010-03-24 13:55:51 -04004476{
4477 u32 tmp;
4478
Alex Deucher6f34be52010-11-21 10:59:01 -05004479 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4480 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4481 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4482 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4483 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4484 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4485 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4486 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
Alex Deucherb7eff392011-07-08 11:44:56 -04004487 if (rdev->num_crtc >= 4) {
4488 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4489 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4490 }
4491 if (rdev->num_crtc >= 6) {
4492 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4493 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4494 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004495
Alex Deucherf122c612012-03-30 08:59:57 -04004496 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4497 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4498 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4499 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4500 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4501 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4502
Alex Deucher6f34be52010-11-21 10:59:01 -05004503 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4504 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4505 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4506 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
Alex Deucher6f34be52010-11-21 10:59:01 -05004507 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
Alex Deucher45f9a392010-03-24 13:55:51 -04004508 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
Alex Deucher6f34be52010-11-21 10:59:01 -05004509 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
Alex Deucher45f9a392010-03-24 13:55:51 -04004510 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
Alex Deucher6f34be52010-11-21 10:59:01 -05004511 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
Alex Deucher45f9a392010-03-24 13:55:51 -04004512 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
Alex Deucher6f34be52010-11-21 10:59:01 -05004513 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
Alex Deucher45f9a392010-03-24 13:55:51 -04004514 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4515
Alex Deucherb7eff392011-07-08 11:44:56 -04004516 if (rdev->num_crtc >= 4) {
4517 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4518 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4519 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4520 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4521 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4522 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4523 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4524 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4525 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4526 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4527 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4528 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4529 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004530
Alex Deucherb7eff392011-07-08 11:44:56 -04004531 if (rdev->num_crtc >= 6) {
4532 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4533 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4534 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4535 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4536 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4537 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4538 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4539 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4540 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4541 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4542 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4543 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4544 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004545
Alex Deucher6f34be52010-11-21 10:59:01 -05004546 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004547 tmp = RREG32(DC_HPD1_INT_CONTROL);
4548 tmp |= DC_HPDx_INT_ACK;
4549 WREG32(DC_HPD1_INT_CONTROL, tmp);
4550 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004551 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004552 tmp = RREG32(DC_HPD2_INT_CONTROL);
4553 tmp |= DC_HPDx_INT_ACK;
4554 WREG32(DC_HPD2_INT_CONTROL, tmp);
4555 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004556 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004557 tmp = RREG32(DC_HPD3_INT_CONTROL);
4558 tmp |= DC_HPDx_INT_ACK;
4559 WREG32(DC_HPD3_INT_CONTROL, tmp);
4560 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004561 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004562 tmp = RREG32(DC_HPD4_INT_CONTROL);
4563 tmp |= DC_HPDx_INT_ACK;
4564 WREG32(DC_HPD4_INT_CONTROL, tmp);
4565 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004566 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004567 tmp = RREG32(DC_HPD5_INT_CONTROL);
4568 tmp |= DC_HPDx_INT_ACK;
4569 WREG32(DC_HPD5_INT_CONTROL, tmp);
4570 }
Alex Deucher6f34be52010-11-21 10:59:01 -05004571 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
Alex Deucher45f9a392010-03-24 13:55:51 -04004572 tmp = RREG32(DC_HPD5_INT_CONTROL);
4573 tmp |= DC_HPDx_INT_ACK;
4574 WREG32(DC_HPD6_INT_CONTROL, tmp);
4575 }
Alex Deucherf122c612012-03-30 08:59:57 -04004576 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4577 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4578 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4579 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4580 }
4581 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4582 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4583 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4584 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4585 }
4586 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4587 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4588 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4589 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4590 }
4591 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4592 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4593 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4594 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4595 }
4596 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4597 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4598 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4599 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4600 }
4601 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4602 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4603 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4604 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4605 }
Alex Deucher45f9a392010-03-24 13:55:51 -04004606}
4607
Lauri Kasanen1109ca02012-08-31 13:43:50 -04004608static void evergreen_irq_disable(struct radeon_device *rdev)
Alex Deucher45f9a392010-03-24 13:55:51 -04004609{
Alex Deucher45f9a392010-03-24 13:55:51 -04004610 r600_disable_interrupts(rdev);
4611 /* Wait and acknowledge irq */
4612 mdelay(1);
Alex Deucher6f34be52010-11-21 10:59:01 -05004613 evergreen_irq_ack(rdev);
Alex Deucher45f9a392010-03-24 13:55:51 -04004614 evergreen_disable_interrupt_state(rdev);
4615}
4616
Alex Deucher755d8192011-03-02 20:07:34 -05004617void evergreen_irq_suspend(struct radeon_device *rdev)
Alex Deucher45f9a392010-03-24 13:55:51 -04004618{
4619 evergreen_irq_disable(rdev);
4620 r600_rlc_stop(rdev);
4621}
4622
Andi Kleencbdd4502011-10-13 16:08:46 -07004623static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
Alex Deucher45f9a392010-03-24 13:55:51 -04004624{
4625 u32 wptr, tmp;
4626
Alex Deucher724c80e2010-08-27 18:25:25 -04004627 if (rdev->wb.enabled)
Cédric Cano204ae242011-04-19 11:07:13 -04004628 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
Alex Deucher724c80e2010-08-27 18:25:25 -04004629 else
4630 wptr = RREG32(IH_RB_WPTR);
Alex Deucher45f9a392010-03-24 13:55:51 -04004631
4632 if (wptr & RB_OVERFLOW) {
4633 /* When a ring buffer overflow happen start parsing interrupt
4634 * from the last not overwritten vector (wptr + 16). Hopefully
4635 * this should allow us to catchup.
4636 */
4637 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4638 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4639 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4640 tmp = RREG32(IH_RB_CNTL);
4641 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4642 WREG32(IH_RB_CNTL, tmp);
4643 }
4644 return (wptr & rdev->ih.ptr_mask);
4645}
4646
4647int evergreen_irq_process(struct radeon_device *rdev)
4648{
Dave Airlie682f1a52011-06-18 03:59:51 +00004649 u32 wptr;
4650 u32 rptr;
Alex Deucher45f9a392010-03-24 13:55:51 -04004651 u32 src_id, src_data;
4652 u32 ring_index;
Alex Deucher45f9a392010-03-24 13:55:51 -04004653 bool queue_hotplug = false;
Alex Deucherf122c612012-03-30 08:59:57 -04004654 bool queue_hdmi = false;
Alex Deucherdc50ba72013-06-26 00:33:35 -04004655 bool queue_thermal = false;
Alex Deucher54e2e492013-06-13 18:26:25 -04004656 u32 status, addr;
Alex Deucher45f9a392010-03-24 13:55:51 -04004657
Dave Airlie682f1a52011-06-18 03:59:51 +00004658 if (!rdev->ih.enabled || rdev->shutdown)
Alex Deucher45f9a392010-03-24 13:55:51 -04004659 return IRQ_NONE;
4660
Dave Airlie682f1a52011-06-18 03:59:51 +00004661 wptr = evergreen_get_ih_wptr(rdev);
Christian Koenigc20dc362012-05-16 21:45:24 +02004662
4663restart_ih:
4664 /* is somebody else already processing irqs? */
4665 if (atomic_xchg(&rdev->ih.lock, 1))
4666 return IRQ_NONE;
4667
Dave Airlie682f1a52011-06-18 03:59:51 +00004668 rptr = rdev->ih.rptr;
4669 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
Alex Deucher45f9a392010-03-24 13:55:51 -04004670
Benjamin Herrenschmidt964f6642011-07-13 16:28:19 +10004671 /* Order reading of wptr vs. reading of IH ring data */
4672 rmb();
4673
Alex Deucher45f9a392010-03-24 13:55:51 -04004674 /* display interrupts */
Alex Deucher6f34be52010-11-21 10:59:01 -05004675 evergreen_irq_ack(rdev);
Alex Deucher45f9a392010-03-24 13:55:51 -04004676
Alex Deucher45f9a392010-03-24 13:55:51 -04004677 while (rptr != wptr) {
4678 /* wptr/rptr are in bytes! */
4679 ring_index = rptr / 4;
Alex Deucher0f234f5f2011-02-13 19:06:33 -05004680 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4681 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
Alex Deucher45f9a392010-03-24 13:55:51 -04004682
4683 switch (src_id) {
4684 case 1: /* D1 vblank/vline */
4685 switch (src_data) {
4686 case 0: /* D1 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004687 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
Alex Deucher6f34be52010-11-21 10:59:01 -05004688 if (rdev->irq.crtc_vblank_int[0]) {
4689 drm_handle_vblank(rdev->ddev, 0);
4690 rdev->pm.vblank_sync = true;
4691 wake_up(&rdev->irq.vblank_queue);
4692 }
Christian Koenig736fc372012-05-17 19:52:00 +02004693 if (atomic_read(&rdev->irq.pflip[0]))
Mario Kleiner3e4ea742010-11-21 10:59:02 -05004694 radeon_crtc_handle_flip(rdev, 0);
Alex Deucher6f34be52010-11-21 10:59:01 -05004695 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004696 DRM_DEBUG("IH: D1 vblank\n");
4697 }
4698 break;
4699 case 1: /* D1 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004700 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4701 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004702 DRM_DEBUG("IH: D1 vline\n");
4703 }
4704 break;
4705 default:
4706 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4707 break;
4708 }
4709 break;
4710 case 2: /* D2 vblank/vline */
4711 switch (src_data) {
4712 case 0: /* D2 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004713 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
Alex Deucher6f34be52010-11-21 10:59:01 -05004714 if (rdev->irq.crtc_vblank_int[1]) {
4715 drm_handle_vblank(rdev->ddev, 1);
4716 rdev->pm.vblank_sync = true;
4717 wake_up(&rdev->irq.vblank_queue);
4718 }
Christian Koenig736fc372012-05-17 19:52:00 +02004719 if (atomic_read(&rdev->irq.pflip[1]))
Mario Kleiner3e4ea742010-11-21 10:59:02 -05004720 radeon_crtc_handle_flip(rdev, 1);
Alex Deucher6f34be52010-11-21 10:59:01 -05004721 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004722 DRM_DEBUG("IH: D2 vblank\n");
4723 }
4724 break;
4725 case 1: /* D2 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004726 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4727 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004728 DRM_DEBUG("IH: D2 vline\n");
4729 }
4730 break;
4731 default:
4732 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4733 break;
4734 }
4735 break;
4736 case 3: /* D3 vblank/vline */
4737 switch (src_data) {
4738 case 0: /* D3 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004739 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4740 if (rdev->irq.crtc_vblank_int[2]) {
4741 drm_handle_vblank(rdev->ddev, 2);
4742 rdev->pm.vblank_sync = true;
4743 wake_up(&rdev->irq.vblank_queue);
4744 }
Christian Koenig736fc372012-05-17 19:52:00 +02004745 if (atomic_read(&rdev->irq.pflip[2]))
Alex Deucher6f34be52010-11-21 10:59:01 -05004746 radeon_crtc_handle_flip(rdev, 2);
4747 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004748 DRM_DEBUG("IH: D3 vblank\n");
4749 }
4750 break;
4751 case 1: /* D3 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004752 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4753 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004754 DRM_DEBUG("IH: D3 vline\n");
4755 }
4756 break;
4757 default:
4758 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4759 break;
4760 }
4761 break;
4762 case 4: /* D4 vblank/vline */
4763 switch (src_data) {
4764 case 0: /* D4 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004765 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4766 if (rdev->irq.crtc_vblank_int[3]) {
4767 drm_handle_vblank(rdev->ddev, 3);
4768 rdev->pm.vblank_sync = true;
4769 wake_up(&rdev->irq.vblank_queue);
4770 }
Christian Koenig736fc372012-05-17 19:52:00 +02004771 if (atomic_read(&rdev->irq.pflip[3]))
Alex Deucher6f34be52010-11-21 10:59:01 -05004772 radeon_crtc_handle_flip(rdev, 3);
4773 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004774 DRM_DEBUG("IH: D4 vblank\n");
4775 }
4776 break;
4777 case 1: /* D4 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004778 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4779 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004780 DRM_DEBUG("IH: D4 vline\n");
4781 }
4782 break;
4783 default:
4784 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4785 break;
4786 }
4787 break;
4788 case 5: /* D5 vblank/vline */
4789 switch (src_data) {
4790 case 0: /* D5 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004791 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4792 if (rdev->irq.crtc_vblank_int[4]) {
4793 drm_handle_vblank(rdev->ddev, 4);
4794 rdev->pm.vblank_sync = true;
4795 wake_up(&rdev->irq.vblank_queue);
4796 }
Christian Koenig736fc372012-05-17 19:52:00 +02004797 if (atomic_read(&rdev->irq.pflip[4]))
Alex Deucher6f34be52010-11-21 10:59:01 -05004798 radeon_crtc_handle_flip(rdev, 4);
4799 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004800 DRM_DEBUG("IH: D5 vblank\n");
4801 }
4802 break;
4803 case 1: /* D5 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004804 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4805 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004806 DRM_DEBUG("IH: D5 vline\n");
4807 }
4808 break;
4809 default:
4810 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4811 break;
4812 }
4813 break;
4814 case 6: /* D6 vblank/vline */
4815 switch (src_data) {
4816 case 0: /* D6 vblank */
Alex Deucher6f34be52010-11-21 10:59:01 -05004817 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4818 if (rdev->irq.crtc_vblank_int[5]) {
4819 drm_handle_vblank(rdev->ddev, 5);
4820 rdev->pm.vblank_sync = true;
4821 wake_up(&rdev->irq.vblank_queue);
4822 }
Christian Koenig736fc372012-05-17 19:52:00 +02004823 if (atomic_read(&rdev->irq.pflip[5]))
Alex Deucher6f34be52010-11-21 10:59:01 -05004824 radeon_crtc_handle_flip(rdev, 5);
4825 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004826 DRM_DEBUG("IH: D6 vblank\n");
4827 }
4828 break;
4829 case 1: /* D6 vline */
Alex Deucher6f34be52010-11-21 10:59:01 -05004830 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4831 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004832 DRM_DEBUG("IH: D6 vline\n");
4833 }
4834 break;
4835 default:
4836 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4837 break;
4838 }
4839 break;
4840 case 42: /* HPD hotplug */
4841 switch (src_data) {
4842 case 0:
Alex Deucher6f34be52010-11-21 10:59:01 -05004843 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4844 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004845 queue_hotplug = true;
4846 DRM_DEBUG("IH: HPD1\n");
4847 }
4848 break;
4849 case 1:
Alex Deucher6f34be52010-11-21 10:59:01 -05004850 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4851 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004852 queue_hotplug = true;
4853 DRM_DEBUG("IH: HPD2\n");
4854 }
4855 break;
4856 case 2:
Alex Deucher6f34be52010-11-21 10:59:01 -05004857 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4858 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004859 queue_hotplug = true;
4860 DRM_DEBUG("IH: HPD3\n");
4861 }
4862 break;
4863 case 3:
Alex Deucher6f34be52010-11-21 10:59:01 -05004864 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4865 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004866 queue_hotplug = true;
4867 DRM_DEBUG("IH: HPD4\n");
4868 }
4869 break;
4870 case 4:
Alex Deucher6f34be52010-11-21 10:59:01 -05004871 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4872 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004873 queue_hotplug = true;
4874 DRM_DEBUG("IH: HPD5\n");
4875 }
4876 break;
4877 case 5:
Alex Deucher6f34be52010-11-21 10:59:01 -05004878 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4879 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
Alex Deucher45f9a392010-03-24 13:55:51 -04004880 queue_hotplug = true;
4881 DRM_DEBUG("IH: HPD6\n");
4882 }
4883 break;
4884 default:
4885 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4886 break;
4887 }
4888 break;
Alex Deucherf122c612012-03-30 08:59:57 -04004889 case 44: /* hdmi */
4890 switch (src_data) {
4891 case 0:
4892 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4893 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4894 queue_hdmi = true;
4895 DRM_DEBUG("IH: HDMI0\n");
4896 }
4897 break;
4898 case 1:
4899 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4900 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4901 queue_hdmi = true;
4902 DRM_DEBUG("IH: HDMI1\n");
4903 }
4904 break;
4905 case 2:
4906 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4907 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4908 queue_hdmi = true;
4909 DRM_DEBUG("IH: HDMI2\n");
4910 }
4911 break;
4912 case 3:
4913 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4914 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4915 queue_hdmi = true;
4916 DRM_DEBUG("IH: HDMI3\n");
4917 }
4918 break;
4919 case 4:
4920 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4921 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4922 queue_hdmi = true;
4923 DRM_DEBUG("IH: HDMI4\n");
4924 }
4925 break;
4926 case 5:
4927 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4928 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4929 queue_hdmi = true;
4930 DRM_DEBUG("IH: HDMI5\n");
4931 }
4932 break;
4933 default:
4934 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4935 break;
4936 }
Christian Königf2ba57b2013-04-08 12:41:29 +02004937 case 124: /* UVD */
4938 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4939 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
Alex Deucherf122c612012-03-30 08:59:57 -04004940 break;
Christian Königae133a12012-09-18 15:30:44 -04004941 case 146:
4942 case 147:
Alex Deucher54e2e492013-06-13 18:26:25 -04004943 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4944 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
Christian Königae133a12012-09-18 15:30:44 -04004945 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4946 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher54e2e492013-06-13 18:26:25 -04004947 addr);
Christian Königae133a12012-09-18 15:30:44 -04004948 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher54e2e492013-06-13 18:26:25 -04004949 status);
4950 cayman_vm_decode_fault(rdev, status, addr);
Christian Königae133a12012-09-18 15:30:44 -04004951 /* reset addr and status */
4952 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4953 break;
Alex Deucher45f9a392010-03-24 13:55:51 -04004954 case 176: /* CP_INT in ring buffer */
4955 case 177: /* CP_INT in IB1 */
4956 case 178: /* CP_INT in IB2 */
4957 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
Alex Deucher74652802011-08-25 13:39:48 -04004958 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
Alex Deucher45f9a392010-03-24 13:55:51 -04004959 break;
4960 case 181: /* CP EOP event */
4961 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher1b370782011-11-17 20:13:28 -05004962 if (rdev->family >= CHIP_CAYMAN) {
4963 switch (src_data) {
4964 case 0:
4965 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4966 break;
4967 case 1:
4968 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4969 break;
4970 case 2:
4971 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4972 break;
4973 }
4974 } else
4975 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
Alex Deucher45f9a392010-03-24 13:55:51 -04004976 break;
Alex Deucher233d1ad2012-12-04 15:25:59 -05004977 case 224: /* DMA trap event */
4978 DRM_DEBUG("IH: DMA trap\n");
4979 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4980 break;
Alex Deucherdc50ba72013-06-26 00:33:35 -04004981 case 230: /* thermal low to high */
4982 DRM_DEBUG("IH: thermal low to high\n");
4983 rdev->pm.dpm.thermal.high_to_low = false;
4984 queue_thermal = true;
4985 break;
4986 case 231: /* thermal high to low */
4987 DRM_DEBUG("IH: thermal high to low\n");
4988 rdev->pm.dpm.thermal.high_to_low = true;
4989 queue_thermal = true;
4990 break;
Alex Deucher2031f772010-04-22 12:52:11 -04004991 case 233: /* GUI IDLE */
Ilija Hadzic303c8052011-06-07 14:54:48 -04004992 DRM_DEBUG("IH: GUI idle\n");
Alex Deucher2031f772010-04-22 12:52:11 -04004993 break;
Alex Deucherf60cbd12012-12-04 15:27:33 -05004994 case 244: /* DMA trap event */
4995 if (rdev->family >= CHIP_CAYMAN) {
4996 DRM_DEBUG("IH: DMA1 trap\n");
4997 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4998 }
4999 break;
Alex Deucher45f9a392010-03-24 13:55:51 -04005000 default:
5001 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5002 break;
5003 }
5004
5005 /* wptr/rptr are in bytes! */
5006 rptr += 16;
5007 rptr &= rdev->ih.ptr_mask;
5008 }
Alex Deucher45f9a392010-03-24 13:55:51 -04005009 if (queue_hotplug)
Tejun Heo32c87fc2011-01-03 14:49:32 +01005010 schedule_work(&rdev->hotplug_work);
Alex Deucherf122c612012-03-30 08:59:57 -04005011 if (queue_hdmi)
5012 schedule_work(&rdev->audio_work);
Alex Deucherdc50ba72013-06-26 00:33:35 -04005013 if (queue_thermal && rdev->pm.dpm_enabled)
5014 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deucher45f9a392010-03-24 13:55:51 -04005015 rdev->ih.rptr = rptr;
5016 WREG32(IH_RB_RPTR, rdev->ih.rptr);
Christian Koenigc20dc362012-05-16 21:45:24 +02005017 atomic_set(&rdev->ih.lock, 0);
5018
5019 /* make sure wptr hasn't changed while processing */
5020 wptr = evergreen_get_ih_wptr(rdev);
5021 if (wptr != rptr)
5022 goto restart_ih;
5023
Alex Deucher45f9a392010-03-24 13:55:51 -04005024 return IRQ_HANDLED;
5025}
5026
Alex Deucher233d1ad2012-12-04 15:25:59 -05005027/**
5028 * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
5029 *
5030 * @rdev: radeon_device pointer
5031 * @fence: radeon fence object
5032 *
5033 * Add a DMA fence packet to the ring to write
5034 * the fence seq number and DMA trap packet to generate
5035 * an interrupt if needed (evergreen-SI).
5036 */
5037void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
5038 struct radeon_fence *fence)
5039{
5040 struct radeon_ring *ring = &rdev->ring[fence->ring];
5041 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
5042 /* write the fence */
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005043 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005044 radeon_ring_write(ring, addr & 0xfffffffc);
5045 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
5046 radeon_ring_write(ring, fence->seq);
5047 /* generate an interrupt */
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005048 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005049 /* flush HDP */
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005050 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
Alex Deucher4b681c22013-01-03 19:54:34 -05005051 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005052 radeon_ring_write(ring, 1);
5053}
5054
5055/**
5056 * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
5057 *
5058 * @rdev: radeon_device pointer
5059 * @ib: IB object to schedule
5060 *
5061 * Schedule an IB in the DMA ring (evergreen).
5062 */
5063void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
5064 struct radeon_ib *ib)
5065{
5066 struct radeon_ring *ring = &rdev->ring[ib->ring];
5067
5068 if (rdev->wb.enabled) {
5069 u32 next_rptr = ring->wptr + 4;
5070 while ((next_rptr & 7) != 5)
5071 next_rptr++;
5072 next_rptr += 3;
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005073 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005074 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5075 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5076 radeon_ring_write(ring, next_rptr);
5077 }
5078
5079 /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5080 * Pad as necessary with NOPs.
5081 */
5082 while ((ring->wptr & 7) != 5)
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005083 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5084 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005085 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5086 radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5087
5088}
5089
5090/**
5091 * evergreen_copy_dma - copy pages using the DMA engine
5092 *
5093 * @rdev: radeon_device pointer
5094 * @src_offset: src GPU address
5095 * @dst_offset: dst GPU address
5096 * @num_gpu_pages: number of GPU pages to xfer
5097 * @fence: radeon fence object
5098 *
5099 * Copy GPU paging using the DMA engine (evergreen-cayman).
5100 * Used by the radeon ttm implementation to move pages if
5101 * registered as the asic copy callback.
5102 */
5103int evergreen_copy_dma(struct radeon_device *rdev,
5104 uint64_t src_offset, uint64_t dst_offset,
5105 unsigned num_gpu_pages,
5106 struct radeon_fence **fence)
5107{
5108 struct radeon_semaphore *sem = NULL;
5109 int ring_index = rdev->asic->copy.dma_ring_index;
5110 struct radeon_ring *ring = &rdev->ring[ring_index];
5111 u32 size_in_dw, cur_size_in_dw;
5112 int i, num_loops;
5113 int r = 0;
5114
5115 r = radeon_semaphore_create(rdev, &sem);
5116 if (r) {
5117 DRM_ERROR("radeon: moving bo (%d).\n", r);
5118 return r;
5119 }
5120
5121 size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5122 num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5123 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5124 if (r) {
5125 DRM_ERROR("radeon: moving bo (%d).\n", r);
5126 radeon_semaphore_free(rdev, &sem, NULL);
5127 return r;
5128 }
5129
5130 if (radeon_fence_need_sync(*fence, ring->idx)) {
5131 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5132 ring->idx);
5133 radeon_fence_note_sync(*fence, ring->idx);
5134 } else {
5135 radeon_semaphore_free(rdev, &sem, NULL);
5136 }
5137
5138 for (i = 0; i < num_loops; i++) {
5139 cur_size_in_dw = size_in_dw;
5140 if (cur_size_in_dw > 0xFFFFF)
5141 cur_size_in_dw = 0xFFFFF;
5142 size_in_dw -= cur_size_in_dw;
Jerome Glisse0fcb6152013-01-14 11:32:27 -05005143 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005144 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5145 radeon_ring_write(ring, src_offset & 0xfffffffc);
5146 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5147 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5148 src_offset += cur_size_in_dw * 4;
5149 dst_offset += cur_size_in_dw * 4;
5150 }
5151
5152 r = radeon_fence_emit(rdev, fence, ring->idx);
5153 if (r) {
5154 radeon_ring_unlock_undo(rdev, ring);
5155 return r;
5156 }
5157
5158 radeon_ring_unlock_commit(rdev, ring);
5159 radeon_semaphore_free(rdev, &sem, *fence);
5160
5161 return r;
5162}
5163
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005164static int evergreen_startup(struct radeon_device *rdev)
5165{
Christian Königf2ba57b2013-04-08 12:41:29 +02005166 struct radeon_ring *ring;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005167 int r;
5168
Alex Deucher9e46a482011-01-06 18:49:35 -05005169 /* enable pcie gen2 link */
Ilija Hadziccd540332011-09-20 10:22:57 -04005170 evergreen_pcie_gen2_enable(rdev);
Alex Deucherf52382d2013-02-15 11:02:50 -05005171 /* enable aspm */
5172 evergreen_program_aspm(rdev);
Alex Deucher9e46a482011-01-06 18:49:35 -05005173
Alex Deucher6fab3feb2013-08-04 12:13:17 -04005174 evergreen_mc_program(rdev);
5175
Alex Deucher0af62b02011-01-06 21:19:31 -05005176 if (ASIC_IS_DCE5(rdev)) {
5177 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5178 r = ni_init_microcode(rdev);
5179 if (r) {
5180 DRM_ERROR("Failed to load firmware!\n");
5181 return r;
5182 }
5183 }
Alex Deucher755d8192011-03-02 20:07:34 -05005184 r = ni_mc_load_microcode(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005185 if (r) {
Alex Deucher0af62b02011-01-06 21:19:31 -05005186 DRM_ERROR("Failed to load MC firmware!\n");
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005187 return r;
5188 }
Alex Deucher0af62b02011-01-06 21:19:31 -05005189 } else {
5190 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5191 r = r600_init_microcode(rdev);
5192 if (r) {
5193 DRM_ERROR("Failed to load firmware!\n");
5194 return r;
5195 }
5196 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005197 }
Alex Deucherfe251e22010-03-24 13:36:43 -04005198
Alex Deucher16cdf042011-10-28 10:30:02 -04005199 r = r600_vram_scratch_init(rdev);
5200 if (r)
5201 return r;
5202
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005203 if (rdev->flags & RADEON_IS_AGP) {
Alex Deucher0fcdb612010-03-24 13:20:41 -04005204 evergreen_agp_enable(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005205 } else {
5206 r = evergreen_pcie_gart_enable(rdev);
5207 if (r)
5208 return r;
5209 }
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005210 evergreen_gpu_init(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005211
Alex Deucher2948f5e2013-04-12 13:52:52 -04005212 /* allocate rlc buffers */
5213 if (rdev->flags & RADEON_IS_IGP) {
5214 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
Alex Deucher1fd11772013-04-17 17:53:50 -04005215 rdev->rlc.reg_list_size =
5216 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
Alex Deucher2948f5e2013-04-12 13:52:52 -04005217 rdev->rlc.cs_data = evergreen_cs_data;
5218 r = sumo_rlc_init(rdev);
5219 if (r) {
5220 DRM_ERROR("Failed to init rlc BOs!\n");
5221 return r;
5222 }
5223 }
5224
Alex Deucher724c80e2010-08-27 18:25:25 -04005225 /* allocate wb buffer */
5226 r = radeon_wb_init(rdev);
5227 if (r)
5228 return r;
5229
Jerome Glisse30eb77f2011-11-20 20:45:34 +00005230 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5231 if (r) {
5232 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5233 return r;
5234 }
5235
Alex Deucher233d1ad2012-12-04 15:25:59 -05005236 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5237 if (r) {
5238 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5239 return r;
5240 }
5241
Christian Königf2ba57b2013-04-08 12:41:29 +02005242 r = rv770_uvd_resume(rdev);
5243 if (!r) {
5244 r = radeon_fence_driver_start_ring(rdev,
5245 R600_RING_TYPE_UVD_INDEX);
5246 if (r)
5247 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5248 }
5249
5250 if (r)
5251 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5252
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005253 /* Enable IRQ */
Adis Hamziće49f3952013-06-02 16:47:54 +02005254 if (!rdev->irq.installed) {
5255 r = radeon_irq_kms_init(rdev);
5256 if (r)
5257 return r;
5258 }
5259
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005260 r = r600_irq_init(rdev);
5261 if (r) {
5262 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5263 radeon_irq_kms_fini(rdev);
5264 return r;
5265 }
Alex Deucher45f9a392010-03-24 13:55:51 -04005266 evergreen_irq_set(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005267
Christian Königf2ba57b2013-04-08 12:41:29 +02005268 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Christian Könige32eb502011-10-23 12:56:27 +02005269 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
Alex Deucher78c55602011-11-17 14:25:56 -05005270 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +02005271 RADEON_CP_PACKET2);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005272 if (r)
5273 return r;
Alex Deucher233d1ad2012-12-04 15:25:59 -05005274
5275 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5276 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5277 DMA_RB_RPTR, DMA_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +02005278 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
Alex Deucher233d1ad2012-12-04 15:25:59 -05005279 if (r)
5280 return r;
5281
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005282 r = evergreen_cp_load_microcode(rdev);
5283 if (r)
5284 return r;
Alex Deucherfe251e22010-03-24 13:36:43 -04005285 r = evergreen_cp_resume(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005286 if (r)
5287 return r;
Alex Deucher233d1ad2012-12-04 15:25:59 -05005288 r = r600_dma_resume(rdev);
5289 if (r)
5290 return r;
Alex Deucherfe251e22010-03-24 13:36:43 -04005291
Christian Königf2ba57b2013-04-08 12:41:29 +02005292 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5293 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +02005294 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian Königf2ba57b2013-04-08 12:41:29 +02005295 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +02005296 RADEON_CP_PACKET2);
Christian Königf2ba57b2013-04-08 12:41:29 +02005297 if (!r)
Alex Deucher5e884f62013-08-06 11:39:38 -04005298 r = r600_uvd_init(rdev, true);
Christian Königf2ba57b2013-04-08 12:41:29 +02005299
5300 if (r)
5301 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5302 }
5303
Christian König2898c342012-07-05 11:55:34 +02005304 r = radeon_ib_pool_init(rdev);
5305 if (r) {
5306 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
Jerome Glisseb15ba512011-11-15 11:48:34 -05005307 return r;
Christian König2898c342012-07-05 11:55:34 +02005308 }
Jerome Glisseb15ba512011-11-15 11:48:34 -05005309
Rafał Miłecki69d2ae52011-12-07 23:32:24 +01005310 r = r600_audio_init(rdev);
5311 if (r) {
5312 DRM_ERROR("radeon: audio init failed\n");
Jerome Glisseb15ba512011-11-15 11:48:34 -05005313 return r;
5314 }
5315
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005316 return 0;
5317}
5318
5319int evergreen_resume(struct radeon_device *rdev)
5320{
5321 int r;
5322
Alex Deucher86f5c9e2010-12-20 12:35:04 -05005323 /* reset the asic, the gfx blocks are often in a bad state
5324 * after the driver is unloaded or after a resume
5325 */
5326 if (radeon_asic_reset(rdev))
5327 dev_warn(rdev->dev, "GPU reset failed !\n");
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005328 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5329 * posting will perform necessary task to bring back GPU into good
5330 * shape.
5331 */
5332 /* post card */
5333 atom_asic_init(rdev->mode_info.atom_context);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005334
Alex Deucherd4788db2013-02-28 14:40:09 -05005335 /* init golden registers */
5336 evergreen_init_golden_registers(rdev);
5337
Jerome Glisseb15ba512011-11-15 11:48:34 -05005338 rdev->accel_working = true;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005339 r = evergreen_startup(rdev);
5340 if (r) {
Alex Deucher755d8192011-03-02 20:07:34 -05005341 DRM_ERROR("evergreen startup failed on resume\n");
Jerome Glisse6b7746e2012-02-20 17:57:20 -05005342 rdev->accel_working = false;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005343 return r;
5344 }
Alex Deucherfe251e22010-03-24 13:36:43 -04005345
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005346 return r;
5347
5348}
5349
5350int evergreen_suspend(struct radeon_device *rdev)
5351{
Rafał Miłecki69d2ae52011-12-07 23:32:24 +01005352 r600_audio_fini(rdev);
Christian König2858c002013-08-01 17:34:07 +02005353 r600_uvd_stop(rdev);
Christian Königf2ba57b2013-04-08 12:41:29 +02005354 radeon_uvd_suspend(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005355 r700_cp_stop(rdev);
Alex Deucher233d1ad2012-12-04 15:25:59 -05005356 r600_dma_stop(rdev);
Alex Deucher45f9a392010-03-24 13:55:51 -04005357 evergreen_irq_suspend(rdev);
Alex Deucher724c80e2010-08-27 18:25:25 -04005358 radeon_wb_disable(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005359 evergreen_pcie_gart_disable(rdev);
Alex Deucherd7ccd8f2010-09-09 11:33:36 -04005360
5361 return 0;
5362}
5363
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005364/* Plan is to move initialization in that function and use
5365 * helper function so that radeon_device_init pretty much
5366 * do nothing more than calling asic specific function. This
5367 * should also allow to remove a bunch of callback function
5368 * like vram_info.
5369 */
5370int evergreen_init(struct radeon_device *rdev)
5371{
5372 int r;
5373
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005374 /* Read BIOS */
5375 if (!radeon_get_bios(rdev)) {
5376 if (ASIC_IS_AVIVO(rdev))
5377 return -EINVAL;
5378 }
5379 /* Must be an ATOMBIOS */
5380 if (!rdev->is_atom_bios) {
Alex Deucher755d8192011-03-02 20:07:34 -05005381 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005382 return -EINVAL;
5383 }
5384 r = radeon_atombios_init(rdev);
5385 if (r)
5386 return r;
Alex Deucher86f5c9e2010-12-20 12:35:04 -05005387 /* reset the asic, the gfx blocks are often in a bad state
5388 * after the driver is unloaded or after a resume
5389 */
5390 if (radeon_asic_reset(rdev))
5391 dev_warn(rdev->dev, "GPU reset failed !\n");
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005392 /* Post card if necessary */
Alex Deucherfd909c32011-01-11 18:08:59 -05005393 if (!radeon_card_posted(rdev)) {
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005394 if (!rdev->bios) {
5395 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5396 return -EINVAL;
5397 }
5398 DRM_INFO("GPU not posted. posting now...\n");
5399 atom_asic_init(rdev->mode_info.atom_context);
5400 }
Alex Deucherd4788db2013-02-28 14:40:09 -05005401 /* init golden registers */
5402 evergreen_init_golden_registers(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005403 /* Initialize scratch registers */
5404 r600_scratch_init(rdev);
5405 /* Initialize surface registers */
5406 radeon_surface_init(rdev);
5407 /* Initialize clocks */
5408 radeon_get_clock_info(rdev->ddev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005409 /* Fence driver */
5410 r = radeon_fence_driver_init(rdev);
5411 if (r)
5412 return r;
Jerome Glissed594e462010-02-17 21:54:29 +00005413 /* initialize AGP */
5414 if (rdev->flags & RADEON_IS_AGP) {
5415 r = radeon_agp_init(rdev);
5416 if (r)
5417 radeon_agp_disable(rdev);
5418 }
5419 /* initialize memory controller */
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005420 r = evergreen_mc_init(rdev);
5421 if (r)
5422 return r;
5423 /* Memory manager */
5424 r = radeon_bo_init(rdev);
5425 if (r)
5426 return r;
Alex Deucher45f9a392010-03-24 13:55:51 -04005427
Christian Könige32eb502011-10-23 12:56:27 +02005428 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5429 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005430
Alex Deucher233d1ad2012-12-04 15:25:59 -05005431 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5432 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5433
Christian Königf2ba57b2013-04-08 12:41:29 +02005434 r = radeon_uvd_init(rdev);
5435 if (!r) {
5436 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5437 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5438 4096);
5439 }
5440
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005441 rdev->ih.ring_obj = NULL;
5442 r600_ih_ring_init(rdev, 64 * 1024);
5443
5444 r = r600_pcie_gart_init(rdev);
5445 if (r)
5446 return r;
Alex Deucher0fcdb612010-03-24 13:20:41 -04005447
Alex Deucher148a03b2010-06-03 19:00:03 -04005448 rdev->accel_working = true;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005449 r = evergreen_startup(rdev);
5450 if (r) {
Alex Deucherfe251e22010-03-24 13:36:43 -04005451 dev_err(rdev->dev, "disabling GPU acceleration\n");
5452 r700_cp_fini(rdev);
Alex Deucher233d1ad2012-12-04 15:25:59 -05005453 r600_dma_fini(rdev);
Alex Deucherfe251e22010-03-24 13:36:43 -04005454 r600_irq_fini(rdev);
Alex Deucher2948f5e2013-04-12 13:52:52 -04005455 if (rdev->flags & RADEON_IS_IGP)
5456 sumo_rlc_fini(rdev);
Alex Deucher724c80e2010-08-27 18:25:25 -04005457 radeon_wb_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02005458 radeon_ib_pool_fini(rdev);
Alex Deucherfe251e22010-03-24 13:36:43 -04005459 radeon_irq_kms_fini(rdev);
Alex Deucher0fcdb612010-03-24 13:20:41 -04005460 evergreen_pcie_gart_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005461 rdev->accel_working = false;
5462 }
Alex Deucher77e00f22011-12-21 11:58:17 -05005463
5464 /* Don't start up if the MC ucode is missing on BTC parts.
5465 * The default clocks and voltages before the MC ucode
5466 * is loaded are not suffient for advanced operations.
5467 */
5468 if (ASIC_IS_DCE5(rdev)) {
5469 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5470 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5471 return -EINVAL;
5472 }
5473 }
5474
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005475 return 0;
5476}
5477
5478void evergreen_fini(struct radeon_device *rdev)
5479{
Rafał Miłecki69d2ae52011-12-07 23:32:24 +01005480 r600_audio_fini(rdev);
Alex Deucher45f9a392010-03-24 13:55:51 -04005481 r700_cp_fini(rdev);
Alex Deucher233d1ad2012-12-04 15:25:59 -05005482 r600_dma_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005483 r600_irq_fini(rdev);
Alex Deucher2948f5e2013-04-12 13:52:52 -04005484 if (rdev->flags & RADEON_IS_IGP)
5485 sumo_rlc_fini(rdev);
Alex Deucher724c80e2010-08-27 18:25:25 -04005486 radeon_wb_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02005487 radeon_ib_pool_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005488 radeon_irq_kms_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005489 evergreen_pcie_gart_fini(rdev);
Christian König2858c002013-08-01 17:34:07 +02005490 r600_uvd_stop(rdev);
Christian Königf2ba57b2013-04-08 12:41:29 +02005491 radeon_uvd_fini(rdev);
Alex Deucher16cdf042011-10-28 10:30:02 -04005492 r600_vram_scratch_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005493 radeon_gem_fini(rdev);
5494 radeon_fence_driver_fini(rdev);
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005495 radeon_agp_fini(rdev);
5496 radeon_bo_fini(rdev);
5497 radeon_atombios_fini(rdev);
5498 kfree(rdev->bios);
5499 rdev->bios = NULL;
Alex Deucherbcc1c2a2010-01-12 17:54:34 -05005500}
Alex Deucher9e46a482011-01-06 18:49:35 -05005501
Ilija Hadzicb07759b2011-09-20 10:22:58 -04005502void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
Alex Deucher9e46a482011-01-06 18:49:35 -05005503{
Kleber Sacilotto de Souza7e0e4192013-05-03 19:43:13 -03005504 u32 link_width_cntl, speed_cntl;
Alex Deucher9e46a482011-01-06 18:49:35 -05005505
Alex Deucherd42dd572011-01-12 20:05:11 -05005506 if (radeon_pcie_gen2 == 0)
5507 return;
5508
Alex Deucher9e46a482011-01-06 18:49:35 -05005509 if (rdev->flags & RADEON_IS_IGP)
5510 return;
5511
5512 if (!(rdev->flags & RADEON_IS_PCIE))
5513 return;
5514
5515 /* x2 cards have a special sequence */
5516 if (ASIC_IS_X2(rdev))
5517 return;
5518
Kleber Sacilotto de Souza7e0e4192013-05-03 19:43:13 -03005519 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5520 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
Dave Airlie197bbb32012-06-27 08:35:54 +01005521 return;
5522
Alex Deucher492d2b62012-10-25 16:06:59 -04005523 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
Alex Deucher3691fee2012-10-08 17:46:27 -04005524 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5525 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5526 return;
5527 }
5528
Dave Airlie197bbb32012-06-27 08:35:54 +01005529 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5530
Alex Deucher9e46a482011-01-06 18:49:35 -05005531 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5532 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5533
Alex Deucher492d2b62012-10-25 16:06:59 -04005534 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005535 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
Alex Deucher492d2b62012-10-25 16:06:59 -04005536 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005537
Alex Deucher492d2b62012-10-25 16:06:59 -04005538 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005539 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
Alex Deucher492d2b62012-10-25 16:06:59 -04005540 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005541
Alex Deucher492d2b62012-10-25 16:06:59 -04005542 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005543 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
Alex Deucher492d2b62012-10-25 16:06:59 -04005544 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005545
Alex Deucher492d2b62012-10-25 16:06:59 -04005546 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005547 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
Alex Deucher492d2b62012-10-25 16:06:59 -04005548 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005549
Alex Deucher492d2b62012-10-25 16:06:59 -04005550 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005551 speed_cntl |= LC_GEN2_EN_STRAP;
Alex Deucher492d2b62012-10-25 16:06:59 -04005552 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005553
5554 } else {
Alex Deucher492d2b62012-10-25 16:06:59 -04005555 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
Alex Deucher9e46a482011-01-06 18:49:35 -05005556 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5557 if (1)
5558 link_width_cntl |= LC_UPCONFIGURE_DIS;
5559 else
5560 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
Alex Deucher492d2b62012-10-25 16:06:59 -04005561 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
Alex Deucher9e46a482011-01-06 18:49:35 -05005562 }
5563}
Alex Deucherf52382d2013-02-15 11:02:50 -05005564
5565void evergreen_program_aspm(struct radeon_device *rdev)
5566{
5567 u32 data, orig;
5568 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5569 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5570 /* fusion_platform = true
5571 * if the system is a fusion system
5572 * (APU or DGPU in a fusion system).
5573 * todo: check if the system is a fusion platform.
5574 */
5575 bool fusion_platform = false;
5576
Alex Deucher1294d4a2013-07-16 15:58:50 -04005577 if (radeon_aspm == 0)
5578 return;
5579
Alex Deucherf52382d2013-02-15 11:02:50 -05005580 if (!(rdev->flags & RADEON_IS_PCIE))
5581 return;
5582
5583 switch (rdev->family) {
5584 case CHIP_CYPRESS:
5585 case CHIP_HEMLOCK:
5586 case CHIP_JUNIPER:
5587 case CHIP_REDWOOD:
5588 case CHIP_CEDAR:
5589 case CHIP_SUMO:
5590 case CHIP_SUMO2:
5591 case CHIP_PALM:
5592 case CHIP_ARUBA:
5593 disable_l0s = true;
5594 break;
5595 default:
5596 disable_l0s = false;
5597 break;
5598 }
5599
5600 if (rdev->flags & RADEON_IS_IGP)
5601 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5602
5603 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5604 if (fusion_platform)
5605 data &= ~MULTI_PIF;
5606 else
5607 data |= MULTI_PIF;
5608 if (data != orig)
5609 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5610
5611 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5612 if (fusion_platform)
5613 data &= ~MULTI_PIF;
5614 else
5615 data |= MULTI_PIF;
5616 if (data != orig)
5617 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5618
5619 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5620 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5621 if (!disable_l0s) {
5622 if (rdev->family >= CHIP_BARTS)
5623 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5624 else
5625 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5626 }
5627
5628 if (!disable_l1) {
5629 if (rdev->family >= CHIP_BARTS)
5630 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5631 else
5632 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5633
5634 if (!disable_plloff_in_l1) {
5635 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5636 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5637 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5638 if (data != orig)
5639 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5640
5641 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5642 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5643 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5644 if (data != orig)
5645 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5646
5647 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5648 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5649 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5650 if (data != orig)
5651 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5652
5653 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5654 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5655 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5656 if (data != orig)
5657 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5658
5659 if (rdev->family >= CHIP_BARTS) {
5660 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5661 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5662 data |= PLL_RAMP_UP_TIME_0(4);
5663 if (data != orig)
5664 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5665
5666 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5667 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5668 data |= PLL_RAMP_UP_TIME_1(4);
5669 if (data != orig)
5670 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5671
5672 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5673 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5674 data |= PLL_RAMP_UP_TIME_0(4);
5675 if (data != orig)
5676 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5677
5678 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5679 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5680 data |= PLL_RAMP_UP_TIME_1(4);
5681 if (data != orig)
5682 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5683 }
5684
5685 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5686 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5687 data |= LC_DYN_LANES_PWR_STATE(3);
5688 if (data != orig)
5689 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5690
5691 if (rdev->family >= CHIP_BARTS) {
5692 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5693 data &= ~LS2_EXIT_TIME_MASK;
5694 data |= LS2_EXIT_TIME(1);
5695 if (data != orig)
5696 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5697
5698 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5699 data &= ~LS2_EXIT_TIME_MASK;
5700 data |= LS2_EXIT_TIME(1);
5701 if (data != orig)
5702 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5703 }
5704 }
5705 }
5706
5707 /* evergreen parts only */
5708 if (rdev->family < CHIP_BARTS)
5709 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5710
5711 if (pcie_lc_cntl != pcie_lc_cntl_old)
5712 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5713}