blob: a1b0da6b580859dfede1703ee822e8cab28fe91f [file] [log] [blame]
Alex Deucher43b3cd92012-03-20 17:18:00 -04001/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
Alex Deucher0f0de062012-03-20 17:18:17 -040024#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
Alex Deucher43b3cd92012-03-20 17:18:00 -040029#include "radeon.h"
30#include "radeon_asic.h"
David Howells760285e2012-10-02 18:01:07 +010031#include <drm/radeon_drm.h>
Alex Deucher43b3cd92012-03-20 17:18:00 -040032#include "sid.h"
33#include "atom.h"
Alex Deucher48c0c902012-03-20 17:18:19 -040034#include "si_blit_shaders.h"
Alex Deucher43b3cd92012-03-20 17:18:00 -040035
Alex Deucher0f0de062012-03-20 17:18:17 -040036#define SI_PFP_UCODE_SIZE 2144
37#define SI_PM4_UCODE_SIZE 2144
38#define SI_CE_UCODE_SIZE 2144
39#define SI_RLC_UCODE_SIZE 2048
40#define SI_MC_UCODE_SIZE 7769
Alex Deucherbcc7f5d2012-07-26 18:36:28 -040041#define OLAND_MC_UCODE_SIZE 7863
Alex Deucher0f0de062012-03-20 17:18:17 -040042
43MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54MODULE_FIRMWARE("radeon/VERDE_me.bin");
55MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
Alex Deucherbcc7f5d2012-07-26 18:36:28 -040058MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59MODULE_FIRMWARE("radeon/OLAND_me.bin");
60MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
Alex Deucherc04c00b2012-07-31 12:57:45 -040063MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
Alex Deucher0f0de062012-03-20 17:18:17 -040068
Alex Deucher25a857f2012-03-20 17:18:22 -040069extern int r600_ih_ring_alloc(struct radeon_device *rdev);
70extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher0a96d722012-03-20 17:18:11 -040071extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
Alex Deucherc476dde2012-03-20 17:18:12 -040072extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
73extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deucherca7db222012-03-20 17:18:30 -040074extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
Alex Deucher1c534672013-01-18 15:08:38 -050075extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
Alex Deucher014bb202013-01-18 19:36:20 -050076extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher0a96d722012-03-20 17:18:11 -040077
Alex Deucher205996c2013-03-01 17:08:42 -050078static const u32 tahiti_golden_rlc_registers[] =
79{
80 0xc424, 0xffffffff, 0x00601005,
81 0xc47c, 0xffffffff, 0x10104040,
82 0xc488, 0xffffffff, 0x0100000a,
83 0xc314, 0xffffffff, 0x00000800,
84 0xc30c, 0xffffffff, 0x800000f4,
85 0xf4a8, 0xffffffff, 0x00000000
86};
87
88static const u32 tahiti_golden_registers[] =
89{
90 0x9a10, 0x00010000, 0x00018208,
91 0x9830, 0xffffffff, 0x00000000,
92 0x9834, 0xf00fffff, 0x00000400,
93 0x9838, 0x0002021c, 0x00020200,
94 0xc78, 0x00000080, 0x00000000,
95 0xd030, 0x000300c0, 0x00800040,
96 0xd830, 0x000300c0, 0x00800040,
97 0x5bb0, 0x000000f0, 0x00000070,
98 0x5bc0, 0x00200000, 0x50100000,
99 0x7030, 0x31000311, 0x00000011,
100 0x277c, 0x00000003, 0x000007ff,
101 0x240c, 0x000007ff, 0x00000000,
102 0x8a14, 0xf000001f, 0x00000007,
103 0x8b24, 0xffffffff, 0x00ffffff,
104 0x8b10, 0x0000ff0f, 0x00000000,
105 0x28a4c, 0x07ffffff, 0x4e000000,
106 0x28350, 0x3f3f3fff, 0x2a00126a,
107 0x30, 0x000000ff, 0x0040,
108 0x34, 0x00000040, 0x00004040,
109 0x9100, 0x07ffffff, 0x03000000,
110 0x8e88, 0x01ff1f3f, 0x00000000,
111 0x8e84, 0x01ff1f3f, 0x00000000,
112 0x9060, 0x0000007f, 0x00000020,
113 0x9508, 0x00010000, 0x00010000,
114 0xac14, 0x00000200, 0x000002fb,
115 0xac10, 0xffffffff, 0x0000543b,
116 0xac0c, 0xffffffff, 0xa9210876,
117 0x88d0, 0xffffffff, 0x000fff40,
118 0x88d4, 0x0000001f, 0x00000010,
119 0x1410, 0x20000000, 0x20fffed8,
120 0x15c0, 0x000c0fc0, 0x000c0400
121};
122
123static const u32 tahiti_golden_registers2[] =
124{
125 0xc64, 0x00000001, 0x00000001
126};
127
128static const u32 pitcairn_golden_rlc_registers[] =
129{
130 0xc424, 0xffffffff, 0x00601004,
131 0xc47c, 0xffffffff, 0x10102020,
132 0xc488, 0xffffffff, 0x01000020,
133 0xc314, 0xffffffff, 0x00000800,
134 0xc30c, 0xffffffff, 0x800000a4
135};
136
137static const u32 pitcairn_golden_registers[] =
138{
139 0x9a10, 0x00010000, 0x00018208,
140 0x9830, 0xffffffff, 0x00000000,
141 0x9834, 0xf00fffff, 0x00000400,
142 0x9838, 0x0002021c, 0x00020200,
143 0xc78, 0x00000080, 0x00000000,
144 0xd030, 0x000300c0, 0x00800040,
145 0xd830, 0x000300c0, 0x00800040,
146 0x5bb0, 0x000000f0, 0x00000070,
147 0x5bc0, 0x00200000, 0x50100000,
148 0x7030, 0x31000311, 0x00000011,
149 0x2ae4, 0x00073ffe, 0x000022a2,
150 0x240c, 0x000007ff, 0x00000000,
151 0x8a14, 0xf000001f, 0x00000007,
152 0x8b24, 0xffffffff, 0x00ffffff,
153 0x8b10, 0x0000ff0f, 0x00000000,
154 0x28a4c, 0x07ffffff, 0x4e000000,
155 0x28350, 0x3f3f3fff, 0x2a00126a,
156 0x30, 0x000000ff, 0x0040,
157 0x34, 0x00000040, 0x00004040,
158 0x9100, 0x07ffffff, 0x03000000,
159 0x9060, 0x0000007f, 0x00000020,
160 0x9508, 0x00010000, 0x00010000,
161 0xac14, 0x000003ff, 0x000000f7,
162 0xac10, 0xffffffff, 0x00000000,
163 0xac0c, 0xffffffff, 0x32761054,
164 0x88d4, 0x0000001f, 0x00000010,
165 0x15c0, 0x000c0fc0, 0x000c0400
166};
167
168static const u32 verde_golden_rlc_registers[] =
169{
170 0xc424, 0xffffffff, 0x033f1005,
171 0xc47c, 0xffffffff, 0x10808020,
172 0xc488, 0xffffffff, 0x00800008,
173 0xc314, 0xffffffff, 0x00001000,
174 0xc30c, 0xffffffff, 0x80010014
175};
176
177static const u32 verde_golden_registers[] =
178{
179 0x9a10, 0x00010000, 0x00018208,
180 0x9830, 0xffffffff, 0x00000000,
181 0x9834, 0xf00fffff, 0x00000400,
182 0x9838, 0x0002021c, 0x00020200,
183 0xc78, 0x00000080, 0x00000000,
184 0xd030, 0x000300c0, 0x00800040,
185 0xd030, 0x000300c0, 0x00800040,
186 0xd830, 0x000300c0, 0x00800040,
187 0xd830, 0x000300c0, 0x00800040,
188 0x5bb0, 0x000000f0, 0x00000070,
189 0x5bc0, 0x00200000, 0x50100000,
190 0x7030, 0x31000311, 0x00000011,
191 0x2ae4, 0x00073ffe, 0x000022a2,
192 0x2ae4, 0x00073ffe, 0x000022a2,
193 0x2ae4, 0x00073ffe, 0x000022a2,
194 0x240c, 0x000007ff, 0x00000000,
195 0x240c, 0x000007ff, 0x00000000,
196 0x240c, 0x000007ff, 0x00000000,
197 0x8a14, 0xf000001f, 0x00000007,
198 0x8a14, 0xf000001f, 0x00000007,
199 0x8a14, 0xf000001f, 0x00000007,
200 0x8b24, 0xffffffff, 0x00ffffff,
201 0x8b10, 0x0000ff0f, 0x00000000,
202 0x28a4c, 0x07ffffff, 0x4e000000,
203 0x28350, 0x3f3f3fff, 0x0000124a,
204 0x28350, 0x3f3f3fff, 0x0000124a,
205 0x28350, 0x3f3f3fff, 0x0000124a,
206 0x30, 0x000000ff, 0x0040,
207 0x34, 0x00000040, 0x00004040,
208 0x9100, 0x07ffffff, 0x03000000,
209 0x9100, 0x07ffffff, 0x03000000,
210 0x8e88, 0x01ff1f3f, 0x00000000,
211 0x8e88, 0x01ff1f3f, 0x00000000,
212 0x8e88, 0x01ff1f3f, 0x00000000,
213 0x8e84, 0x01ff1f3f, 0x00000000,
214 0x8e84, 0x01ff1f3f, 0x00000000,
215 0x8e84, 0x01ff1f3f, 0x00000000,
216 0x9060, 0x0000007f, 0x00000020,
217 0x9508, 0x00010000, 0x00010000,
218 0xac14, 0x000003ff, 0x00000003,
219 0xac14, 0x000003ff, 0x00000003,
220 0xac14, 0x000003ff, 0x00000003,
221 0xac10, 0xffffffff, 0x00000000,
222 0xac10, 0xffffffff, 0x00000000,
223 0xac10, 0xffffffff, 0x00000000,
224 0xac0c, 0xffffffff, 0x00001032,
225 0xac0c, 0xffffffff, 0x00001032,
226 0xac0c, 0xffffffff, 0x00001032,
227 0x88d4, 0x0000001f, 0x00000010,
228 0x88d4, 0x0000001f, 0x00000010,
229 0x88d4, 0x0000001f, 0x00000010,
230 0x15c0, 0x000c0fc0, 0x000c0400
231};
232
233static const u32 oland_golden_rlc_registers[] =
234{
235 0xc424, 0xffffffff, 0x00601005,
236 0xc47c, 0xffffffff, 0x10104040,
237 0xc488, 0xffffffff, 0x0100000a,
238 0xc314, 0xffffffff, 0x00000800,
239 0xc30c, 0xffffffff, 0x800000f4
240};
241
242static const u32 oland_golden_registers[] =
243{
244 0x9a10, 0x00010000, 0x00018208,
245 0x9830, 0xffffffff, 0x00000000,
246 0x9834, 0xf00fffff, 0x00000400,
247 0x9838, 0x0002021c, 0x00020200,
248 0xc78, 0x00000080, 0x00000000,
249 0xd030, 0x000300c0, 0x00800040,
250 0xd830, 0x000300c0, 0x00800040,
251 0x5bb0, 0x000000f0, 0x00000070,
252 0x5bc0, 0x00200000, 0x50100000,
253 0x7030, 0x31000311, 0x00000011,
254 0x2ae4, 0x00073ffe, 0x000022a2,
255 0x240c, 0x000007ff, 0x00000000,
256 0x8a14, 0xf000001f, 0x00000007,
257 0x8b24, 0xffffffff, 0x00ffffff,
258 0x8b10, 0x0000ff0f, 0x00000000,
259 0x28a4c, 0x07ffffff, 0x4e000000,
260 0x28350, 0x3f3f3fff, 0x00000082,
261 0x30, 0x000000ff, 0x0040,
262 0x34, 0x00000040, 0x00004040,
263 0x9100, 0x07ffffff, 0x03000000,
264 0x9060, 0x0000007f, 0x00000020,
265 0x9508, 0x00010000, 0x00010000,
266 0xac14, 0x000003ff, 0x000000f3,
267 0xac10, 0xffffffff, 0x00000000,
268 0xac0c, 0xffffffff, 0x00003210,
269 0x88d4, 0x0000001f, 0x00000010,
270 0x15c0, 0x000c0fc0, 0x000c0400
271};
272
Alex Deucherfffbdda2013-05-13 13:36:23 -0400273static const u32 hainan_golden_registers[] =
274{
275 0x9a10, 0x00010000, 0x00018208,
276 0x9830, 0xffffffff, 0x00000000,
277 0x9834, 0xf00fffff, 0x00000400,
278 0x9838, 0x0002021c, 0x00020200,
279 0xd0c0, 0xff000fff, 0x00000100,
280 0xd030, 0x000300c0, 0x00800040,
281 0xd8c0, 0xff000fff, 0x00000100,
282 0xd830, 0x000300c0, 0x00800040,
283 0x2ae4, 0x00073ffe, 0x000022a2,
284 0x240c, 0x000007ff, 0x00000000,
285 0x8a14, 0xf000001f, 0x00000007,
286 0x8b24, 0xffffffff, 0x00ffffff,
287 0x8b10, 0x0000ff0f, 0x00000000,
288 0x28a4c, 0x07ffffff, 0x4e000000,
289 0x28350, 0x3f3f3fff, 0x00000000,
290 0x30, 0x000000ff, 0x0040,
291 0x34, 0x00000040, 0x00004040,
292 0x9100, 0x03e00000, 0x03600000,
293 0x9060, 0x0000007f, 0x00000020,
294 0x9508, 0x00010000, 0x00010000,
295 0xac14, 0x000003ff, 0x000000f1,
296 0xac10, 0xffffffff, 0x00000000,
297 0xac0c, 0xffffffff, 0x00003210,
298 0x88d4, 0x0000001f, 0x00000010,
299 0x15c0, 0x000c0fc0, 0x000c0400
300};
301
302static const u32 hainan_golden_registers2[] =
303{
304 0x98f8, 0xffffffff, 0x02010001
305};
306
Alex Deucher205996c2013-03-01 17:08:42 -0500307static const u32 tahiti_mgcg_cgcg_init[] =
308{
309 0xc400, 0xffffffff, 0xfffffffc,
310 0x802c, 0xffffffff, 0xe0000000,
311 0x9a60, 0xffffffff, 0x00000100,
312 0x92a4, 0xffffffff, 0x00000100,
313 0xc164, 0xffffffff, 0x00000100,
314 0x9774, 0xffffffff, 0x00000100,
315 0x8984, 0xffffffff, 0x06000100,
316 0x8a18, 0xffffffff, 0x00000100,
317 0x92a0, 0xffffffff, 0x00000100,
318 0xc380, 0xffffffff, 0x00000100,
319 0x8b28, 0xffffffff, 0x00000100,
320 0x9144, 0xffffffff, 0x00000100,
321 0x8d88, 0xffffffff, 0x00000100,
322 0x8d8c, 0xffffffff, 0x00000100,
323 0x9030, 0xffffffff, 0x00000100,
324 0x9034, 0xffffffff, 0x00000100,
325 0x9038, 0xffffffff, 0x00000100,
326 0x903c, 0xffffffff, 0x00000100,
327 0xad80, 0xffffffff, 0x00000100,
328 0xac54, 0xffffffff, 0x00000100,
329 0x897c, 0xffffffff, 0x06000100,
330 0x9868, 0xffffffff, 0x00000100,
331 0x9510, 0xffffffff, 0x00000100,
332 0xaf04, 0xffffffff, 0x00000100,
333 0xae04, 0xffffffff, 0x00000100,
334 0x949c, 0xffffffff, 0x00000100,
335 0x802c, 0xffffffff, 0xe0000000,
336 0x9160, 0xffffffff, 0x00010000,
337 0x9164, 0xffffffff, 0x00030002,
338 0x9168, 0xffffffff, 0x00040007,
339 0x916c, 0xffffffff, 0x00060005,
340 0x9170, 0xffffffff, 0x00090008,
341 0x9174, 0xffffffff, 0x00020001,
342 0x9178, 0xffffffff, 0x00040003,
343 0x917c, 0xffffffff, 0x00000007,
344 0x9180, 0xffffffff, 0x00060005,
345 0x9184, 0xffffffff, 0x00090008,
346 0x9188, 0xffffffff, 0x00030002,
347 0x918c, 0xffffffff, 0x00050004,
348 0x9190, 0xffffffff, 0x00000008,
349 0x9194, 0xffffffff, 0x00070006,
350 0x9198, 0xffffffff, 0x000a0009,
351 0x919c, 0xffffffff, 0x00040003,
352 0x91a0, 0xffffffff, 0x00060005,
353 0x91a4, 0xffffffff, 0x00000009,
354 0x91a8, 0xffffffff, 0x00080007,
355 0x91ac, 0xffffffff, 0x000b000a,
356 0x91b0, 0xffffffff, 0x00050004,
357 0x91b4, 0xffffffff, 0x00070006,
358 0x91b8, 0xffffffff, 0x0008000b,
359 0x91bc, 0xffffffff, 0x000a0009,
360 0x91c0, 0xffffffff, 0x000d000c,
361 0x91c4, 0xffffffff, 0x00060005,
362 0x91c8, 0xffffffff, 0x00080007,
363 0x91cc, 0xffffffff, 0x0000000b,
364 0x91d0, 0xffffffff, 0x000a0009,
365 0x91d4, 0xffffffff, 0x000d000c,
366 0x91d8, 0xffffffff, 0x00070006,
367 0x91dc, 0xffffffff, 0x00090008,
368 0x91e0, 0xffffffff, 0x0000000c,
369 0x91e4, 0xffffffff, 0x000b000a,
370 0x91e8, 0xffffffff, 0x000e000d,
371 0x91ec, 0xffffffff, 0x00080007,
372 0x91f0, 0xffffffff, 0x000a0009,
373 0x91f4, 0xffffffff, 0x0000000d,
374 0x91f8, 0xffffffff, 0x000c000b,
375 0x91fc, 0xffffffff, 0x000f000e,
376 0x9200, 0xffffffff, 0x00090008,
377 0x9204, 0xffffffff, 0x000b000a,
378 0x9208, 0xffffffff, 0x000c000f,
379 0x920c, 0xffffffff, 0x000e000d,
380 0x9210, 0xffffffff, 0x00110010,
381 0x9214, 0xffffffff, 0x000a0009,
382 0x9218, 0xffffffff, 0x000c000b,
383 0x921c, 0xffffffff, 0x0000000f,
384 0x9220, 0xffffffff, 0x000e000d,
385 0x9224, 0xffffffff, 0x00110010,
386 0x9228, 0xffffffff, 0x000b000a,
387 0x922c, 0xffffffff, 0x000d000c,
388 0x9230, 0xffffffff, 0x00000010,
389 0x9234, 0xffffffff, 0x000f000e,
390 0x9238, 0xffffffff, 0x00120011,
391 0x923c, 0xffffffff, 0x000c000b,
392 0x9240, 0xffffffff, 0x000e000d,
393 0x9244, 0xffffffff, 0x00000011,
394 0x9248, 0xffffffff, 0x0010000f,
395 0x924c, 0xffffffff, 0x00130012,
396 0x9250, 0xffffffff, 0x000d000c,
397 0x9254, 0xffffffff, 0x000f000e,
398 0x9258, 0xffffffff, 0x00100013,
399 0x925c, 0xffffffff, 0x00120011,
400 0x9260, 0xffffffff, 0x00150014,
401 0x9264, 0xffffffff, 0x000e000d,
402 0x9268, 0xffffffff, 0x0010000f,
403 0x926c, 0xffffffff, 0x00000013,
404 0x9270, 0xffffffff, 0x00120011,
405 0x9274, 0xffffffff, 0x00150014,
406 0x9278, 0xffffffff, 0x000f000e,
407 0x927c, 0xffffffff, 0x00110010,
408 0x9280, 0xffffffff, 0x00000014,
409 0x9284, 0xffffffff, 0x00130012,
410 0x9288, 0xffffffff, 0x00160015,
411 0x928c, 0xffffffff, 0x0010000f,
412 0x9290, 0xffffffff, 0x00120011,
413 0x9294, 0xffffffff, 0x00000015,
414 0x9298, 0xffffffff, 0x00140013,
415 0x929c, 0xffffffff, 0x00170016,
416 0x9150, 0xffffffff, 0x96940200,
417 0x8708, 0xffffffff, 0x00900100,
418 0xc478, 0xffffffff, 0x00000080,
419 0xc404, 0xffffffff, 0x0020003f,
420 0x30, 0xffffffff, 0x0000001c,
421 0x34, 0x000f0000, 0x000f0000,
422 0x160c, 0xffffffff, 0x00000100,
423 0x1024, 0xffffffff, 0x00000100,
424 0x102c, 0x00000101, 0x00000000,
425 0x20a8, 0xffffffff, 0x00000104,
426 0x264c, 0x000c0000, 0x000c0000,
427 0x2648, 0x000c0000, 0x000c0000,
428 0x55e4, 0xff000fff, 0x00000100,
429 0x55e8, 0x00000001, 0x00000001,
430 0x2f50, 0x00000001, 0x00000001,
431 0x30cc, 0xc0000fff, 0x00000104,
432 0xc1e4, 0x00000001, 0x00000001,
433 0xd0c0, 0xfffffff0, 0x00000100,
434 0xd8c0, 0xfffffff0, 0x00000100
435};
436
437static const u32 pitcairn_mgcg_cgcg_init[] =
438{
439 0xc400, 0xffffffff, 0xfffffffc,
440 0x802c, 0xffffffff, 0xe0000000,
441 0x9a60, 0xffffffff, 0x00000100,
442 0x92a4, 0xffffffff, 0x00000100,
443 0xc164, 0xffffffff, 0x00000100,
444 0x9774, 0xffffffff, 0x00000100,
445 0x8984, 0xffffffff, 0x06000100,
446 0x8a18, 0xffffffff, 0x00000100,
447 0x92a0, 0xffffffff, 0x00000100,
448 0xc380, 0xffffffff, 0x00000100,
449 0x8b28, 0xffffffff, 0x00000100,
450 0x9144, 0xffffffff, 0x00000100,
451 0x8d88, 0xffffffff, 0x00000100,
452 0x8d8c, 0xffffffff, 0x00000100,
453 0x9030, 0xffffffff, 0x00000100,
454 0x9034, 0xffffffff, 0x00000100,
455 0x9038, 0xffffffff, 0x00000100,
456 0x903c, 0xffffffff, 0x00000100,
457 0xad80, 0xffffffff, 0x00000100,
458 0xac54, 0xffffffff, 0x00000100,
459 0x897c, 0xffffffff, 0x06000100,
460 0x9868, 0xffffffff, 0x00000100,
461 0x9510, 0xffffffff, 0x00000100,
462 0xaf04, 0xffffffff, 0x00000100,
463 0xae04, 0xffffffff, 0x00000100,
464 0x949c, 0xffffffff, 0x00000100,
465 0x802c, 0xffffffff, 0xe0000000,
466 0x9160, 0xffffffff, 0x00010000,
467 0x9164, 0xffffffff, 0x00030002,
468 0x9168, 0xffffffff, 0x00040007,
469 0x916c, 0xffffffff, 0x00060005,
470 0x9170, 0xffffffff, 0x00090008,
471 0x9174, 0xffffffff, 0x00020001,
472 0x9178, 0xffffffff, 0x00040003,
473 0x917c, 0xffffffff, 0x00000007,
474 0x9180, 0xffffffff, 0x00060005,
475 0x9184, 0xffffffff, 0x00090008,
476 0x9188, 0xffffffff, 0x00030002,
477 0x918c, 0xffffffff, 0x00050004,
478 0x9190, 0xffffffff, 0x00000008,
479 0x9194, 0xffffffff, 0x00070006,
480 0x9198, 0xffffffff, 0x000a0009,
481 0x919c, 0xffffffff, 0x00040003,
482 0x91a0, 0xffffffff, 0x00060005,
483 0x91a4, 0xffffffff, 0x00000009,
484 0x91a8, 0xffffffff, 0x00080007,
485 0x91ac, 0xffffffff, 0x000b000a,
486 0x91b0, 0xffffffff, 0x00050004,
487 0x91b4, 0xffffffff, 0x00070006,
488 0x91b8, 0xffffffff, 0x0008000b,
489 0x91bc, 0xffffffff, 0x000a0009,
490 0x91c0, 0xffffffff, 0x000d000c,
491 0x9200, 0xffffffff, 0x00090008,
492 0x9204, 0xffffffff, 0x000b000a,
493 0x9208, 0xffffffff, 0x000c000f,
494 0x920c, 0xffffffff, 0x000e000d,
495 0x9210, 0xffffffff, 0x00110010,
496 0x9214, 0xffffffff, 0x000a0009,
497 0x9218, 0xffffffff, 0x000c000b,
498 0x921c, 0xffffffff, 0x0000000f,
499 0x9220, 0xffffffff, 0x000e000d,
500 0x9224, 0xffffffff, 0x00110010,
501 0x9228, 0xffffffff, 0x000b000a,
502 0x922c, 0xffffffff, 0x000d000c,
503 0x9230, 0xffffffff, 0x00000010,
504 0x9234, 0xffffffff, 0x000f000e,
505 0x9238, 0xffffffff, 0x00120011,
506 0x923c, 0xffffffff, 0x000c000b,
507 0x9240, 0xffffffff, 0x000e000d,
508 0x9244, 0xffffffff, 0x00000011,
509 0x9248, 0xffffffff, 0x0010000f,
510 0x924c, 0xffffffff, 0x00130012,
511 0x9250, 0xffffffff, 0x000d000c,
512 0x9254, 0xffffffff, 0x000f000e,
513 0x9258, 0xffffffff, 0x00100013,
514 0x925c, 0xffffffff, 0x00120011,
515 0x9260, 0xffffffff, 0x00150014,
516 0x9150, 0xffffffff, 0x96940200,
517 0x8708, 0xffffffff, 0x00900100,
518 0xc478, 0xffffffff, 0x00000080,
519 0xc404, 0xffffffff, 0x0020003f,
520 0x30, 0xffffffff, 0x0000001c,
521 0x34, 0x000f0000, 0x000f0000,
522 0x160c, 0xffffffff, 0x00000100,
523 0x1024, 0xffffffff, 0x00000100,
524 0x102c, 0x00000101, 0x00000000,
525 0x20a8, 0xffffffff, 0x00000104,
526 0x55e4, 0xff000fff, 0x00000100,
527 0x55e8, 0x00000001, 0x00000001,
528 0x2f50, 0x00000001, 0x00000001,
529 0x30cc, 0xc0000fff, 0x00000104,
530 0xc1e4, 0x00000001, 0x00000001,
531 0xd0c0, 0xfffffff0, 0x00000100,
532 0xd8c0, 0xfffffff0, 0x00000100
533};
534
535static const u32 verde_mgcg_cgcg_init[] =
536{
537 0xc400, 0xffffffff, 0xfffffffc,
538 0x802c, 0xffffffff, 0xe0000000,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x92a4, 0xffffffff, 0x00000100,
541 0xc164, 0xffffffff, 0x00000100,
542 0x9774, 0xffffffff, 0x00000100,
543 0x8984, 0xffffffff, 0x06000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x92a0, 0xffffffff, 0x00000100,
546 0xc380, 0xffffffff, 0x00000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x8d88, 0xffffffff, 0x00000100,
550 0x8d8c, 0xffffffff, 0x00000100,
551 0x9030, 0xffffffff, 0x00000100,
552 0x9034, 0xffffffff, 0x00000100,
553 0x9038, 0xffffffff, 0x00000100,
554 0x903c, 0xffffffff, 0x00000100,
555 0xad80, 0xffffffff, 0x00000100,
556 0xac54, 0xffffffff, 0x00000100,
557 0x897c, 0xffffffff, 0x06000100,
558 0x9868, 0xffffffff, 0x00000100,
559 0x9510, 0xffffffff, 0x00000100,
560 0xaf04, 0xffffffff, 0x00000100,
561 0xae04, 0xffffffff, 0x00000100,
562 0x949c, 0xffffffff, 0x00000100,
563 0x802c, 0xffffffff, 0xe0000000,
564 0x9160, 0xffffffff, 0x00010000,
565 0x9164, 0xffffffff, 0x00030002,
566 0x9168, 0xffffffff, 0x00040007,
567 0x916c, 0xffffffff, 0x00060005,
568 0x9170, 0xffffffff, 0x00090008,
569 0x9174, 0xffffffff, 0x00020001,
570 0x9178, 0xffffffff, 0x00040003,
571 0x917c, 0xffffffff, 0x00000007,
572 0x9180, 0xffffffff, 0x00060005,
573 0x9184, 0xffffffff, 0x00090008,
574 0x9188, 0xffffffff, 0x00030002,
575 0x918c, 0xffffffff, 0x00050004,
576 0x9190, 0xffffffff, 0x00000008,
577 0x9194, 0xffffffff, 0x00070006,
578 0x9198, 0xffffffff, 0x000a0009,
579 0x919c, 0xffffffff, 0x00040003,
580 0x91a0, 0xffffffff, 0x00060005,
581 0x91a4, 0xffffffff, 0x00000009,
582 0x91a8, 0xffffffff, 0x00080007,
583 0x91ac, 0xffffffff, 0x000b000a,
584 0x91b0, 0xffffffff, 0x00050004,
585 0x91b4, 0xffffffff, 0x00070006,
586 0x91b8, 0xffffffff, 0x0008000b,
587 0x91bc, 0xffffffff, 0x000a0009,
588 0x91c0, 0xffffffff, 0x000d000c,
589 0x9200, 0xffffffff, 0x00090008,
590 0x9204, 0xffffffff, 0x000b000a,
591 0x9208, 0xffffffff, 0x000c000f,
592 0x920c, 0xffffffff, 0x000e000d,
593 0x9210, 0xffffffff, 0x00110010,
594 0x9214, 0xffffffff, 0x000a0009,
595 0x9218, 0xffffffff, 0x000c000b,
596 0x921c, 0xffffffff, 0x0000000f,
597 0x9220, 0xffffffff, 0x000e000d,
598 0x9224, 0xffffffff, 0x00110010,
599 0x9228, 0xffffffff, 0x000b000a,
600 0x922c, 0xffffffff, 0x000d000c,
601 0x9230, 0xffffffff, 0x00000010,
602 0x9234, 0xffffffff, 0x000f000e,
603 0x9238, 0xffffffff, 0x00120011,
604 0x923c, 0xffffffff, 0x000c000b,
605 0x9240, 0xffffffff, 0x000e000d,
606 0x9244, 0xffffffff, 0x00000011,
607 0x9248, 0xffffffff, 0x0010000f,
608 0x924c, 0xffffffff, 0x00130012,
609 0x9250, 0xffffffff, 0x000d000c,
610 0x9254, 0xffffffff, 0x000f000e,
611 0x9258, 0xffffffff, 0x00100013,
612 0x925c, 0xffffffff, 0x00120011,
613 0x9260, 0xffffffff, 0x00150014,
614 0x9150, 0xffffffff, 0x96940200,
615 0x8708, 0xffffffff, 0x00900100,
616 0xc478, 0xffffffff, 0x00000080,
617 0xc404, 0xffffffff, 0x0020003f,
618 0x30, 0xffffffff, 0x0000001c,
619 0x34, 0x000f0000, 0x000f0000,
620 0x160c, 0xffffffff, 0x00000100,
621 0x1024, 0xffffffff, 0x00000100,
622 0x102c, 0x00000101, 0x00000000,
623 0x20a8, 0xffffffff, 0x00000104,
624 0x264c, 0x000c0000, 0x000c0000,
625 0x2648, 0x000c0000, 0x000c0000,
626 0x55e4, 0xff000fff, 0x00000100,
627 0x55e8, 0x00000001, 0x00000001,
628 0x2f50, 0x00000001, 0x00000001,
629 0x30cc, 0xc0000fff, 0x00000104,
630 0xc1e4, 0x00000001, 0x00000001,
631 0xd0c0, 0xfffffff0, 0x00000100,
632 0xd8c0, 0xfffffff0, 0x00000100
633};
634
635static const u32 oland_mgcg_cgcg_init[] =
636{
637 0xc400, 0xffffffff, 0xfffffffc,
638 0x802c, 0xffffffff, 0xe0000000,
639 0x9a60, 0xffffffff, 0x00000100,
640 0x92a4, 0xffffffff, 0x00000100,
641 0xc164, 0xffffffff, 0x00000100,
642 0x9774, 0xffffffff, 0x00000100,
643 0x8984, 0xffffffff, 0x06000100,
644 0x8a18, 0xffffffff, 0x00000100,
645 0x92a0, 0xffffffff, 0x00000100,
646 0xc380, 0xffffffff, 0x00000100,
647 0x8b28, 0xffffffff, 0x00000100,
648 0x9144, 0xffffffff, 0x00000100,
649 0x8d88, 0xffffffff, 0x00000100,
650 0x8d8c, 0xffffffff, 0x00000100,
651 0x9030, 0xffffffff, 0x00000100,
652 0x9034, 0xffffffff, 0x00000100,
653 0x9038, 0xffffffff, 0x00000100,
654 0x903c, 0xffffffff, 0x00000100,
655 0xad80, 0xffffffff, 0x00000100,
656 0xac54, 0xffffffff, 0x00000100,
657 0x897c, 0xffffffff, 0x06000100,
658 0x9868, 0xffffffff, 0x00000100,
659 0x9510, 0xffffffff, 0x00000100,
660 0xaf04, 0xffffffff, 0x00000100,
661 0xae04, 0xffffffff, 0x00000100,
662 0x949c, 0xffffffff, 0x00000100,
663 0x802c, 0xffffffff, 0xe0000000,
664 0x9160, 0xffffffff, 0x00010000,
665 0x9164, 0xffffffff, 0x00030002,
666 0x9168, 0xffffffff, 0x00040007,
667 0x916c, 0xffffffff, 0x00060005,
668 0x9170, 0xffffffff, 0x00090008,
669 0x9174, 0xffffffff, 0x00020001,
670 0x9178, 0xffffffff, 0x00040003,
671 0x917c, 0xffffffff, 0x00000007,
672 0x9180, 0xffffffff, 0x00060005,
673 0x9184, 0xffffffff, 0x00090008,
674 0x9188, 0xffffffff, 0x00030002,
675 0x918c, 0xffffffff, 0x00050004,
676 0x9190, 0xffffffff, 0x00000008,
677 0x9194, 0xffffffff, 0x00070006,
678 0x9198, 0xffffffff, 0x000a0009,
679 0x919c, 0xffffffff, 0x00040003,
680 0x91a0, 0xffffffff, 0x00060005,
681 0x91a4, 0xffffffff, 0x00000009,
682 0x91a8, 0xffffffff, 0x00080007,
683 0x91ac, 0xffffffff, 0x000b000a,
684 0x91b0, 0xffffffff, 0x00050004,
685 0x91b4, 0xffffffff, 0x00070006,
686 0x91b8, 0xffffffff, 0x0008000b,
687 0x91bc, 0xffffffff, 0x000a0009,
688 0x91c0, 0xffffffff, 0x000d000c,
689 0x91c4, 0xffffffff, 0x00060005,
690 0x91c8, 0xffffffff, 0x00080007,
691 0x91cc, 0xffffffff, 0x0000000b,
692 0x91d0, 0xffffffff, 0x000a0009,
693 0x91d4, 0xffffffff, 0x000d000c,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
713};
714
Alex Deucherfffbdda2013-05-13 13:36:23 -0400715static const u32 hainan_mgcg_cgcg_init[] =
716{
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x91c4, 0xffffffff, 0x00060005,
770 0x91c8, 0xffffffff, 0x00080007,
771 0x91cc, 0xffffffff, 0x0000000b,
772 0x91d0, 0xffffffff, 0x000a0009,
773 0x91d4, 0xffffffff, 0x000d000c,
774 0x9150, 0xffffffff, 0x96940200,
775 0x8708, 0xffffffff, 0x00900100,
776 0xc478, 0xffffffff, 0x00000080,
777 0xc404, 0xffffffff, 0x0020003f,
778 0x30, 0xffffffff, 0x0000001c,
779 0x34, 0x000f0000, 0x000f0000,
780 0x160c, 0xffffffff, 0x00000100,
781 0x1024, 0xffffffff, 0x00000100,
782 0x20a8, 0xffffffff, 0x00000104,
783 0x264c, 0x000c0000, 0x000c0000,
784 0x2648, 0x000c0000, 0x000c0000,
785 0x2f50, 0x00000001, 0x00000001,
786 0x30cc, 0xc0000fff, 0x00000104,
787 0xc1e4, 0x00000001, 0x00000001,
788 0xd0c0, 0xfffffff0, 0x00000100,
789 0xd8c0, 0xfffffff0, 0x00000100
790};
791
Alex Deucher205996c2013-03-01 17:08:42 -0500792static u32 verde_pg_init[] =
793{
794 0x353c, 0xffffffff, 0x40000,
795 0x3538, 0xffffffff, 0x200010ff,
796 0x353c, 0xffffffff, 0x0,
797 0x353c, 0xffffffff, 0x0,
798 0x353c, 0xffffffff, 0x0,
799 0x353c, 0xffffffff, 0x0,
800 0x353c, 0xffffffff, 0x0,
801 0x353c, 0xffffffff, 0x7007,
802 0x3538, 0xffffffff, 0x300010ff,
803 0x353c, 0xffffffff, 0x0,
804 0x353c, 0xffffffff, 0x0,
805 0x353c, 0xffffffff, 0x0,
806 0x353c, 0xffffffff, 0x0,
807 0x353c, 0xffffffff, 0x0,
808 0x353c, 0xffffffff, 0x400000,
809 0x3538, 0xffffffff, 0x100010ff,
810 0x353c, 0xffffffff, 0x0,
811 0x353c, 0xffffffff, 0x0,
812 0x353c, 0xffffffff, 0x0,
813 0x353c, 0xffffffff, 0x0,
814 0x353c, 0xffffffff, 0x0,
815 0x353c, 0xffffffff, 0x120200,
816 0x3538, 0xffffffff, 0x500010ff,
817 0x353c, 0xffffffff, 0x0,
818 0x353c, 0xffffffff, 0x0,
819 0x353c, 0xffffffff, 0x0,
820 0x353c, 0xffffffff, 0x0,
821 0x353c, 0xffffffff, 0x0,
822 0x353c, 0xffffffff, 0x1e1e16,
823 0x3538, 0xffffffff, 0x600010ff,
824 0x353c, 0xffffffff, 0x0,
825 0x353c, 0xffffffff, 0x0,
826 0x353c, 0xffffffff, 0x0,
827 0x353c, 0xffffffff, 0x0,
828 0x353c, 0xffffffff, 0x0,
829 0x353c, 0xffffffff, 0x171f1e,
830 0x3538, 0xffffffff, 0x700010ff,
831 0x353c, 0xffffffff, 0x0,
832 0x353c, 0xffffffff, 0x0,
833 0x353c, 0xffffffff, 0x0,
834 0x353c, 0xffffffff, 0x0,
835 0x353c, 0xffffffff, 0x0,
836 0x353c, 0xffffffff, 0x0,
837 0x3538, 0xffffffff, 0x9ff,
838 0x3500, 0xffffffff, 0x0,
839 0x3504, 0xffffffff, 0x10000800,
840 0x3504, 0xffffffff, 0xf,
841 0x3504, 0xffffffff, 0xf,
842 0x3500, 0xffffffff, 0x4,
843 0x3504, 0xffffffff, 0x1000051e,
844 0x3504, 0xffffffff, 0xffff,
845 0x3504, 0xffffffff, 0xffff,
846 0x3500, 0xffffffff, 0x8,
847 0x3504, 0xffffffff, 0x80500,
848 0x3500, 0xffffffff, 0x12,
849 0x3504, 0xffffffff, 0x9050c,
850 0x3500, 0xffffffff, 0x1d,
851 0x3504, 0xffffffff, 0xb052c,
852 0x3500, 0xffffffff, 0x2a,
853 0x3504, 0xffffffff, 0x1053e,
854 0x3500, 0xffffffff, 0x2d,
855 0x3504, 0xffffffff, 0x10546,
856 0x3500, 0xffffffff, 0x30,
857 0x3504, 0xffffffff, 0xa054e,
858 0x3500, 0xffffffff, 0x3c,
859 0x3504, 0xffffffff, 0x1055f,
860 0x3500, 0xffffffff, 0x3f,
861 0x3504, 0xffffffff, 0x10567,
862 0x3500, 0xffffffff, 0x42,
863 0x3504, 0xffffffff, 0x1056f,
864 0x3500, 0xffffffff, 0x45,
865 0x3504, 0xffffffff, 0x10572,
866 0x3500, 0xffffffff, 0x48,
867 0x3504, 0xffffffff, 0x20575,
868 0x3500, 0xffffffff, 0x4c,
869 0x3504, 0xffffffff, 0x190801,
870 0x3500, 0xffffffff, 0x67,
871 0x3504, 0xffffffff, 0x1082a,
872 0x3500, 0xffffffff, 0x6a,
873 0x3504, 0xffffffff, 0x1b082d,
874 0x3500, 0xffffffff, 0x87,
875 0x3504, 0xffffffff, 0x310851,
876 0x3500, 0xffffffff, 0xba,
877 0x3504, 0xffffffff, 0x891,
878 0x3500, 0xffffffff, 0xbc,
879 0x3504, 0xffffffff, 0x893,
880 0x3500, 0xffffffff, 0xbe,
881 0x3504, 0xffffffff, 0x20895,
882 0x3500, 0xffffffff, 0xc2,
883 0x3504, 0xffffffff, 0x20899,
884 0x3500, 0xffffffff, 0xc6,
885 0x3504, 0xffffffff, 0x2089d,
886 0x3500, 0xffffffff, 0xca,
887 0x3504, 0xffffffff, 0x8a1,
888 0x3500, 0xffffffff, 0xcc,
889 0x3504, 0xffffffff, 0x8a3,
890 0x3500, 0xffffffff, 0xce,
891 0x3504, 0xffffffff, 0x308a5,
892 0x3500, 0xffffffff, 0xd3,
893 0x3504, 0xffffffff, 0x6d08cd,
894 0x3500, 0xffffffff, 0x142,
895 0x3504, 0xffffffff, 0x2000095a,
896 0x3504, 0xffffffff, 0x1,
897 0x3500, 0xffffffff, 0x144,
898 0x3504, 0xffffffff, 0x301f095b,
899 0x3500, 0xffffffff, 0x165,
900 0x3504, 0xffffffff, 0xc094d,
901 0x3500, 0xffffffff, 0x173,
902 0x3504, 0xffffffff, 0xf096d,
903 0x3500, 0xffffffff, 0x184,
904 0x3504, 0xffffffff, 0x15097f,
905 0x3500, 0xffffffff, 0x19b,
906 0x3504, 0xffffffff, 0xc0998,
907 0x3500, 0xffffffff, 0x1a9,
908 0x3504, 0xffffffff, 0x409a7,
909 0x3500, 0xffffffff, 0x1af,
910 0x3504, 0xffffffff, 0xcdc,
911 0x3500, 0xffffffff, 0x1b1,
912 0x3504, 0xffffffff, 0x800,
913 0x3508, 0xffffffff, 0x6c9b2000,
914 0x3510, 0xfc00, 0x2000,
915 0x3544, 0xffffffff, 0xfc0,
916 0x28d4, 0x00000100, 0x100
917};
918
919static void si_init_golden_registers(struct radeon_device *rdev)
920{
921 switch (rdev->family) {
922 case CHIP_TAHITI:
923 radeon_program_register_sequence(rdev,
924 tahiti_golden_registers,
925 (const u32)ARRAY_SIZE(tahiti_golden_registers));
926 radeon_program_register_sequence(rdev,
927 tahiti_golden_rlc_registers,
928 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
929 radeon_program_register_sequence(rdev,
930 tahiti_mgcg_cgcg_init,
931 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
932 radeon_program_register_sequence(rdev,
933 tahiti_golden_registers2,
934 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
935 break;
936 case CHIP_PITCAIRN:
937 radeon_program_register_sequence(rdev,
938 pitcairn_golden_registers,
939 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
940 radeon_program_register_sequence(rdev,
941 pitcairn_golden_rlc_registers,
942 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
943 radeon_program_register_sequence(rdev,
944 pitcairn_mgcg_cgcg_init,
945 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
946 break;
947 case CHIP_VERDE:
948 radeon_program_register_sequence(rdev,
949 verde_golden_registers,
950 (const u32)ARRAY_SIZE(verde_golden_registers));
951 radeon_program_register_sequence(rdev,
952 verde_golden_rlc_registers,
953 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
954 radeon_program_register_sequence(rdev,
955 verde_mgcg_cgcg_init,
956 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
957 radeon_program_register_sequence(rdev,
958 verde_pg_init,
959 (const u32)ARRAY_SIZE(verde_pg_init));
960 break;
961 case CHIP_OLAND:
962 radeon_program_register_sequence(rdev,
963 oland_golden_registers,
964 (const u32)ARRAY_SIZE(oland_golden_registers));
965 radeon_program_register_sequence(rdev,
966 oland_golden_rlc_registers,
967 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
968 radeon_program_register_sequence(rdev,
969 oland_mgcg_cgcg_init,
970 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
971 break;
Alex Deucherfffbdda2013-05-13 13:36:23 -0400972 case CHIP_HAINAN:
973 radeon_program_register_sequence(rdev,
974 hainan_golden_registers,
975 (const u32)ARRAY_SIZE(hainan_golden_registers));
976 radeon_program_register_sequence(rdev,
977 hainan_golden_registers2,
978 (const u32)ARRAY_SIZE(hainan_golden_registers2));
979 radeon_program_register_sequence(rdev,
980 hainan_mgcg_cgcg_init,
981 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
982 break;
Alex Deucher205996c2013-03-01 17:08:42 -0500983 default:
984 break;
985 }
986}
987
Alex Deucher454d2e22013-02-14 10:04:02 -0500988#define PCIE_BUS_CLK 10000
989#define TCLK (PCIE_BUS_CLK / 10)
990
991/**
992 * si_get_xclk - get the xclk
993 *
994 * @rdev: radeon_device pointer
995 *
996 * Returns the reference clock used by the gfx engine
997 * (SI).
998 */
999u32 si_get_xclk(struct radeon_device *rdev)
1000{
1001 u32 reference_clock = rdev->clock.spll.reference_freq;
1002 u32 tmp;
1003
1004 tmp = RREG32(CG_CLKPIN_CNTL_2);
1005 if (tmp & MUX_TCLK_TO_XCLK)
1006 return TCLK;
1007
1008 tmp = RREG32(CG_CLKPIN_CNTL);
1009 if (tmp & XTALIN_DIVIDE)
1010 return reference_clock / 4;
1011
1012 return reference_clock;
1013}
1014
Alex Deucher1bd47d22012-03-20 17:18:10 -04001015/* get temperature in millidegrees */
1016int si_get_temp(struct radeon_device *rdev)
1017{
1018 u32 temp;
1019 int actual_temp = 0;
1020
1021 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1022 CTF_TEMP_SHIFT;
1023
1024 if (temp & 0x200)
1025 actual_temp = 255;
1026 else
1027 actual_temp = temp & 0x1ff;
1028
1029 actual_temp = (actual_temp * 1000);
1030
1031 return actual_temp;
1032}
1033
Alex Deucher8b074dd2012-03-20 17:18:18 -04001034#define TAHITI_IO_MC_REGS_SIZE 36
1035
1036static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1037 {0x0000006f, 0x03044000},
1038 {0x00000070, 0x0480c018},
1039 {0x00000071, 0x00000040},
1040 {0x00000072, 0x01000000},
1041 {0x00000074, 0x000000ff},
1042 {0x00000075, 0x00143400},
1043 {0x00000076, 0x08ec0800},
1044 {0x00000077, 0x040000cc},
1045 {0x00000079, 0x00000000},
1046 {0x0000007a, 0x21000409},
1047 {0x0000007c, 0x00000000},
1048 {0x0000007d, 0xe8000000},
1049 {0x0000007e, 0x044408a8},
1050 {0x0000007f, 0x00000003},
1051 {0x00000080, 0x00000000},
1052 {0x00000081, 0x01000000},
1053 {0x00000082, 0x02000000},
1054 {0x00000083, 0x00000000},
1055 {0x00000084, 0xe3f3e4f4},
1056 {0x00000085, 0x00052024},
1057 {0x00000087, 0x00000000},
1058 {0x00000088, 0x66036603},
1059 {0x00000089, 0x01000000},
1060 {0x0000008b, 0x1c0a0000},
1061 {0x0000008c, 0xff010000},
1062 {0x0000008e, 0xffffefff},
1063 {0x0000008f, 0xfff3efff},
1064 {0x00000090, 0xfff3efbf},
1065 {0x00000094, 0x00101101},
1066 {0x00000095, 0x00000fff},
1067 {0x00000096, 0x00116fff},
1068 {0x00000097, 0x60010000},
1069 {0x00000098, 0x10010000},
1070 {0x00000099, 0x00006000},
1071 {0x0000009a, 0x00001000},
1072 {0x0000009f, 0x00a77400}
1073};
1074
1075static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1076 {0x0000006f, 0x03044000},
1077 {0x00000070, 0x0480c018},
1078 {0x00000071, 0x00000040},
1079 {0x00000072, 0x01000000},
1080 {0x00000074, 0x000000ff},
1081 {0x00000075, 0x00143400},
1082 {0x00000076, 0x08ec0800},
1083 {0x00000077, 0x040000cc},
1084 {0x00000079, 0x00000000},
1085 {0x0000007a, 0x21000409},
1086 {0x0000007c, 0x00000000},
1087 {0x0000007d, 0xe8000000},
1088 {0x0000007e, 0x044408a8},
1089 {0x0000007f, 0x00000003},
1090 {0x00000080, 0x00000000},
1091 {0x00000081, 0x01000000},
1092 {0x00000082, 0x02000000},
1093 {0x00000083, 0x00000000},
1094 {0x00000084, 0xe3f3e4f4},
1095 {0x00000085, 0x00052024},
1096 {0x00000087, 0x00000000},
1097 {0x00000088, 0x66036603},
1098 {0x00000089, 0x01000000},
1099 {0x0000008b, 0x1c0a0000},
1100 {0x0000008c, 0xff010000},
1101 {0x0000008e, 0xffffefff},
1102 {0x0000008f, 0xfff3efff},
1103 {0x00000090, 0xfff3efbf},
1104 {0x00000094, 0x00101101},
1105 {0x00000095, 0x00000fff},
1106 {0x00000096, 0x00116fff},
1107 {0x00000097, 0x60010000},
1108 {0x00000098, 0x10010000},
1109 {0x00000099, 0x00006000},
1110 {0x0000009a, 0x00001000},
1111 {0x0000009f, 0x00a47400}
1112};
1113
1114static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1115 {0x0000006f, 0x03044000},
1116 {0x00000070, 0x0480c018},
1117 {0x00000071, 0x00000040},
1118 {0x00000072, 0x01000000},
1119 {0x00000074, 0x000000ff},
1120 {0x00000075, 0x00143400},
1121 {0x00000076, 0x08ec0800},
1122 {0x00000077, 0x040000cc},
1123 {0x00000079, 0x00000000},
1124 {0x0000007a, 0x21000409},
1125 {0x0000007c, 0x00000000},
1126 {0x0000007d, 0xe8000000},
1127 {0x0000007e, 0x044408a8},
1128 {0x0000007f, 0x00000003},
1129 {0x00000080, 0x00000000},
1130 {0x00000081, 0x01000000},
1131 {0x00000082, 0x02000000},
1132 {0x00000083, 0x00000000},
1133 {0x00000084, 0xe3f3e4f4},
1134 {0x00000085, 0x00052024},
1135 {0x00000087, 0x00000000},
1136 {0x00000088, 0x66036603},
1137 {0x00000089, 0x01000000},
1138 {0x0000008b, 0x1c0a0000},
1139 {0x0000008c, 0xff010000},
1140 {0x0000008e, 0xffffefff},
1141 {0x0000008f, 0xfff3efff},
1142 {0x00000090, 0xfff3efbf},
1143 {0x00000094, 0x00101101},
1144 {0x00000095, 0x00000fff},
1145 {0x00000096, 0x00116fff},
1146 {0x00000097, 0x60010000},
1147 {0x00000098, 0x10010000},
1148 {0x00000099, 0x00006000},
1149 {0x0000009a, 0x00001000},
1150 {0x0000009f, 0x00a37400}
1151};
1152
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001153static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1154 {0x0000006f, 0x03044000},
1155 {0x00000070, 0x0480c018},
1156 {0x00000071, 0x00000040},
1157 {0x00000072, 0x01000000},
1158 {0x00000074, 0x000000ff},
1159 {0x00000075, 0x00143400},
1160 {0x00000076, 0x08ec0800},
1161 {0x00000077, 0x040000cc},
1162 {0x00000079, 0x00000000},
1163 {0x0000007a, 0x21000409},
1164 {0x0000007c, 0x00000000},
1165 {0x0000007d, 0xe8000000},
1166 {0x0000007e, 0x044408a8},
1167 {0x0000007f, 0x00000003},
1168 {0x00000080, 0x00000000},
1169 {0x00000081, 0x01000000},
1170 {0x00000082, 0x02000000},
1171 {0x00000083, 0x00000000},
1172 {0x00000084, 0xe3f3e4f4},
1173 {0x00000085, 0x00052024},
1174 {0x00000087, 0x00000000},
1175 {0x00000088, 0x66036603},
1176 {0x00000089, 0x01000000},
1177 {0x0000008b, 0x1c0a0000},
1178 {0x0000008c, 0xff010000},
1179 {0x0000008e, 0xffffefff},
1180 {0x0000008f, 0xfff3efff},
1181 {0x00000090, 0xfff3efbf},
1182 {0x00000094, 0x00101101},
1183 {0x00000095, 0x00000fff},
1184 {0x00000096, 0x00116fff},
1185 {0x00000097, 0x60010000},
1186 {0x00000098, 0x10010000},
1187 {0x00000099, 0x00006000},
1188 {0x0000009a, 0x00001000},
1189 {0x0000009f, 0x00a17730}
1190};
1191
Alex Deucherc04c00b2012-07-31 12:57:45 -04001192static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1193 {0x0000006f, 0x03044000},
1194 {0x00000070, 0x0480c018},
1195 {0x00000071, 0x00000040},
1196 {0x00000072, 0x01000000},
1197 {0x00000074, 0x000000ff},
1198 {0x00000075, 0x00143400},
1199 {0x00000076, 0x08ec0800},
1200 {0x00000077, 0x040000cc},
1201 {0x00000079, 0x00000000},
1202 {0x0000007a, 0x21000409},
1203 {0x0000007c, 0x00000000},
1204 {0x0000007d, 0xe8000000},
1205 {0x0000007e, 0x044408a8},
1206 {0x0000007f, 0x00000003},
1207 {0x00000080, 0x00000000},
1208 {0x00000081, 0x01000000},
1209 {0x00000082, 0x02000000},
1210 {0x00000083, 0x00000000},
1211 {0x00000084, 0xe3f3e4f4},
1212 {0x00000085, 0x00052024},
1213 {0x00000087, 0x00000000},
1214 {0x00000088, 0x66036603},
1215 {0x00000089, 0x01000000},
1216 {0x0000008b, 0x1c0a0000},
1217 {0x0000008c, 0xff010000},
1218 {0x0000008e, 0xffffefff},
1219 {0x0000008f, 0xfff3efff},
1220 {0x00000090, 0xfff3efbf},
1221 {0x00000094, 0x00101101},
1222 {0x00000095, 0x00000fff},
1223 {0x00000096, 0x00116fff},
1224 {0x00000097, 0x60010000},
1225 {0x00000098, 0x10010000},
1226 {0x00000099, 0x00006000},
1227 {0x0000009a, 0x00001000},
1228 {0x0000009f, 0x00a07730}
1229};
1230
Alex Deucher8b074dd2012-03-20 17:18:18 -04001231/* ucode loading */
1232static int si_mc_load_microcode(struct radeon_device *rdev)
1233{
1234 const __be32 *fw_data;
1235 u32 running, blackout = 0;
1236 u32 *io_mc_regs;
1237 int i, ucode_size, regs_size;
1238
1239 if (!rdev->mc_fw)
1240 return -EINVAL;
1241
1242 switch (rdev->family) {
1243 case CHIP_TAHITI:
1244 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1245 ucode_size = SI_MC_UCODE_SIZE;
1246 regs_size = TAHITI_IO_MC_REGS_SIZE;
1247 break;
1248 case CHIP_PITCAIRN:
1249 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1250 ucode_size = SI_MC_UCODE_SIZE;
1251 regs_size = TAHITI_IO_MC_REGS_SIZE;
1252 break;
1253 case CHIP_VERDE:
1254 default:
1255 io_mc_regs = (u32 *)&verde_io_mc_regs;
1256 ucode_size = SI_MC_UCODE_SIZE;
1257 regs_size = TAHITI_IO_MC_REGS_SIZE;
1258 break;
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001259 case CHIP_OLAND:
1260 io_mc_regs = (u32 *)&oland_io_mc_regs;
1261 ucode_size = OLAND_MC_UCODE_SIZE;
1262 regs_size = TAHITI_IO_MC_REGS_SIZE;
1263 break;
Alex Deucherc04c00b2012-07-31 12:57:45 -04001264 case CHIP_HAINAN:
1265 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1266 ucode_size = OLAND_MC_UCODE_SIZE;
1267 regs_size = TAHITI_IO_MC_REGS_SIZE;
1268 break;
Alex Deucher8b074dd2012-03-20 17:18:18 -04001269 }
1270
1271 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1272
1273 if (running == 0) {
1274 if (running) {
1275 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1276 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1277 }
1278
1279 /* reset the engine and set to writable */
1280 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1281 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1282
1283 /* load mc io regs */
1284 for (i = 0; i < regs_size; i++) {
1285 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1286 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1287 }
1288 /* load the MC ucode */
1289 fw_data = (const __be32 *)rdev->mc_fw->data;
1290 for (i = 0; i < ucode_size; i++)
1291 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1292
1293 /* put the engine back into the active state */
1294 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1295 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1296 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1297
1298 /* wait for training to complete */
1299 for (i = 0; i < rdev->usec_timeout; i++) {
1300 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1301 break;
1302 udelay(1);
1303 }
1304 for (i = 0; i < rdev->usec_timeout; i++) {
1305 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1306 break;
1307 udelay(1);
1308 }
1309
1310 if (running)
1311 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1312 }
1313
1314 return 0;
1315}
1316
Alex Deucher0f0de062012-03-20 17:18:17 -04001317static int si_init_microcode(struct radeon_device *rdev)
1318{
1319 struct platform_device *pdev;
1320 const char *chip_name;
1321 const char *rlc_chip_name;
1322 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1323 char fw_name[30];
1324 int err;
1325
1326 DRM_DEBUG("\n");
1327
1328 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1329 err = IS_ERR(pdev);
1330 if (err) {
1331 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1332 return -EINVAL;
1333 }
1334
1335 switch (rdev->family) {
1336 case CHIP_TAHITI:
1337 chip_name = "TAHITI";
1338 rlc_chip_name = "TAHITI";
1339 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1340 me_req_size = SI_PM4_UCODE_SIZE * 4;
1341 ce_req_size = SI_CE_UCODE_SIZE * 4;
1342 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1343 mc_req_size = SI_MC_UCODE_SIZE * 4;
1344 break;
1345 case CHIP_PITCAIRN:
1346 chip_name = "PITCAIRN";
1347 rlc_chip_name = "PITCAIRN";
1348 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1349 me_req_size = SI_PM4_UCODE_SIZE * 4;
1350 ce_req_size = SI_CE_UCODE_SIZE * 4;
1351 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1352 mc_req_size = SI_MC_UCODE_SIZE * 4;
1353 break;
1354 case CHIP_VERDE:
1355 chip_name = "VERDE";
1356 rlc_chip_name = "VERDE";
1357 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1358 me_req_size = SI_PM4_UCODE_SIZE * 4;
1359 ce_req_size = SI_CE_UCODE_SIZE * 4;
1360 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1361 mc_req_size = SI_MC_UCODE_SIZE * 4;
1362 break;
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001363 case CHIP_OLAND:
1364 chip_name = "OLAND";
1365 rlc_chip_name = "OLAND";
1366 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1367 me_req_size = SI_PM4_UCODE_SIZE * 4;
1368 ce_req_size = SI_CE_UCODE_SIZE * 4;
1369 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1370 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1371 break;
Alex Deucherc04c00b2012-07-31 12:57:45 -04001372 case CHIP_HAINAN:
1373 chip_name = "HAINAN";
1374 rlc_chip_name = "HAINAN";
1375 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1376 me_req_size = SI_PM4_UCODE_SIZE * 4;
1377 ce_req_size = SI_CE_UCODE_SIZE * 4;
1378 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1379 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1380 break;
Alex Deucher0f0de062012-03-20 17:18:17 -04001381 default: BUG();
1382 }
1383
1384 DRM_INFO("Loading %s Microcode\n", chip_name);
1385
1386 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1387 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1388 if (err)
1389 goto out;
1390 if (rdev->pfp_fw->size != pfp_req_size) {
1391 printk(KERN_ERR
1392 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1393 rdev->pfp_fw->size, fw_name);
1394 err = -EINVAL;
1395 goto out;
1396 }
1397
1398 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1399 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1400 if (err)
1401 goto out;
1402 if (rdev->me_fw->size != me_req_size) {
1403 printk(KERN_ERR
1404 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1405 rdev->me_fw->size, fw_name);
1406 err = -EINVAL;
1407 }
1408
1409 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1410 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1411 if (err)
1412 goto out;
1413 if (rdev->ce_fw->size != ce_req_size) {
1414 printk(KERN_ERR
1415 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1416 rdev->ce_fw->size, fw_name);
1417 err = -EINVAL;
1418 }
1419
1420 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1421 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1422 if (err)
1423 goto out;
1424 if (rdev->rlc_fw->size != rlc_req_size) {
1425 printk(KERN_ERR
1426 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1427 rdev->rlc_fw->size, fw_name);
1428 err = -EINVAL;
1429 }
1430
1431 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1432 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1433 if (err)
1434 goto out;
1435 if (rdev->mc_fw->size != mc_req_size) {
1436 printk(KERN_ERR
1437 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1438 rdev->mc_fw->size, fw_name);
1439 err = -EINVAL;
1440 }
1441
1442out:
1443 platform_device_unregister(pdev);
1444
1445 if (err) {
1446 if (err != -EINVAL)
1447 printk(KERN_ERR
1448 "si_cp: Failed to load firmware \"%s\"\n",
1449 fw_name);
1450 release_firmware(rdev->pfp_fw);
1451 rdev->pfp_fw = NULL;
1452 release_firmware(rdev->me_fw);
1453 rdev->me_fw = NULL;
1454 release_firmware(rdev->ce_fw);
1455 rdev->ce_fw = NULL;
1456 release_firmware(rdev->rlc_fw);
1457 rdev->rlc_fw = NULL;
1458 release_firmware(rdev->mc_fw);
1459 rdev->mc_fw = NULL;
1460 }
1461 return err;
1462}
1463
Alex Deucher43b3cd92012-03-20 17:18:00 -04001464/* watermark setup */
1465static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1466 struct radeon_crtc *radeon_crtc,
1467 struct drm_display_mode *mode,
1468 struct drm_display_mode *other_mode)
1469{
1470 u32 tmp;
1471 /*
1472 * Line Buffer Setup
1473 * There are 3 line buffers, each one shared by 2 display controllers.
1474 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1475 * the display controllers. The paritioning is done via one of four
1476 * preset allocations specified in bits 21:20:
1477 * 0 - half lb
1478 * 2 - whole lb, other crtc must be disabled
1479 */
1480 /* this can get tricky if we have two large displays on a paired group
1481 * of crtcs. Ideally for multiple large displays we'd assign them to
1482 * non-linked crtcs for maximum line buffer allocation.
1483 */
1484 if (radeon_crtc->base.enabled && mode) {
1485 if (other_mode)
1486 tmp = 0; /* 1/2 */
1487 else
1488 tmp = 2; /* whole */
1489 } else
1490 tmp = 0;
1491
1492 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1493 DC_LB_MEMORY_CONFIG(tmp));
1494
1495 if (radeon_crtc->base.enabled && mode) {
1496 switch (tmp) {
1497 case 0:
1498 default:
1499 return 4096 * 2;
1500 case 2:
1501 return 8192 * 2;
1502 }
1503 }
1504
1505 /* controller not enabled, so no lb used */
1506 return 0;
1507}
1508
Alex Deucherca7db222012-03-20 17:18:30 -04001509static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
Alex Deucher43b3cd92012-03-20 17:18:00 -04001510{
1511 u32 tmp = RREG32(MC_SHARED_CHMAP);
1512
1513 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1514 case 0:
1515 default:
1516 return 1;
1517 case 1:
1518 return 2;
1519 case 2:
1520 return 4;
1521 case 3:
1522 return 8;
1523 case 4:
1524 return 3;
1525 case 5:
1526 return 6;
1527 case 6:
1528 return 10;
1529 case 7:
1530 return 12;
1531 case 8:
1532 return 16;
1533 }
1534}
1535
1536struct dce6_wm_params {
1537 u32 dram_channels; /* number of dram channels */
1538 u32 yclk; /* bandwidth per dram data pin in kHz */
1539 u32 sclk; /* engine clock in kHz */
1540 u32 disp_clk; /* display clock in kHz */
1541 u32 src_width; /* viewport width */
1542 u32 active_time; /* active display time in ns */
1543 u32 blank_time; /* blank time in ns */
1544 bool interlaced; /* mode is interlaced */
1545 fixed20_12 vsc; /* vertical scale ratio */
1546 u32 num_heads; /* number of active crtcs */
1547 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1548 u32 lb_size; /* line buffer allocated to pipe */
1549 u32 vtaps; /* vertical scaler taps */
1550};
1551
1552static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1553{
1554 /* Calculate raw DRAM Bandwidth */
1555 fixed20_12 dram_efficiency; /* 0.7 */
1556 fixed20_12 yclk, dram_channels, bandwidth;
1557 fixed20_12 a;
1558
1559 a.full = dfixed_const(1000);
1560 yclk.full = dfixed_const(wm->yclk);
1561 yclk.full = dfixed_div(yclk, a);
1562 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1563 a.full = dfixed_const(10);
1564 dram_efficiency.full = dfixed_const(7);
1565 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1566 bandwidth.full = dfixed_mul(dram_channels, yclk);
1567 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1568
1569 return dfixed_trunc(bandwidth);
1570}
1571
1572static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1573{
1574 /* Calculate DRAM Bandwidth and the part allocated to display. */
1575 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1576 fixed20_12 yclk, dram_channels, bandwidth;
1577 fixed20_12 a;
1578
1579 a.full = dfixed_const(1000);
1580 yclk.full = dfixed_const(wm->yclk);
1581 yclk.full = dfixed_div(yclk, a);
1582 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1583 a.full = dfixed_const(10);
1584 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1585 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1586 bandwidth.full = dfixed_mul(dram_channels, yclk);
1587 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1588
1589 return dfixed_trunc(bandwidth);
1590}
1591
1592static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1593{
1594 /* Calculate the display Data return Bandwidth */
1595 fixed20_12 return_efficiency; /* 0.8 */
1596 fixed20_12 sclk, bandwidth;
1597 fixed20_12 a;
1598
1599 a.full = dfixed_const(1000);
1600 sclk.full = dfixed_const(wm->sclk);
1601 sclk.full = dfixed_div(sclk, a);
1602 a.full = dfixed_const(10);
1603 return_efficiency.full = dfixed_const(8);
1604 return_efficiency.full = dfixed_div(return_efficiency, a);
1605 a.full = dfixed_const(32);
1606 bandwidth.full = dfixed_mul(a, sclk);
1607 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1608
1609 return dfixed_trunc(bandwidth);
1610}
1611
1612static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1613{
1614 return 32;
1615}
1616
1617static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1618{
1619 /* Calculate the DMIF Request Bandwidth */
1620 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1621 fixed20_12 disp_clk, sclk, bandwidth;
1622 fixed20_12 a, b1, b2;
1623 u32 min_bandwidth;
1624
1625 a.full = dfixed_const(1000);
1626 disp_clk.full = dfixed_const(wm->disp_clk);
1627 disp_clk.full = dfixed_div(disp_clk, a);
1628 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1629 b1.full = dfixed_mul(a, disp_clk);
1630
1631 a.full = dfixed_const(1000);
1632 sclk.full = dfixed_const(wm->sclk);
1633 sclk.full = dfixed_div(sclk, a);
1634 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1635 b2.full = dfixed_mul(a, sclk);
1636
1637 a.full = dfixed_const(10);
1638 disp_clk_request_efficiency.full = dfixed_const(8);
1639 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1640
1641 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1642
1643 a.full = dfixed_const(min_bandwidth);
1644 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1645
1646 return dfixed_trunc(bandwidth);
1647}
1648
1649static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1650{
1651 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1652 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1653 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1654 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1655
1656 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1657}
1658
1659static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1660{
1661 /* Calculate the display mode Average Bandwidth
1662 * DisplayMode should contain the source and destination dimensions,
1663 * timing, etc.
1664 */
1665 fixed20_12 bpp;
1666 fixed20_12 line_time;
1667 fixed20_12 src_width;
1668 fixed20_12 bandwidth;
1669 fixed20_12 a;
1670
1671 a.full = dfixed_const(1000);
1672 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1673 line_time.full = dfixed_div(line_time, a);
1674 bpp.full = dfixed_const(wm->bytes_per_pixel);
1675 src_width.full = dfixed_const(wm->src_width);
1676 bandwidth.full = dfixed_mul(src_width, bpp);
1677 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1678 bandwidth.full = dfixed_div(bandwidth, line_time);
1679
1680 return dfixed_trunc(bandwidth);
1681}
1682
1683static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1684{
1685 /* First calcualte the latency in ns */
1686 u32 mc_latency = 2000; /* 2000 ns. */
1687 u32 available_bandwidth = dce6_available_bandwidth(wm);
1688 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1689 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1690 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1691 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1692 (wm->num_heads * cursor_line_pair_return_time);
1693 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1694 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1695 u32 tmp, dmif_size = 12288;
1696 fixed20_12 a, b, c;
1697
1698 if (wm->num_heads == 0)
1699 return 0;
1700
1701 a.full = dfixed_const(2);
1702 b.full = dfixed_const(1);
1703 if ((wm->vsc.full > a.full) ||
1704 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1705 (wm->vtaps >= 5) ||
1706 ((wm->vsc.full >= a.full) && wm->interlaced))
1707 max_src_lines_per_dst_line = 4;
1708 else
1709 max_src_lines_per_dst_line = 2;
1710
1711 a.full = dfixed_const(available_bandwidth);
1712 b.full = dfixed_const(wm->num_heads);
1713 a.full = dfixed_div(a, b);
1714
1715 b.full = dfixed_const(mc_latency + 512);
1716 c.full = dfixed_const(wm->disp_clk);
1717 b.full = dfixed_div(b, c);
1718
1719 c.full = dfixed_const(dmif_size);
1720 b.full = dfixed_div(c, b);
1721
1722 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1723
1724 b.full = dfixed_const(1000);
1725 c.full = dfixed_const(wm->disp_clk);
1726 b.full = dfixed_div(c, b);
1727 c.full = dfixed_const(wm->bytes_per_pixel);
1728 b.full = dfixed_mul(b, c);
1729
1730 lb_fill_bw = min(tmp, dfixed_trunc(b));
1731
1732 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1733 b.full = dfixed_const(1000);
1734 c.full = dfixed_const(lb_fill_bw);
1735 b.full = dfixed_div(c, b);
1736 a.full = dfixed_div(a, b);
1737 line_fill_time = dfixed_trunc(a);
1738
1739 if (line_fill_time < wm->active_time)
1740 return latency;
1741 else
1742 return latency + (line_fill_time - wm->active_time);
1743
1744}
1745
1746static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1747{
1748 if (dce6_average_bandwidth(wm) <=
1749 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1750 return true;
1751 else
1752 return false;
1753};
1754
1755static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1756{
1757 if (dce6_average_bandwidth(wm) <=
1758 (dce6_available_bandwidth(wm) / wm->num_heads))
1759 return true;
1760 else
1761 return false;
1762};
1763
1764static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1765{
1766 u32 lb_partitions = wm->lb_size / wm->src_width;
1767 u32 line_time = wm->active_time + wm->blank_time;
1768 u32 latency_tolerant_lines;
1769 u32 latency_hiding;
1770 fixed20_12 a;
1771
1772 a.full = dfixed_const(1);
1773 if (wm->vsc.full > a.full)
1774 latency_tolerant_lines = 1;
1775 else {
1776 if (lb_partitions <= (wm->vtaps + 1))
1777 latency_tolerant_lines = 1;
1778 else
1779 latency_tolerant_lines = 2;
1780 }
1781
1782 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1783
1784 if (dce6_latency_watermark(wm) <= latency_hiding)
1785 return true;
1786 else
1787 return false;
1788}
1789
1790static void dce6_program_watermarks(struct radeon_device *rdev,
1791 struct radeon_crtc *radeon_crtc,
1792 u32 lb_size, u32 num_heads)
1793{
1794 struct drm_display_mode *mode = &radeon_crtc->base.mode;
1795 struct dce6_wm_params wm;
1796 u32 pixel_period;
1797 u32 line_time = 0;
1798 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1799 u32 priority_a_mark = 0, priority_b_mark = 0;
1800 u32 priority_a_cnt = PRIORITY_OFF;
1801 u32 priority_b_cnt = PRIORITY_OFF;
1802 u32 tmp, arb_control3;
1803 fixed20_12 a, b, c;
1804
1805 if (radeon_crtc->base.enabled && num_heads && mode) {
1806 pixel_period = 1000000 / (u32)mode->clock;
1807 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1808 priority_a_cnt = 0;
1809 priority_b_cnt = 0;
1810
1811 wm.yclk = rdev->pm.current_mclk * 10;
1812 wm.sclk = rdev->pm.current_sclk * 10;
1813 wm.disp_clk = mode->clock;
1814 wm.src_width = mode->crtc_hdisplay;
1815 wm.active_time = mode->crtc_hdisplay * pixel_period;
1816 wm.blank_time = line_time - wm.active_time;
1817 wm.interlaced = false;
1818 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1819 wm.interlaced = true;
1820 wm.vsc = radeon_crtc->vsc;
1821 wm.vtaps = 1;
1822 if (radeon_crtc->rmx_type != RMX_OFF)
1823 wm.vtaps = 2;
1824 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1825 wm.lb_size = lb_size;
Alex Deucherca7db222012-03-20 17:18:30 -04001826 if (rdev->family == CHIP_ARUBA)
1827 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1828 else
1829 wm.dram_channels = si_get_number_of_dram_channels(rdev);
Alex Deucher43b3cd92012-03-20 17:18:00 -04001830 wm.num_heads = num_heads;
1831
1832 /* set for high clocks */
1833 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
1834 /* set for low clocks */
1835 /* wm.yclk = low clk; wm.sclk = low clk */
1836 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1837
1838 /* possibly force display priority to high */
1839 /* should really do this at mode validation time... */
1840 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1841 !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
1842 !dce6_check_latency_hiding(&wm) ||
1843 (rdev->disp_priority == 2)) {
1844 DRM_DEBUG_KMS("force priority to high\n");
1845 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1846 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1847 }
1848
1849 a.full = dfixed_const(1000);
1850 b.full = dfixed_const(mode->clock);
1851 b.full = dfixed_div(b, a);
1852 c.full = dfixed_const(latency_watermark_a);
1853 c.full = dfixed_mul(c, b);
1854 c.full = dfixed_mul(c, radeon_crtc->hsc);
1855 c.full = dfixed_div(c, a);
1856 a.full = dfixed_const(16);
1857 c.full = dfixed_div(c, a);
1858 priority_a_mark = dfixed_trunc(c);
1859 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1860
1861 a.full = dfixed_const(1000);
1862 b.full = dfixed_const(mode->clock);
1863 b.full = dfixed_div(b, a);
1864 c.full = dfixed_const(latency_watermark_b);
1865 c.full = dfixed_mul(c, b);
1866 c.full = dfixed_mul(c, radeon_crtc->hsc);
1867 c.full = dfixed_div(c, a);
1868 a.full = dfixed_const(16);
1869 c.full = dfixed_div(c, a);
1870 priority_b_mark = dfixed_trunc(c);
1871 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1872 }
1873
1874 /* select wm A */
1875 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1876 tmp = arb_control3;
1877 tmp &= ~LATENCY_WATERMARK_MASK(3);
1878 tmp |= LATENCY_WATERMARK_MASK(1);
1879 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1880 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1881 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1882 LATENCY_HIGH_WATERMARK(line_time)));
1883 /* select wm B */
1884 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1885 tmp &= ~LATENCY_WATERMARK_MASK(3);
1886 tmp |= LATENCY_WATERMARK_MASK(2);
1887 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1888 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1889 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1890 LATENCY_HIGH_WATERMARK(line_time)));
1891 /* restore original selection */
1892 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1893
1894 /* write the priority marks */
1895 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1896 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1897
1898}
1899
1900void dce6_bandwidth_update(struct radeon_device *rdev)
1901{
1902 struct drm_display_mode *mode0 = NULL;
1903 struct drm_display_mode *mode1 = NULL;
1904 u32 num_heads = 0, lb_size;
1905 int i;
1906
1907 radeon_update_display_priority(rdev);
1908
1909 for (i = 0; i < rdev->num_crtc; i++) {
1910 if (rdev->mode_info.crtcs[i]->base.enabled)
1911 num_heads++;
1912 }
1913 for (i = 0; i < rdev->num_crtc; i += 2) {
1914 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1915 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1916 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1917 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1918 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1919 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1920 }
1921}
1922
Alex Deucher0a96d722012-03-20 17:18:11 -04001923/*
1924 * Core functions
1925 */
Alex Deucher0a96d722012-03-20 17:18:11 -04001926static void si_tiling_mode_table_init(struct radeon_device *rdev)
1927{
1928 const u32 num_tile_mode_states = 32;
1929 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1930
1931 switch (rdev->config.si.mem_row_size_in_kb) {
1932 case 1:
1933 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1934 break;
1935 case 2:
1936 default:
1937 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1938 break;
1939 case 4:
1940 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1941 break;
1942 }
1943
1944 if ((rdev->family == CHIP_TAHITI) ||
1945 (rdev->family == CHIP_PITCAIRN)) {
1946 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1947 switch (reg_offset) {
1948 case 0: /* non-AA compressed depth or any compressed stencil */
1949 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1950 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1951 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1952 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1953 NUM_BANKS(ADDR_SURF_16_BANK) |
1954 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1957 break;
1958 case 1: /* 2xAA/4xAA compressed depth only */
1959 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1960 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1961 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1962 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1963 NUM_BANKS(ADDR_SURF_16_BANK) |
1964 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1967 break;
1968 case 2: /* 8xAA compressed depth only */
1969 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1970 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1971 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1972 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1973 NUM_BANKS(ADDR_SURF_16_BANK) |
1974 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1977 break;
1978 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1979 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1981 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1982 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1983 NUM_BANKS(ADDR_SURF_16_BANK) |
1984 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1987 break;
1988 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1989 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1990 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1991 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1992 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1993 NUM_BANKS(ADDR_SURF_16_BANK) |
1994 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1997 break;
1998 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1999 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2000 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2001 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2002 TILE_SPLIT(split_equal_to_row_size) |
2003 NUM_BANKS(ADDR_SURF_16_BANK) |
2004 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2007 break;
2008 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2009 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2010 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2011 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2012 TILE_SPLIT(split_equal_to_row_size) |
2013 NUM_BANKS(ADDR_SURF_16_BANK) |
2014 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2015 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2016 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2017 break;
2018 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2019 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2020 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2021 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2022 TILE_SPLIT(split_equal_to_row_size) |
2023 NUM_BANKS(ADDR_SURF_16_BANK) |
2024 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2025 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2026 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2027 break;
2028 case 8: /* 1D and 1D Array Surfaces */
2029 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2030 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2031 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2032 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2033 NUM_BANKS(ADDR_SURF_16_BANK) |
2034 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2035 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2036 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2037 break;
2038 case 9: /* Displayable maps. */
2039 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2040 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2041 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2042 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2043 NUM_BANKS(ADDR_SURF_16_BANK) |
2044 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2045 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2046 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2047 break;
2048 case 10: /* Display 8bpp. */
2049 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2050 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2051 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2052 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2053 NUM_BANKS(ADDR_SURF_16_BANK) |
2054 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2055 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2056 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2057 break;
2058 case 11: /* Display 16bpp. */
2059 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2060 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2061 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2062 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2063 NUM_BANKS(ADDR_SURF_16_BANK) |
2064 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2065 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2066 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2067 break;
2068 case 12: /* Display 32bpp. */
2069 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2070 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2071 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2072 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2073 NUM_BANKS(ADDR_SURF_16_BANK) |
2074 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2077 break;
2078 case 13: /* Thin. */
2079 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2080 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2081 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2082 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2083 NUM_BANKS(ADDR_SURF_16_BANK) |
2084 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2085 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2086 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2087 break;
2088 case 14: /* Thin 8 bpp. */
2089 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2090 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2091 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2092 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2093 NUM_BANKS(ADDR_SURF_16_BANK) |
2094 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2095 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2096 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2097 break;
2098 case 15: /* Thin 16 bpp. */
2099 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2100 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2101 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2102 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2103 NUM_BANKS(ADDR_SURF_16_BANK) |
2104 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2105 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2106 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2107 break;
2108 case 16: /* Thin 32 bpp. */
2109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2110 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2111 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2113 NUM_BANKS(ADDR_SURF_16_BANK) |
2114 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2115 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2116 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2117 break;
2118 case 17: /* Thin 64 bpp. */
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2120 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2122 TILE_SPLIT(split_equal_to_row_size) |
2123 NUM_BANKS(ADDR_SURF_16_BANK) |
2124 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2125 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2126 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2127 break;
2128 case 21: /* 8 bpp PRT. */
2129 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2130 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2131 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2132 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2133 NUM_BANKS(ADDR_SURF_16_BANK) |
2134 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2135 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2136 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2137 break;
2138 case 22: /* 16 bpp PRT */
2139 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2140 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2141 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2142 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2143 NUM_BANKS(ADDR_SURF_16_BANK) |
2144 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2147 break;
2148 case 23: /* 32 bpp PRT */
2149 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2150 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2151 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2152 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2153 NUM_BANKS(ADDR_SURF_16_BANK) |
2154 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2155 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2156 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2157 break;
2158 case 24: /* 64 bpp PRT */
2159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2160 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2161 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2162 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2163 NUM_BANKS(ADDR_SURF_16_BANK) |
2164 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2167 break;
2168 case 25: /* 128 bpp PRT */
2169 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2170 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2171 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2172 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2173 NUM_BANKS(ADDR_SURF_8_BANK) |
2174 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2175 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2176 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2177 break;
2178 default:
2179 gb_tile_moden = 0;
2180 break;
2181 }
Jerome Glisse64d7b8b2013-04-09 11:17:08 -04002182 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher0a96d722012-03-20 17:18:11 -04002183 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2184 }
Alex Deucherd0ae7fc2012-07-26 17:42:25 -04002185 } else if ((rdev->family == CHIP_VERDE) ||
Alex Deucher8b028592012-07-31 12:42:48 -04002186 (rdev->family == CHIP_OLAND) ||
2187 (rdev->family == CHIP_HAINAN)) {
Alex Deucher0a96d722012-03-20 17:18:11 -04002188 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2189 switch (reg_offset) {
2190 case 0: /* non-AA compressed depth or any compressed stencil */
2191 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2192 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2193 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2194 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2195 NUM_BANKS(ADDR_SURF_16_BANK) |
2196 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2199 break;
2200 case 1: /* 2xAA/4xAA compressed depth only */
2201 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2202 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2203 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2204 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2205 NUM_BANKS(ADDR_SURF_16_BANK) |
2206 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2209 break;
2210 case 2: /* 8xAA compressed depth only */
2211 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2212 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2213 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2214 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2215 NUM_BANKS(ADDR_SURF_16_BANK) |
2216 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2217 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2218 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2219 break;
2220 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2221 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2222 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2223 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2224 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2225 NUM_BANKS(ADDR_SURF_16_BANK) |
2226 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2227 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2228 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2229 break;
2230 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2231 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2232 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2233 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2234 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2235 NUM_BANKS(ADDR_SURF_16_BANK) |
2236 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2239 break;
2240 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2241 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2242 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2243 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2244 TILE_SPLIT(split_equal_to_row_size) |
2245 NUM_BANKS(ADDR_SURF_16_BANK) |
2246 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2249 break;
2250 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2251 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2253 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2254 TILE_SPLIT(split_equal_to_row_size) |
2255 NUM_BANKS(ADDR_SURF_16_BANK) |
2256 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2259 break;
2260 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2261 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2263 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2264 TILE_SPLIT(split_equal_to_row_size) |
2265 NUM_BANKS(ADDR_SURF_16_BANK) |
2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2269 break;
2270 case 8: /* 1D and 1D Array Surfaces */
2271 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2272 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2273 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2275 NUM_BANKS(ADDR_SURF_16_BANK) |
2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2279 break;
2280 case 9: /* Displayable maps. */
2281 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2282 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2283 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2285 NUM_BANKS(ADDR_SURF_16_BANK) |
2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2289 break;
2290 case 10: /* Display 8bpp. */
2291 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2293 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2295 NUM_BANKS(ADDR_SURF_16_BANK) |
2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2299 break;
2300 case 11: /* Display 16bpp. */
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2302 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2305 NUM_BANKS(ADDR_SURF_16_BANK) |
2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2309 break;
2310 case 12: /* Display 32bpp. */
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2314 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2315 NUM_BANKS(ADDR_SURF_16_BANK) |
2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2319 break;
2320 case 13: /* Thin. */
2321 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2322 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2323 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2324 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2325 NUM_BANKS(ADDR_SURF_16_BANK) |
2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2329 break;
2330 case 14: /* Thin 8 bpp. */
2331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2332 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2333 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2334 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2335 NUM_BANKS(ADDR_SURF_16_BANK) |
2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2339 break;
2340 case 15: /* Thin 16 bpp. */
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2342 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2345 NUM_BANKS(ADDR_SURF_16_BANK) |
2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2349 break;
2350 case 16: /* Thin 32 bpp. */
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2355 NUM_BANKS(ADDR_SURF_16_BANK) |
2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2359 break;
2360 case 17: /* Thin 64 bpp. */
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2363 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2364 TILE_SPLIT(split_equal_to_row_size) |
2365 NUM_BANKS(ADDR_SURF_16_BANK) |
2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2369 break;
2370 case 21: /* 8 bpp PRT. */
2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2372 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2375 NUM_BANKS(ADDR_SURF_16_BANK) |
2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2379 break;
2380 case 22: /* 16 bpp PRT */
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2385 NUM_BANKS(ADDR_SURF_16_BANK) |
2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2389 break;
2390 case 23: /* 32 bpp PRT */
2391 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2395 NUM_BANKS(ADDR_SURF_16_BANK) |
2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2399 break;
2400 case 24: /* 64 bpp PRT */
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2405 NUM_BANKS(ADDR_SURF_16_BANK) |
2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2409 break;
2410 case 25: /* 128 bpp PRT */
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2415 NUM_BANKS(ADDR_SURF_8_BANK) |
2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2419 break;
2420 default:
2421 gb_tile_moden = 0;
2422 break;
2423 }
Jerome Glisse64d7b8b2013-04-09 11:17:08 -04002424 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher0a96d722012-03-20 17:18:11 -04002425 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2426 }
2427 } else
2428 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2429}
2430
Alex Deucher1a8ca752012-06-01 18:58:22 -04002431static void si_select_se_sh(struct radeon_device *rdev,
2432 u32 se_num, u32 sh_num)
2433{
2434 u32 data = INSTANCE_BROADCAST_WRITES;
2435
2436 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucher79b52d62013-04-18 16:26:36 -04002437 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002438 else if (se_num == 0xffffffff)
2439 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2440 else if (sh_num == 0xffffffff)
2441 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2442 else
2443 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2444 WREG32(GRBM_GFX_INDEX, data);
2445}
2446
2447static u32 si_create_bitmask(u32 bit_width)
2448{
2449 u32 i, mask = 0;
2450
2451 for (i = 0; i < bit_width; i++) {
2452 mask <<= 1;
2453 mask |= 1;
2454 }
2455 return mask;
2456}
2457
2458static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2459{
2460 u32 data, mask;
2461
2462 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2463 if (data & 1)
2464 data &= INACTIVE_CUS_MASK;
2465 else
2466 data = 0;
2467 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2468
2469 data >>= INACTIVE_CUS_SHIFT;
2470
2471 mask = si_create_bitmask(cu_per_sh);
2472
2473 return ~data & mask;
2474}
2475
2476static void si_setup_spi(struct radeon_device *rdev,
2477 u32 se_num, u32 sh_per_se,
2478 u32 cu_per_sh)
2479{
2480 int i, j, k;
2481 u32 data, mask, active_cu;
2482
2483 for (i = 0; i < se_num; i++) {
2484 for (j = 0; j < sh_per_se; j++) {
2485 si_select_se_sh(rdev, i, j);
2486 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2487 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2488
2489 mask = 1;
2490 for (k = 0; k < 16; k++) {
2491 mask <<= k;
2492 if (active_cu & mask) {
2493 data &= ~mask;
2494 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2495 break;
2496 }
2497 }
2498 }
2499 }
2500 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2501}
2502
2503static u32 si_get_rb_disabled(struct radeon_device *rdev,
2504 u32 max_rb_num, u32 se_num,
2505 u32 sh_per_se)
2506{
2507 u32 data, mask;
2508
2509 data = RREG32(CC_RB_BACKEND_DISABLE);
2510 if (data & 1)
2511 data &= BACKEND_DISABLE_MASK;
2512 else
2513 data = 0;
2514 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2515
2516 data >>= BACKEND_DISABLE_SHIFT;
2517
2518 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2519
2520 return data & mask;
2521}
2522
2523static void si_setup_rb(struct radeon_device *rdev,
2524 u32 se_num, u32 sh_per_se,
2525 u32 max_rb_num)
2526{
2527 int i, j;
2528 u32 data, mask;
2529 u32 disabled_rbs = 0;
2530 u32 enabled_rbs = 0;
2531
2532 for (i = 0; i < se_num; i++) {
2533 for (j = 0; j < sh_per_se; j++) {
2534 si_select_se_sh(rdev, i, j);
2535 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2536 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2537 }
2538 }
2539 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2540
2541 mask = 1;
2542 for (i = 0; i < max_rb_num; i++) {
2543 if (!(disabled_rbs & mask))
2544 enabled_rbs |= mask;
2545 mask <<= 1;
2546 }
2547
2548 for (i = 0; i < se_num; i++) {
2549 si_select_se_sh(rdev, i, 0xffffffff);
2550 data = 0;
2551 for (j = 0; j < sh_per_se; j++) {
2552 switch (enabled_rbs & 3) {
2553 case 1:
2554 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2555 break;
2556 case 2:
2557 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2558 break;
2559 case 3:
2560 default:
2561 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2562 break;
2563 }
2564 enabled_rbs >>= 2;
2565 }
2566 WREG32(PA_SC_RASTER_CONFIG, data);
2567 }
2568 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2569}
2570
Alex Deucher0a96d722012-03-20 17:18:11 -04002571static void si_gpu_init(struct radeon_device *rdev)
2572{
Alex Deucher0a96d722012-03-20 17:18:11 -04002573 u32 gb_addr_config = 0;
2574 u32 mc_shared_chmap, mc_arb_ramcfg;
Alex Deucher0a96d722012-03-20 17:18:11 -04002575 u32 sx_debug_1;
Alex Deucher0a96d722012-03-20 17:18:11 -04002576 u32 hdp_host_path_cntl;
2577 u32 tmp;
2578 int i, j;
2579
2580 switch (rdev->family) {
2581 case CHIP_TAHITI:
2582 rdev->config.si.max_shader_engines = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04002583 rdev->config.si.max_tile_pipes = 12;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002584 rdev->config.si.max_cu_per_sh = 8;
2585 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04002586 rdev->config.si.max_backends_per_se = 4;
2587 rdev->config.si.max_texture_channel_caches = 12;
2588 rdev->config.si.max_gprs = 256;
2589 rdev->config.si.max_gs_threads = 32;
2590 rdev->config.si.max_hw_contexts = 8;
2591
2592 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2593 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2594 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2595 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002596 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04002597 break;
2598 case CHIP_PITCAIRN:
2599 rdev->config.si.max_shader_engines = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04002600 rdev->config.si.max_tile_pipes = 8;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002601 rdev->config.si.max_cu_per_sh = 5;
2602 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04002603 rdev->config.si.max_backends_per_se = 4;
2604 rdev->config.si.max_texture_channel_caches = 8;
2605 rdev->config.si.max_gprs = 256;
2606 rdev->config.si.max_gs_threads = 32;
2607 rdev->config.si.max_hw_contexts = 8;
2608
2609 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2610 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2611 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2612 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002613 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04002614 break;
2615 case CHIP_VERDE:
2616 default:
2617 rdev->config.si.max_shader_engines = 1;
Alex Deucher0a96d722012-03-20 17:18:11 -04002618 rdev->config.si.max_tile_pipes = 4;
Alex Deucher468ef1a2013-05-21 13:35:19 -04002619 rdev->config.si.max_cu_per_sh = 5;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002620 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04002621 rdev->config.si.max_backends_per_se = 4;
2622 rdev->config.si.max_texture_channel_caches = 4;
2623 rdev->config.si.max_gprs = 256;
2624 rdev->config.si.max_gs_threads = 32;
2625 rdev->config.si.max_hw_contexts = 8;
2626
2627 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2628 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2629 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2630 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002631 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04002632 break;
Alex Deucherd0ae7fc2012-07-26 17:42:25 -04002633 case CHIP_OLAND:
2634 rdev->config.si.max_shader_engines = 1;
2635 rdev->config.si.max_tile_pipes = 4;
2636 rdev->config.si.max_cu_per_sh = 6;
2637 rdev->config.si.max_sh_per_se = 1;
2638 rdev->config.si.max_backends_per_se = 2;
2639 rdev->config.si.max_texture_channel_caches = 4;
2640 rdev->config.si.max_gprs = 256;
2641 rdev->config.si.max_gs_threads = 16;
2642 rdev->config.si.max_hw_contexts = 8;
2643
2644 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2645 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2646 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2647 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2648 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2649 break;
Alex Deucher8b028592012-07-31 12:42:48 -04002650 case CHIP_HAINAN:
2651 rdev->config.si.max_shader_engines = 1;
2652 rdev->config.si.max_tile_pipes = 4;
2653 rdev->config.si.max_cu_per_sh = 5;
2654 rdev->config.si.max_sh_per_se = 1;
2655 rdev->config.si.max_backends_per_se = 1;
2656 rdev->config.si.max_texture_channel_caches = 2;
2657 rdev->config.si.max_gprs = 256;
2658 rdev->config.si.max_gs_threads = 16;
2659 rdev->config.si.max_hw_contexts = 8;
2660
2661 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2662 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2663 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2664 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2665 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2666 break;
Alex Deucher0a96d722012-03-20 17:18:11 -04002667 }
2668
2669 /* Initialize HDP */
2670 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2671 WREG32((0x2c14 + j), 0x00000000);
2672 WREG32((0x2c18 + j), 0x00000000);
2673 WREG32((0x2c1c + j), 0x00000000);
2674 WREG32((0x2c20 + j), 0x00000000);
2675 WREG32((0x2c24 + j), 0x00000000);
2676 }
2677
2678 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2679
2680 evergreen_fix_pci_max_read_req_size(rdev);
2681
2682 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2683
2684 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2685 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2686
Alex Deucher0a96d722012-03-20 17:18:11 -04002687 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
Alex Deucher0a96d722012-03-20 17:18:11 -04002688 rdev->config.si.mem_max_burst_length_bytes = 256;
2689 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2690 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2691 if (rdev->config.si.mem_row_size_in_kb > 4)
2692 rdev->config.si.mem_row_size_in_kb = 4;
2693 /* XXX use MC settings? */
2694 rdev->config.si.shader_engine_tile_size = 32;
2695 rdev->config.si.num_gpus = 1;
2696 rdev->config.si.multi_gpu_tile_size = 64;
2697
Alex Deucher1a8ca752012-06-01 18:58:22 -04002698 /* fix up row size */
2699 gb_addr_config &= ~ROW_SIZE_MASK;
Alex Deucher0a96d722012-03-20 17:18:11 -04002700 switch (rdev->config.si.mem_row_size_in_kb) {
2701 case 1:
2702 default:
2703 gb_addr_config |= ROW_SIZE(0);
2704 break;
2705 case 2:
2706 gb_addr_config |= ROW_SIZE(1);
2707 break;
2708 case 4:
2709 gb_addr_config |= ROW_SIZE(2);
2710 break;
2711 }
2712
Alex Deucher0a96d722012-03-20 17:18:11 -04002713 /* setup tiling info dword. gb_addr_config is not adequate since it does
2714 * not have bank info, so create a custom tiling dword.
2715 * bits 3:0 num_pipes
2716 * bits 7:4 num_banks
2717 * bits 11:8 group_size
2718 * bits 15:12 row_size
2719 */
2720 rdev->config.si.tile_config = 0;
2721 switch (rdev->config.si.num_tile_pipes) {
2722 case 1:
2723 rdev->config.si.tile_config |= (0 << 0);
2724 break;
2725 case 2:
2726 rdev->config.si.tile_config |= (1 << 0);
2727 break;
2728 case 4:
2729 rdev->config.si.tile_config |= (2 << 0);
2730 break;
2731 case 8:
2732 default:
2733 /* XXX what about 12? */
2734 rdev->config.si.tile_config |= (3 << 0);
2735 break;
Christian Königdca571a2012-07-31 13:48:51 +02002736 }
2737 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2738 case 0: /* four banks */
Alex Deucher1a8ca752012-06-01 18:58:22 -04002739 rdev->config.si.tile_config |= 0 << 4;
Christian Königdca571a2012-07-31 13:48:51 +02002740 break;
2741 case 1: /* eight banks */
2742 rdev->config.si.tile_config |= 1 << 4;
2743 break;
2744 case 2: /* sixteen banks */
2745 default:
2746 rdev->config.si.tile_config |= 2 << 4;
2747 break;
2748 }
Alex Deucher0a96d722012-03-20 17:18:11 -04002749 rdev->config.si.tile_config |=
2750 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2751 rdev->config.si.tile_config |=
2752 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2753
Alex Deucher0a96d722012-03-20 17:18:11 -04002754 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2755 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
Alex Deucher7c1c7c12013-04-05 10:28:08 -04002756 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher0a96d722012-03-20 17:18:11 -04002757 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05002758 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2759 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
Alex Deucher1df0d522013-04-26 18:03:44 -04002760 if (rdev->has_uvd) {
2761 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2762 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2763 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2764 }
Alex Deucher0a96d722012-03-20 17:18:11 -04002765
Alex Deucher0a96d722012-03-20 17:18:11 -04002766 si_tiling_mode_table_init(rdev);
2767
Alex Deucher1a8ca752012-06-01 18:58:22 -04002768 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2769 rdev->config.si.max_sh_per_se,
2770 rdev->config.si.max_backends_per_se);
2771
2772 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2773 rdev->config.si.max_sh_per_se,
2774 rdev->config.si.max_cu_per_sh);
2775
2776
Alex Deucher0a96d722012-03-20 17:18:11 -04002777 /* set HW defaults for 3D engine */
2778 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2779 ROQ_IB2_START(0x2b)));
2780 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2781
2782 sx_debug_1 = RREG32(SX_DEBUG_1);
2783 WREG32(SX_DEBUG_1, sx_debug_1);
2784
2785 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2786
2787 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2788 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2789 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2790 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2791
2792 WREG32(VGT_NUM_INSTANCES, 1);
2793
2794 WREG32(CP_PERFMON_CNTL, 0);
2795
2796 WREG32(SQ_CONFIG, 0);
2797
2798 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2799 FORCE_EOV_MAX_REZ_CNT(255)));
2800
2801 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2802 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2803
2804 WREG32(VGT_GS_VERTEX_REUSE, 16);
2805 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2806
2807 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2808 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2809 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2810 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2811 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2812 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2813 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2814 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2815
2816 tmp = RREG32(HDP_MISC_CNTL);
2817 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2818 WREG32(HDP_MISC_CNTL, tmp);
2819
2820 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2821 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2822
2823 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2824
2825 udelay(50);
2826}
Alex Deucherc476dde2012-03-20 17:18:12 -04002827
Alex Deucher48c0c902012-03-20 17:18:19 -04002828/*
Alex Deucher2ece2e82012-03-20 17:18:20 -04002829 * GPU scratch registers helpers function.
2830 */
2831static void si_scratch_init(struct radeon_device *rdev)
2832{
2833 int i;
2834
2835 rdev->scratch.num_reg = 7;
2836 rdev->scratch.reg_base = SCRATCH_REG0;
2837 for (i = 0; i < rdev->scratch.num_reg; i++) {
2838 rdev->scratch.free[i] = true;
2839 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2840 }
2841}
2842
2843void si_fence_ring_emit(struct radeon_device *rdev,
2844 struct radeon_fence *fence)
2845{
2846 struct radeon_ring *ring = &rdev->ring[fence->ring];
2847 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2848
2849 /* flush read cache over gart */
2850 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2851 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2852 radeon_ring_write(ring, 0);
2853 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2854 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2855 PACKET3_TC_ACTION_ENA |
2856 PACKET3_SH_KCACHE_ACTION_ENA |
2857 PACKET3_SH_ICACHE_ACTION_ENA);
2858 radeon_ring_write(ring, 0xFFFFFFFF);
2859 radeon_ring_write(ring, 0);
2860 radeon_ring_write(ring, 10); /* poll interval */
2861 /* EVENT_WRITE_EOP - flush caches, send int */
2862 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2863 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2864 radeon_ring_write(ring, addr & 0xffffffff);
2865 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2866 radeon_ring_write(ring, fence->seq);
2867 radeon_ring_write(ring, 0);
2868}
2869
2870/*
2871 * IB stuff
2872 */
2873void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2874{
Christian König876dc9f2012-05-08 14:24:01 +02002875 struct radeon_ring *ring = &rdev->ring[ib->ring];
Alex Deucher2ece2e82012-03-20 17:18:20 -04002876 u32 header;
2877
Alex Deuchera85a7da42012-07-17 14:02:29 -04002878 if (ib->is_const_ib) {
2879 /* set switch buffer packet before const IB */
2880 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2881 radeon_ring_write(ring, 0);
Christian König45df6802012-07-06 16:22:55 +02002882
Alex Deucher2ece2e82012-03-20 17:18:20 -04002883 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
Alex Deuchera85a7da42012-07-17 14:02:29 -04002884 } else {
Alex Deucher89d35802012-07-17 14:02:31 -04002885 u32 next_rptr;
Alex Deuchera85a7da42012-07-17 14:02:29 -04002886 if (ring->rptr_save_reg) {
Alex Deucher89d35802012-07-17 14:02:31 -04002887 next_rptr = ring->wptr + 3 + 4 + 8;
Alex Deuchera85a7da42012-07-17 14:02:29 -04002888 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2889 radeon_ring_write(ring, ((ring->rptr_save_reg -
2890 PACKET3_SET_CONFIG_REG_START) >> 2));
2891 radeon_ring_write(ring, next_rptr);
Alex Deucher89d35802012-07-17 14:02:31 -04002892 } else if (rdev->wb.enabled) {
2893 next_rptr = ring->wptr + 5 + 4 + 8;
2894 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2895 radeon_ring_write(ring, (1 << 8));
2896 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2897 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2898 radeon_ring_write(ring, next_rptr);
Alex Deuchera85a7da42012-07-17 14:02:29 -04002899 }
2900
Alex Deucher2ece2e82012-03-20 17:18:20 -04002901 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
Alex Deuchera85a7da42012-07-17 14:02:29 -04002902 }
Alex Deucher2ece2e82012-03-20 17:18:20 -04002903
2904 radeon_ring_write(ring, header);
2905 radeon_ring_write(ring,
2906#ifdef __BIG_ENDIAN
2907 (2 << 0) |
2908#endif
2909 (ib->gpu_addr & 0xFFFFFFFC));
2910 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
Christian König4bf3dd92012-08-06 18:57:44 +02002911 radeon_ring_write(ring, ib->length_dw |
2912 (ib->vm ? (ib->vm->id << 24) : 0));
Alex Deucher2ece2e82012-03-20 17:18:20 -04002913
Alex Deuchera85a7da42012-07-17 14:02:29 -04002914 if (!ib->is_const_ib) {
2915 /* flush read cache over gart for this vmid */
2916 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2917 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
Christian König4bf3dd92012-08-06 18:57:44 +02002918 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
Alex Deuchera85a7da42012-07-17 14:02:29 -04002919 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2920 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2921 PACKET3_TC_ACTION_ENA |
2922 PACKET3_SH_KCACHE_ACTION_ENA |
2923 PACKET3_SH_ICACHE_ACTION_ENA);
2924 radeon_ring_write(ring, 0xFFFFFFFF);
2925 radeon_ring_write(ring, 0);
2926 radeon_ring_write(ring, 10); /* poll interval */
2927 }
Alex Deucher2ece2e82012-03-20 17:18:20 -04002928}
2929
2930/*
Alex Deucher48c0c902012-03-20 17:18:19 -04002931 * CP.
2932 */
2933static void si_cp_enable(struct radeon_device *rdev, bool enable)
2934{
2935 if (enable)
2936 WREG32(CP_ME_CNTL, 0);
2937 else {
2938 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2939 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2940 WREG32(SCRATCH_UMSK, 0);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05002941 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2942 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2943 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
Alex Deucher48c0c902012-03-20 17:18:19 -04002944 }
2945 udelay(50);
2946}
2947
2948static int si_cp_load_microcode(struct radeon_device *rdev)
2949{
2950 const __be32 *fw_data;
2951 int i;
2952
2953 if (!rdev->me_fw || !rdev->pfp_fw)
2954 return -EINVAL;
2955
2956 si_cp_enable(rdev, false);
2957
2958 /* PFP */
2959 fw_data = (const __be32 *)rdev->pfp_fw->data;
2960 WREG32(CP_PFP_UCODE_ADDR, 0);
2961 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
2962 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2963 WREG32(CP_PFP_UCODE_ADDR, 0);
2964
2965 /* CE */
2966 fw_data = (const __be32 *)rdev->ce_fw->data;
2967 WREG32(CP_CE_UCODE_ADDR, 0);
2968 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
2969 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2970 WREG32(CP_CE_UCODE_ADDR, 0);
2971
2972 /* ME */
2973 fw_data = (const __be32 *)rdev->me_fw->data;
2974 WREG32(CP_ME_RAM_WADDR, 0);
2975 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
2976 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2977 WREG32(CP_ME_RAM_WADDR, 0);
2978
2979 WREG32(CP_PFP_UCODE_ADDR, 0);
2980 WREG32(CP_CE_UCODE_ADDR, 0);
2981 WREG32(CP_ME_RAM_WADDR, 0);
2982 WREG32(CP_ME_RAM_RADDR, 0);
2983 return 0;
2984}
2985
2986static int si_cp_start(struct radeon_device *rdev)
2987{
2988 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2989 int r, i;
2990
2991 r = radeon_ring_lock(rdev, ring, 7 + 4);
2992 if (r) {
2993 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2994 return r;
2995 }
2996 /* init the CP */
2997 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2998 radeon_ring_write(ring, 0x1);
2999 radeon_ring_write(ring, 0x0);
3000 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3001 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3002 radeon_ring_write(ring, 0);
3003 radeon_ring_write(ring, 0);
3004
3005 /* init the CE partitions */
3006 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3007 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3008 radeon_ring_write(ring, 0xc000);
3009 radeon_ring_write(ring, 0xe000);
3010 radeon_ring_unlock_commit(rdev, ring);
3011
3012 si_cp_enable(rdev, true);
3013
3014 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3015 if (r) {
3016 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3017 return r;
3018 }
3019
3020 /* setup clear context state */
3021 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3022 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3023
3024 for (i = 0; i < si_default_size; i++)
3025 radeon_ring_write(ring, si_default_state[i]);
3026
3027 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3028 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3029
3030 /* set clear context state */
3031 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3032 radeon_ring_write(ring, 0);
3033
3034 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3035 radeon_ring_write(ring, 0x00000316);
3036 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3037 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3038
3039 radeon_ring_unlock_commit(rdev, ring);
3040
3041 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3042 ring = &rdev->ring[i];
3043 r = radeon_ring_lock(rdev, ring, 2);
3044
3045 /* clear the compute context state */
3046 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3047 radeon_ring_write(ring, 0);
3048
3049 radeon_ring_unlock_commit(rdev, ring);
3050 }
3051
3052 return 0;
3053}
3054
3055static void si_cp_fini(struct radeon_device *rdev)
3056{
Christian König45df6802012-07-06 16:22:55 +02003057 struct radeon_ring *ring;
Alex Deucher48c0c902012-03-20 17:18:19 -04003058 si_cp_enable(rdev, false);
Christian König45df6802012-07-06 16:22:55 +02003059
3060 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3061 radeon_ring_fini(rdev, ring);
3062 radeon_scratch_free(rdev, ring->rptr_save_reg);
3063
3064 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3065 radeon_ring_fini(rdev, ring);
3066 radeon_scratch_free(rdev, ring->rptr_save_reg);
3067
3068 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3069 radeon_ring_fini(rdev, ring);
3070 radeon_scratch_free(rdev, ring->rptr_save_reg);
Alex Deucher48c0c902012-03-20 17:18:19 -04003071}
3072
3073static int si_cp_resume(struct radeon_device *rdev)
3074{
3075 struct radeon_ring *ring;
3076 u32 tmp;
3077 u32 rb_bufsz;
3078 int r;
3079
3080 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3081 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3082 SOFT_RESET_PA |
3083 SOFT_RESET_VGT |
3084 SOFT_RESET_SPI |
3085 SOFT_RESET_SX));
3086 RREG32(GRBM_SOFT_RESET);
3087 mdelay(15);
3088 WREG32(GRBM_SOFT_RESET, 0);
3089 RREG32(GRBM_SOFT_RESET);
3090
3091 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3092 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3093
3094 /* Set the write pointer delay */
3095 WREG32(CP_RB_WPTR_DELAY, 0);
3096
3097 WREG32(CP_DEBUG, 0);
3098 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3099
3100 /* ring 0 - compute and gfx */
3101 /* Set ring buffer size */
3102 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3103 rb_bufsz = drm_order(ring->ring_size / 8);
3104 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3105#ifdef __BIG_ENDIAN
3106 tmp |= BUF_SWAP_32BIT;
3107#endif
3108 WREG32(CP_RB0_CNTL, tmp);
3109
3110 /* Initialize the ring buffer's read and write pointers */
3111 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3112 ring->wptr = 0;
3113 WREG32(CP_RB0_WPTR, ring->wptr);
3114
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003115 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003116 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3117 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3118
3119 if (rdev->wb.enabled)
3120 WREG32(SCRATCH_UMSK, 0xff);
3121 else {
3122 tmp |= RB_NO_UPDATE;
3123 WREG32(SCRATCH_UMSK, 0);
3124 }
3125
3126 mdelay(1);
3127 WREG32(CP_RB0_CNTL, tmp);
3128
3129 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3130
3131 ring->rptr = RREG32(CP_RB0_RPTR);
3132
3133 /* ring1 - compute only */
3134 /* Set ring buffer size */
3135 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3136 rb_bufsz = drm_order(ring->ring_size / 8);
3137 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3138#ifdef __BIG_ENDIAN
3139 tmp |= BUF_SWAP_32BIT;
3140#endif
3141 WREG32(CP_RB1_CNTL, tmp);
3142
3143 /* Initialize the ring buffer's read and write pointers */
3144 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3145 ring->wptr = 0;
3146 WREG32(CP_RB1_WPTR, ring->wptr);
3147
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003148 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003149 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3150 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3151
3152 mdelay(1);
3153 WREG32(CP_RB1_CNTL, tmp);
3154
3155 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3156
3157 ring->rptr = RREG32(CP_RB1_RPTR);
3158
3159 /* ring2 - compute only */
3160 /* Set ring buffer size */
3161 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3162 rb_bufsz = drm_order(ring->ring_size / 8);
3163 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3164#ifdef __BIG_ENDIAN
3165 tmp |= BUF_SWAP_32BIT;
3166#endif
3167 WREG32(CP_RB2_CNTL, tmp);
3168
3169 /* Initialize the ring buffer's read and write pointers */
3170 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3171 ring->wptr = 0;
3172 WREG32(CP_RB2_WPTR, ring->wptr);
3173
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003174 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003175 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3176 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3177
3178 mdelay(1);
3179 WREG32(CP_RB2_CNTL, tmp);
3180
3181 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3182
3183 ring->rptr = RREG32(CP_RB2_RPTR);
3184
3185 /* start the rings */
3186 si_cp_start(rdev);
3187 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3188 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3189 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3190 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3191 if (r) {
3192 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3193 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3194 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3195 return r;
3196 }
3197 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3198 if (r) {
3199 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3200 }
3201 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3202 if (r) {
3203 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3204 }
3205
3206 return 0;
3207}
3208
Alex Deucher014bb202013-01-18 19:36:20 -05003209static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3210{
3211 u32 reset_mask = 0;
3212 u32 tmp;
3213
3214 /* GRBM_STATUS */
3215 tmp = RREG32(GRBM_STATUS);
3216 if (tmp & (PA_BUSY | SC_BUSY |
3217 BCI_BUSY | SX_BUSY |
3218 TA_BUSY | VGT_BUSY |
3219 DB_BUSY | CB_BUSY |
3220 GDS_BUSY | SPI_BUSY |
3221 IA_BUSY | IA_BUSY_NO_DMA))
3222 reset_mask |= RADEON_RESET_GFX;
3223
3224 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3225 CP_BUSY | CP_COHERENCY_BUSY))
3226 reset_mask |= RADEON_RESET_CP;
3227
3228 if (tmp & GRBM_EE_BUSY)
3229 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3230
3231 /* GRBM_STATUS2 */
3232 tmp = RREG32(GRBM_STATUS2);
3233 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3234 reset_mask |= RADEON_RESET_RLC;
3235
3236 /* DMA_STATUS_REG 0 */
3237 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3238 if (!(tmp & DMA_IDLE))
3239 reset_mask |= RADEON_RESET_DMA;
3240
3241 /* DMA_STATUS_REG 1 */
3242 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3243 if (!(tmp & DMA_IDLE))
3244 reset_mask |= RADEON_RESET_DMA1;
3245
3246 /* SRBM_STATUS2 */
3247 tmp = RREG32(SRBM_STATUS2);
3248 if (tmp & DMA_BUSY)
3249 reset_mask |= RADEON_RESET_DMA;
3250
3251 if (tmp & DMA1_BUSY)
3252 reset_mask |= RADEON_RESET_DMA1;
3253
3254 /* SRBM_STATUS */
3255 tmp = RREG32(SRBM_STATUS);
3256
3257 if (tmp & IH_BUSY)
3258 reset_mask |= RADEON_RESET_IH;
3259
3260 if (tmp & SEM_BUSY)
3261 reset_mask |= RADEON_RESET_SEM;
3262
3263 if (tmp & GRBM_RQ_PENDING)
3264 reset_mask |= RADEON_RESET_GRBM;
3265
3266 if (tmp & VMC_BUSY)
3267 reset_mask |= RADEON_RESET_VMC;
3268
3269 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3270 MCC_BUSY | MCD_BUSY))
3271 reset_mask |= RADEON_RESET_MC;
3272
3273 if (evergreen_is_display_hung(rdev))
3274 reset_mask |= RADEON_RESET_DISPLAY;
3275
3276 /* VM_L2_STATUS */
3277 tmp = RREG32(VM_L2_STATUS);
3278 if (tmp & L2_BUSY)
3279 reset_mask |= RADEON_RESET_VMC;
3280
Alex Deucherd808fc82013-02-28 10:03:08 -05003281 /* Skip MC reset as it's mostly likely not hung, just busy */
3282 if (reset_mask & RADEON_RESET_MC) {
3283 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3284 reset_mask &= ~RADEON_RESET_MC;
3285 }
3286
Alex Deucher014bb202013-01-18 19:36:20 -05003287 return reset_mask;
3288}
3289
3290static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
Alex Deucher06bc6df2013-01-03 13:15:30 -05003291{
3292 struct evergreen_mc_save save;
Alex Deucher1c534672013-01-18 15:08:38 -05003293 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3294 u32 tmp;
Alex Deucher19fc42e2013-01-14 11:04:39 -05003295
Alex Deucher06bc6df2013-01-03 13:15:30 -05003296 if (reset_mask == 0)
Alex Deucher014bb202013-01-18 19:36:20 -05003297 return;
Alex Deucher06bc6df2013-01-03 13:15:30 -05003298
3299 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3300
Alex Deucher1c534672013-01-18 15:08:38 -05003301 evergreen_print_gpu_status_regs(rdev);
Alex Deucher06bc6df2013-01-03 13:15:30 -05003302 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3303 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3304 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3305 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3306
Alex Deucher1c534672013-01-18 15:08:38 -05003307 /* Disable CP parsing/prefetching */
3308 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3309
3310 if (reset_mask & RADEON_RESET_DMA) {
3311 /* dma0 */
3312 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3313 tmp &= ~DMA_RB_ENABLE;
3314 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
Alex Deucher014bb202013-01-18 19:36:20 -05003315 }
3316 if (reset_mask & RADEON_RESET_DMA1) {
Alex Deucher1c534672013-01-18 15:08:38 -05003317 /* dma1 */
3318 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3319 tmp &= ~DMA_RB_ENABLE;
3320 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3321 }
3322
Alex Deucherf770d782013-01-23 19:00:25 -05003323 udelay(50);
3324
3325 evergreen_mc_stop(rdev, &save);
3326 if (evergreen_mc_wait_for_idle(rdev)) {
3327 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3328 }
3329
Alex Deucher1c534672013-01-18 15:08:38 -05003330 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3331 grbm_soft_reset = SOFT_RESET_CB |
3332 SOFT_RESET_DB |
3333 SOFT_RESET_GDS |
3334 SOFT_RESET_PA |
3335 SOFT_RESET_SC |
3336 SOFT_RESET_BCI |
3337 SOFT_RESET_SPI |
3338 SOFT_RESET_SX |
3339 SOFT_RESET_TC |
3340 SOFT_RESET_TA |
3341 SOFT_RESET_VGT |
3342 SOFT_RESET_IA;
3343 }
3344
3345 if (reset_mask & RADEON_RESET_CP) {
3346 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3347
3348 srbm_soft_reset |= SOFT_RESET_GRBM;
3349 }
Alex Deucher06bc6df2013-01-03 13:15:30 -05003350
3351 if (reset_mask & RADEON_RESET_DMA)
Alex Deucher014bb202013-01-18 19:36:20 -05003352 srbm_soft_reset |= SOFT_RESET_DMA;
3353
3354 if (reset_mask & RADEON_RESET_DMA1)
3355 srbm_soft_reset |= SOFT_RESET_DMA1;
3356
3357 if (reset_mask & RADEON_RESET_DISPLAY)
3358 srbm_soft_reset |= SOFT_RESET_DC;
3359
3360 if (reset_mask & RADEON_RESET_RLC)
3361 grbm_soft_reset |= SOFT_RESET_RLC;
3362
3363 if (reset_mask & RADEON_RESET_SEM)
3364 srbm_soft_reset |= SOFT_RESET_SEM;
3365
3366 if (reset_mask & RADEON_RESET_IH)
3367 srbm_soft_reset |= SOFT_RESET_IH;
3368
3369 if (reset_mask & RADEON_RESET_GRBM)
3370 srbm_soft_reset |= SOFT_RESET_GRBM;
3371
3372 if (reset_mask & RADEON_RESET_VMC)
3373 srbm_soft_reset |= SOFT_RESET_VMC;
3374
3375 if (reset_mask & RADEON_RESET_MC)
3376 srbm_soft_reset |= SOFT_RESET_MC;
Alex Deucher1c534672013-01-18 15:08:38 -05003377
3378 if (grbm_soft_reset) {
3379 tmp = RREG32(GRBM_SOFT_RESET);
3380 tmp |= grbm_soft_reset;
3381 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3382 WREG32(GRBM_SOFT_RESET, tmp);
3383 tmp = RREG32(GRBM_SOFT_RESET);
3384
3385 udelay(50);
3386
3387 tmp &= ~grbm_soft_reset;
3388 WREG32(GRBM_SOFT_RESET, tmp);
3389 tmp = RREG32(GRBM_SOFT_RESET);
3390 }
3391
3392 if (srbm_soft_reset) {
3393 tmp = RREG32(SRBM_SOFT_RESET);
3394 tmp |= srbm_soft_reset;
3395 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3396 WREG32(SRBM_SOFT_RESET, tmp);
3397 tmp = RREG32(SRBM_SOFT_RESET);
3398
3399 udelay(50);
3400
3401 tmp &= ~srbm_soft_reset;
3402 WREG32(SRBM_SOFT_RESET, tmp);
3403 tmp = RREG32(SRBM_SOFT_RESET);
3404 }
Alex Deucher06bc6df2013-01-03 13:15:30 -05003405
3406 /* Wait a little for things to settle down */
3407 udelay(50);
3408
Alex Deucherc476dde2012-03-20 17:18:12 -04003409 evergreen_mc_resume(rdev, &save);
Alex Deucher1c534672013-01-18 15:08:38 -05003410 udelay(50);
Alex Deucher410a3412013-01-18 13:05:39 -05003411
Alex Deucher1c534672013-01-18 15:08:38 -05003412 evergreen_print_gpu_status_regs(rdev);
Alex Deucherc476dde2012-03-20 17:18:12 -04003413}
3414
3415int si_asic_reset(struct radeon_device *rdev)
3416{
Alex Deucher014bb202013-01-18 19:36:20 -05003417 u32 reset_mask;
3418
3419 reset_mask = si_gpu_check_soft_reset(rdev);
3420
3421 if (reset_mask)
3422 r600_set_bios_scratch_engine_hung(rdev, true);
3423
3424 si_gpu_soft_reset(rdev, reset_mask);
3425
3426 reset_mask = si_gpu_check_soft_reset(rdev);
3427
3428 if (!reset_mask)
3429 r600_set_bios_scratch_engine_hung(rdev, false);
3430
3431 return 0;
Alex Deucherc476dde2012-03-20 17:18:12 -04003432}
3433
Alex Deucher123bc182013-01-24 11:37:19 -05003434/**
3435 * si_gfx_is_lockup - Check if the GFX engine is locked up
3436 *
3437 * @rdev: radeon_device pointer
3438 * @ring: radeon_ring structure holding ring information
3439 *
3440 * Check if the GFX engine is locked up.
3441 * Returns true if the engine appears to be locked up, false if not.
3442 */
3443bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3444{
3445 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3446
3447 if (!(reset_mask & (RADEON_RESET_GFX |
3448 RADEON_RESET_COMPUTE |
3449 RADEON_RESET_CP))) {
3450 radeon_ring_lockup_update(ring);
3451 return false;
3452 }
3453 /* force CP activities */
3454 radeon_ring_force_activity(rdev, ring);
3455 return radeon_ring_test_lockup(rdev, ring);
3456}
3457
3458/**
3459 * si_dma_is_lockup - Check if the DMA engine is locked up
3460 *
3461 * @rdev: radeon_device pointer
3462 * @ring: radeon_ring structure holding ring information
3463 *
3464 * Check if the async DMA engine is locked up.
3465 * Returns true if the engine appears to be locked up, false if not.
3466 */
3467bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3468{
3469 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3470 u32 mask;
3471
3472 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3473 mask = RADEON_RESET_DMA;
3474 else
3475 mask = RADEON_RESET_DMA1;
3476
3477 if (!(reset_mask & mask)) {
3478 radeon_ring_lockup_update(ring);
3479 return false;
3480 }
3481 /* force ring activities */
3482 radeon_ring_force_activity(rdev, ring);
3483 return radeon_ring_test_lockup(rdev, ring);
3484}
3485
Alex Deucherd2800ee2012-03-20 17:18:13 -04003486/* MC */
3487static void si_mc_program(struct radeon_device *rdev)
3488{
3489 struct evergreen_mc_save save;
3490 u32 tmp;
3491 int i, j;
3492
3493 /* Initialize HDP */
3494 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3495 WREG32((0x2c14 + j), 0x00000000);
3496 WREG32((0x2c18 + j), 0x00000000);
3497 WREG32((0x2c1c + j), 0x00000000);
3498 WREG32((0x2c20 + j), 0x00000000);
3499 WREG32((0x2c24 + j), 0x00000000);
3500 }
3501 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3502
3503 evergreen_mc_stop(rdev, &save);
3504 if (radeon_mc_wait_for_idle(rdev)) {
3505 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3506 }
Alex Deucher51535502012-08-30 14:34:30 -04003507 if (!ASIC_IS_NODCE(rdev))
3508 /* Lockout access through VGA aperture*/
3509 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
Alex Deucherd2800ee2012-03-20 17:18:13 -04003510 /* Update configuration */
3511 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3512 rdev->mc.vram_start >> 12);
3513 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3514 rdev->mc.vram_end >> 12);
3515 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3516 rdev->vram_scratch.gpu_addr >> 12);
3517 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3518 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3519 WREG32(MC_VM_FB_LOCATION, tmp);
3520 /* XXX double check these! */
3521 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3522 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3523 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3524 WREG32(MC_VM_AGP_BASE, 0);
3525 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3526 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3527 if (radeon_mc_wait_for_idle(rdev)) {
3528 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3529 }
3530 evergreen_mc_resume(rdev, &save);
Alex Deucher51535502012-08-30 14:34:30 -04003531 if (!ASIC_IS_NODCE(rdev)) {
3532 /* we need to own VRAM, so turn off the VGA renderer here
3533 * to stop it overwriting our objects */
3534 rv515_vga_render_disable(rdev);
3535 }
Alex Deucherd2800ee2012-03-20 17:18:13 -04003536}
3537
Alex Deucherd2800ee2012-03-20 17:18:13 -04003538static void si_vram_gtt_location(struct radeon_device *rdev,
3539 struct radeon_mc *mc)
3540{
3541 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3542 /* leave room for at least 1024M GTT */
3543 dev_warn(rdev->dev, "limiting VRAM\n");
3544 mc->real_vram_size = 0xFFC0000000ULL;
3545 mc->mc_vram_size = 0xFFC0000000ULL;
3546 }
Alex Deucher9ed8b1f2013-04-08 11:13:01 -04003547 radeon_vram_location(rdev, &rdev->mc, 0);
Alex Deucherd2800ee2012-03-20 17:18:13 -04003548 rdev->mc.gtt_base_align = 0;
Alex Deucher9ed8b1f2013-04-08 11:13:01 -04003549 radeon_gtt_location(rdev, mc);
Alex Deucherd2800ee2012-03-20 17:18:13 -04003550}
3551
3552static int si_mc_init(struct radeon_device *rdev)
3553{
3554 u32 tmp;
3555 int chansize, numchan;
3556
3557 /* Get VRAM informations */
3558 rdev->mc.vram_is_ddr = true;
3559 tmp = RREG32(MC_ARB_RAMCFG);
3560 if (tmp & CHANSIZE_OVERRIDE) {
3561 chansize = 16;
3562 } else if (tmp & CHANSIZE_MASK) {
3563 chansize = 64;
3564 } else {
3565 chansize = 32;
3566 }
3567 tmp = RREG32(MC_SHARED_CHMAP);
3568 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3569 case 0:
3570 default:
3571 numchan = 1;
3572 break;
3573 case 1:
3574 numchan = 2;
3575 break;
3576 case 2:
3577 numchan = 4;
3578 break;
3579 case 3:
3580 numchan = 8;
3581 break;
3582 case 4:
3583 numchan = 3;
3584 break;
3585 case 5:
3586 numchan = 6;
3587 break;
3588 case 6:
3589 numchan = 10;
3590 break;
3591 case 7:
3592 numchan = 12;
3593 break;
3594 case 8:
3595 numchan = 16;
3596 break;
3597 }
3598 rdev->mc.vram_width = numchan * chansize;
3599 /* Could aper size report 0 ? */
3600 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3601 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3602 /* size in MB on si */
Niels Ole Salscheiderfc986032013-05-18 21:19:23 +02003603 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3604 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucherd2800ee2012-03-20 17:18:13 -04003605 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3606 si_vram_gtt_location(rdev, &rdev->mc);
3607 radeon_update_bandwidth_info(rdev);
3608
3609 return 0;
3610}
3611
3612/*
3613 * GART
3614 */
3615void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3616{
3617 /* flush hdp cache */
3618 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3619
3620 /* bits 0-15 are the VM contexts0-15 */
3621 WREG32(VM_INVALIDATE_REQUEST, 1);
3622}
3623
Lauri Kasanen1109ca02012-08-31 13:43:50 -04003624static int si_pcie_gart_enable(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04003625{
3626 int r, i;
3627
3628 if (rdev->gart.robj == NULL) {
3629 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3630 return -EINVAL;
3631 }
3632 r = radeon_gart_table_vram_pin(rdev);
3633 if (r)
3634 return r;
3635 radeon_gart_restore(rdev);
3636 /* Setup TLB control */
3637 WREG32(MC_VM_MX_L1_TLB_CNTL,
3638 (0xA << 7) |
3639 ENABLE_L1_TLB |
3640 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3641 ENABLE_ADVANCED_DRIVER_MODEL |
3642 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3643 /* Setup L2 cache */
3644 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3645 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3646 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3647 EFFECTIVE_L2_QUEUE_SIZE(7) |
3648 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3649 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3650 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3651 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3652 /* setup context0 */
3653 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3654 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3655 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3656 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3657 (u32)(rdev->dummy_page.addr >> 12));
3658 WREG32(VM_CONTEXT0_CNTL2, 0);
3659 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3660 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3661
3662 WREG32(0x15D4, 0);
3663 WREG32(0x15D8, 0);
3664 WREG32(0x15DC, 0);
3665
3666 /* empty context1-15 */
Alex Deucherd2800ee2012-03-20 17:18:13 -04003667 /* set vm size, must be a multiple of 4 */
3668 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
Alex Deucherc21b3282012-06-28 17:53:07 -04003669 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
Alex Deucher23d4f1f2012-10-08 09:45:46 -04003670 /* Assign the pt base to something valid for now; the pts used for
3671 * the VMs are determined by the application and setup and assigned
3672 * on the fly in the vm part of radeon_gart.c
3673 */
Alex Deucherd2800ee2012-03-20 17:18:13 -04003674 for (i = 1; i < 16; i++) {
3675 if (i < 8)
3676 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3677 rdev->gart.table_addr >> 12);
3678 else
3679 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3680 rdev->gart.table_addr >> 12);
3681 }
3682
3683 /* enable context1-15 */
3684 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3685 (u32)(rdev->dummy_page.addr >> 12));
Christian Königae133a12012-09-18 15:30:44 -04003686 WREG32(VM_CONTEXT1_CNTL2, 4);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02003687 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Christian Königae133a12012-09-18 15:30:44 -04003688 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3689 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3690 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3691 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3692 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3693 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3694 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3695 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3696 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3697 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3698 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3699 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucherd2800ee2012-03-20 17:18:13 -04003700
3701 si_pcie_gart_tlb_flush(rdev);
3702 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3703 (unsigned)(rdev->mc.gtt_size >> 20),
3704 (unsigned long long)rdev->gart.table_addr);
3705 rdev->gart.ready = true;
3706 return 0;
3707}
3708
Lauri Kasanen1109ca02012-08-31 13:43:50 -04003709static void si_pcie_gart_disable(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04003710{
3711 /* Disable all tables */
3712 WREG32(VM_CONTEXT0_CNTL, 0);
3713 WREG32(VM_CONTEXT1_CNTL, 0);
3714 /* Setup TLB control */
3715 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3716 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3717 /* Setup L2 cache */
3718 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3719 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3720 EFFECTIVE_L2_QUEUE_SIZE(7) |
3721 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3722 WREG32(VM_L2_CNTL2, 0);
3723 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3724 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3725 radeon_gart_table_vram_unpin(rdev);
3726}
3727
Lauri Kasanen1109ca02012-08-31 13:43:50 -04003728static void si_pcie_gart_fini(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04003729{
3730 si_pcie_gart_disable(rdev);
3731 radeon_gart_table_vram_free(rdev);
3732 radeon_gart_fini(rdev);
3733}
3734
Alex Deucher498dd8b2012-03-20 17:18:15 -04003735/* vm parser */
3736static bool si_vm_reg_valid(u32 reg)
3737{
3738 /* context regs are fine */
3739 if (reg >= 0x28000)
3740 return true;
3741
3742 /* check config regs */
3743 switch (reg) {
3744 case GRBM_GFX_INDEX:
Alex Deucherf418b882012-11-08 10:13:24 -05003745 case CP_STRMOUT_CNTL:
Alex Deucher498dd8b2012-03-20 17:18:15 -04003746 case VGT_VTX_VECT_EJECT_REG:
3747 case VGT_CACHE_INVALIDATION:
3748 case VGT_ESGS_RING_SIZE:
3749 case VGT_GSVS_RING_SIZE:
3750 case VGT_GS_VERTEX_REUSE:
3751 case VGT_PRIMITIVE_TYPE:
3752 case VGT_INDEX_TYPE:
3753 case VGT_NUM_INDICES:
3754 case VGT_NUM_INSTANCES:
3755 case VGT_TF_RING_SIZE:
3756 case VGT_HS_OFFCHIP_PARAM:
3757 case VGT_TF_MEMORY_BASE:
3758 case PA_CL_ENHANCE:
3759 case PA_SU_LINE_STIPPLE_VALUE:
3760 case PA_SC_LINE_STIPPLE_STATE:
3761 case PA_SC_ENHANCE:
3762 case SQC_CACHES:
3763 case SPI_STATIC_THREAD_MGMT_1:
3764 case SPI_STATIC_THREAD_MGMT_2:
3765 case SPI_STATIC_THREAD_MGMT_3:
3766 case SPI_PS_MAX_WAVE_ID:
3767 case SPI_CONFIG_CNTL:
3768 case SPI_CONFIG_CNTL_1:
3769 case TA_CNTL_AUX:
3770 return true;
3771 default:
3772 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3773 return false;
3774 }
3775}
3776
3777static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3778 u32 *ib, struct radeon_cs_packet *pkt)
3779{
3780 switch (pkt->opcode) {
3781 case PACKET3_NOP:
3782 case PACKET3_SET_BASE:
3783 case PACKET3_SET_CE_DE_COUNTERS:
3784 case PACKET3_LOAD_CONST_RAM:
3785 case PACKET3_WRITE_CONST_RAM:
3786 case PACKET3_WRITE_CONST_RAM_OFFSET:
3787 case PACKET3_DUMP_CONST_RAM:
3788 case PACKET3_INCREMENT_CE_COUNTER:
3789 case PACKET3_WAIT_ON_DE_COUNTER:
3790 case PACKET3_CE_WRITE:
3791 break;
3792 default:
3793 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3794 return -EINVAL;
3795 }
3796 return 0;
3797}
3798
3799static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3800 u32 *ib, struct radeon_cs_packet *pkt)
3801{
3802 u32 idx = pkt->idx + 1;
3803 u32 idx_value = ib[idx];
3804 u32 start_reg, end_reg, reg, i;
Alex Deucher5aa709b2012-12-03 19:42:37 -05003805 u32 command, info;
Alex Deucher498dd8b2012-03-20 17:18:15 -04003806
3807 switch (pkt->opcode) {
3808 case PACKET3_NOP:
3809 case PACKET3_SET_BASE:
3810 case PACKET3_CLEAR_STATE:
3811 case PACKET3_INDEX_BUFFER_SIZE:
3812 case PACKET3_DISPATCH_DIRECT:
3813 case PACKET3_DISPATCH_INDIRECT:
3814 case PACKET3_ALLOC_GDS:
3815 case PACKET3_WRITE_GDS_RAM:
3816 case PACKET3_ATOMIC_GDS:
3817 case PACKET3_ATOMIC:
3818 case PACKET3_OCCLUSION_QUERY:
3819 case PACKET3_SET_PREDICATION:
3820 case PACKET3_COND_EXEC:
3821 case PACKET3_PRED_EXEC:
3822 case PACKET3_DRAW_INDIRECT:
3823 case PACKET3_DRAW_INDEX_INDIRECT:
3824 case PACKET3_INDEX_BASE:
3825 case PACKET3_DRAW_INDEX_2:
3826 case PACKET3_CONTEXT_CONTROL:
3827 case PACKET3_INDEX_TYPE:
3828 case PACKET3_DRAW_INDIRECT_MULTI:
3829 case PACKET3_DRAW_INDEX_AUTO:
3830 case PACKET3_DRAW_INDEX_IMMD:
3831 case PACKET3_NUM_INSTANCES:
3832 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3833 case PACKET3_STRMOUT_BUFFER_UPDATE:
3834 case PACKET3_DRAW_INDEX_OFFSET_2:
3835 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3836 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3837 case PACKET3_MPEG_INDEX:
3838 case PACKET3_WAIT_REG_MEM:
3839 case PACKET3_MEM_WRITE:
3840 case PACKET3_PFP_SYNC_ME:
3841 case PACKET3_SURFACE_SYNC:
3842 case PACKET3_EVENT_WRITE:
3843 case PACKET3_EVENT_WRITE_EOP:
3844 case PACKET3_EVENT_WRITE_EOS:
3845 case PACKET3_SET_CONTEXT_REG:
3846 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3847 case PACKET3_SET_SH_REG:
3848 case PACKET3_SET_SH_REG_OFFSET:
3849 case PACKET3_INCREMENT_DE_COUNTER:
3850 case PACKET3_WAIT_ON_CE_COUNTER:
3851 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3852 case PACKET3_ME_WRITE:
3853 break;
3854 case PACKET3_COPY_DATA:
3855 if ((idx_value & 0xf00) == 0) {
3856 reg = ib[idx + 3] * 4;
3857 if (!si_vm_reg_valid(reg))
3858 return -EINVAL;
3859 }
3860 break;
3861 case PACKET3_WRITE_DATA:
3862 if ((idx_value & 0xf00) == 0) {
3863 start_reg = ib[idx + 1] * 4;
3864 if (idx_value & 0x10000) {
3865 if (!si_vm_reg_valid(start_reg))
3866 return -EINVAL;
3867 } else {
3868 for (i = 0; i < (pkt->count - 2); i++) {
3869 reg = start_reg + (4 * i);
3870 if (!si_vm_reg_valid(reg))
3871 return -EINVAL;
3872 }
3873 }
3874 }
3875 break;
3876 case PACKET3_COND_WRITE:
3877 if (idx_value & 0x100) {
3878 reg = ib[idx + 5] * 4;
3879 if (!si_vm_reg_valid(reg))
3880 return -EINVAL;
3881 }
3882 break;
3883 case PACKET3_COPY_DW:
3884 if (idx_value & 0x2) {
3885 reg = ib[idx + 3] * 4;
3886 if (!si_vm_reg_valid(reg))
3887 return -EINVAL;
3888 }
3889 break;
3890 case PACKET3_SET_CONFIG_REG:
3891 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3892 end_reg = 4 * pkt->count + start_reg - 4;
3893 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3894 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3895 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3896 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3897 return -EINVAL;
3898 }
3899 for (i = 0; i < pkt->count; i++) {
3900 reg = start_reg + (4 * i);
3901 if (!si_vm_reg_valid(reg))
3902 return -EINVAL;
3903 }
3904 break;
Alex Deucher5aa709b2012-12-03 19:42:37 -05003905 case PACKET3_CP_DMA:
3906 command = ib[idx + 4];
3907 info = ib[idx + 1];
3908 if (command & PACKET3_CP_DMA_CMD_SAS) {
3909 /* src address space is register */
3910 if (((info & 0x60000000) >> 29) == 0) {
3911 start_reg = idx_value << 2;
3912 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3913 reg = start_reg;
3914 if (!si_vm_reg_valid(reg)) {
3915 DRM_ERROR("CP DMA Bad SRC register\n");
3916 return -EINVAL;
3917 }
3918 } else {
3919 for (i = 0; i < (command & 0x1fffff); i++) {
3920 reg = start_reg + (4 * i);
3921 if (!si_vm_reg_valid(reg)) {
3922 DRM_ERROR("CP DMA Bad SRC register\n");
3923 return -EINVAL;
3924 }
3925 }
3926 }
3927 }
3928 }
3929 if (command & PACKET3_CP_DMA_CMD_DAS) {
3930 /* dst address space is register */
3931 if (((info & 0x00300000) >> 20) == 0) {
3932 start_reg = ib[idx + 2];
3933 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3934 reg = start_reg;
3935 if (!si_vm_reg_valid(reg)) {
3936 DRM_ERROR("CP DMA Bad DST register\n");
3937 return -EINVAL;
3938 }
3939 } else {
3940 for (i = 0; i < (command & 0x1fffff); i++) {
3941 reg = start_reg + (4 * i);
3942 if (!si_vm_reg_valid(reg)) {
3943 DRM_ERROR("CP DMA Bad DST register\n");
3944 return -EINVAL;
3945 }
3946 }
3947 }
3948 }
3949 }
3950 break;
Alex Deucher498dd8b2012-03-20 17:18:15 -04003951 default:
3952 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3953 return -EINVAL;
3954 }
3955 return 0;
3956}
3957
3958static int si_vm_packet3_compute_check(struct radeon_device *rdev,
3959 u32 *ib, struct radeon_cs_packet *pkt)
3960{
3961 u32 idx = pkt->idx + 1;
3962 u32 idx_value = ib[idx];
3963 u32 start_reg, reg, i;
3964
3965 switch (pkt->opcode) {
3966 case PACKET3_NOP:
3967 case PACKET3_SET_BASE:
3968 case PACKET3_CLEAR_STATE:
3969 case PACKET3_DISPATCH_DIRECT:
3970 case PACKET3_DISPATCH_INDIRECT:
3971 case PACKET3_ALLOC_GDS:
3972 case PACKET3_WRITE_GDS_RAM:
3973 case PACKET3_ATOMIC_GDS:
3974 case PACKET3_ATOMIC:
3975 case PACKET3_OCCLUSION_QUERY:
3976 case PACKET3_SET_PREDICATION:
3977 case PACKET3_COND_EXEC:
3978 case PACKET3_PRED_EXEC:
3979 case PACKET3_CONTEXT_CONTROL:
3980 case PACKET3_STRMOUT_BUFFER_UPDATE:
3981 case PACKET3_WAIT_REG_MEM:
3982 case PACKET3_MEM_WRITE:
3983 case PACKET3_PFP_SYNC_ME:
3984 case PACKET3_SURFACE_SYNC:
3985 case PACKET3_EVENT_WRITE:
3986 case PACKET3_EVENT_WRITE_EOP:
3987 case PACKET3_EVENT_WRITE_EOS:
3988 case PACKET3_SET_CONTEXT_REG:
3989 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3990 case PACKET3_SET_SH_REG:
3991 case PACKET3_SET_SH_REG_OFFSET:
3992 case PACKET3_INCREMENT_DE_COUNTER:
3993 case PACKET3_WAIT_ON_CE_COUNTER:
3994 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3995 case PACKET3_ME_WRITE:
3996 break;
3997 case PACKET3_COPY_DATA:
3998 if ((idx_value & 0xf00) == 0) {
3999 reg = ib[idx + 3] * 4;
4000 if (!si_vm_reg_valid(reg))
4001 return -EINVAL;
4002 }
4003 break;
4004 case PACKET3_WRITE_DATA:
4005 if ((idx_value & 0xf00) == 0) {
4006 start_reg = ib[idx + 1] * 4;
4007 if (idx_value & 0x10000) {
4008 if (!si_vm_reg_valid(start_reg))
4009 return -EINVAL;
4010 } else {
4011 for (i = 0; i < (pkt->count - 2); i++) {
4012 reg = start_reg + (4 * i);
4013 if (!si_vm_reg_valid(reg))
4014 return -EINVAL;
4015 }
4016 }
4017 }
4018 break;
4019 case PACKET3_COND_WRITE:
4020 if (idx_value & 0x100) {
4021 reg = ib[idx + 5] * 4;
4022 if (!si_vm_reg_valid(reg))
4023 return -EINVAL;
4024 }
4025 break;
4026 case PACKET3_COPY_DW:
4027 if (idx_value & 0x2) {
4028 reg = ib[idx + 3] * 4;
4029 if (!si_vm_reg_valid(reg))
4030 return -EINVAL;
4031 }
4032 break;
4033 default:
4034 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4035 return -EINVAL;
4036 }
4037 return 0;
4038}
4039
4040int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4041{
4042 int ret = 0;
4043 u32 idx = 0;
4044 struct radeon_cs_packet pkt;
4045
4046 do {
4047 pkt.idx = idx;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004048 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4049 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004050 pkt.one_reg_wr = 0;
4051 switch (pkt.type) {
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004052 case RADEON_PACKET_TYPE0:
Alex Deucher498dd8b2012-03-20 17:18:15 -04004053 dev_err(rdev->dev, "Packet0 not allowed!\n");
4054 ret = -EINVAL;
4055 break;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004056 case RADEON_PACKET_TYPE2:
Alex Deucher498dd8b2012-03-20 17:18:15 -04004057 idx += 1;
4058 break;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004059 case RADEON_PACKET_TYPE3:
4060 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004061 if (ib->is_const_ib)
4062 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4063 else {
Christian König876dc9f2012-05-08 14:24:01 +02004064 switch (ib->ring) {
Alex Deucher498dd8b2012-03-20 17:18:15 -04004065 case RADEON_RING_TYPE_GFX_INDEX:
4066 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4067 break;
4068 case CAYMAN_RING_TYPE_CP1_INDEX:
4069 case CAYMAN_RING_TYPE_CP2_INDEX:
4070 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4071 break;
4072 default:
Christian König876dc9f2012-05-08 14:24:01 +02004073 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004074 ret = -EINVAL;
4075 break;
4076 }
4077 }
4078 idx += pkt.count + 2;
4079 break;
4080 default:
4081 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4082 ret = -EINVAL;
4083 break;
4084 }
4085 if (ret)
4086 break;
4087 } while (idx < ib->length_dw);
4088
4089 return ret;
4090}
4091
Alex Deucherd2800ee2012-03-20 17:18:13 -04004092/*
4093 * vm
4094 */
4095int si_vm_init(struct radeon_device *rdev)
4096{
4097 /* number of VMs */
4098 rdev->vm_manager.nvm = 16;
4099 /* base offset of vram pages */
4100 rdev->vm_manager.vram_base_offset = 0;
4101
4102 return 0;
4103}
4104
4105void si_vm_fini(struct radeon_device *rdev)
4106{
4107}
4108
Alex Deucher82ffd922012-10-02 14:47:46 -04004109/**
4110 * si_vm_set_page - update the page tables using the CP
4111 *
4112 * @rdev: radeon_device pointer
Alex Deucher43f12142013-02-01 17:32:42 +01004113 * @ib: indirect buffer to fill with commands
Alex Deucher82ffd922012-10-02 14:47:46 -04004114 * @pe: addr of the page entry
4115 * @addr: dst addr to write into pe
4116 * @count: number of page entries to update
4117 * @incr: increase next addr by incr bytes
4118 * @flags: access flags
4119 *
Alex Deucher43f12142013-02-01 17:32:42 +01004120 * Update the page tables using the CP (SI).
Alex Deucher82ffd922012-10-02 14:47:46 -04004121 */
Alex Deucher43f12142013-02-01 17:32:42 +01004122void si_vm_set_page(struct radeon_device *rdev,
4123 struct radeon_ib *ib,
4124 uint64_t pe,
Alex Deucher82ffd922012-10-02 14:47:46 -04004125 uint64_t addr, unsigned count,
4126 uint32_t incr, uint32_t flags)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004127{
Alex Deucher82ffd922012-10-02 14:47:46 -04004128 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
Alex Deucherdeab48f2012-10-22 12:32:54 -04004129 uint64_t value;
4130 unsigned ndw;
Alex Deucher82ffd922012-10-02 14:47:46 -04004131
Alex Deucherdeab48f2012-10-22 12:32:54 -04004132 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4133 while (count) {
4134 ndw = 2 + count * 2;
4135 if (ndw > 0x3FFE)
4136 ndw = 0x3FFE;
Christian Königd7025d82012-10-22 17:42:37 +02004137
Alex Deucher43f12142013-02-01 17:32:42 +01004138 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4139 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4140 WRITE_DATA_DST_SEL(1));
4141 ib->ptr[ib->length_dw++] = pe;
4142 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
Alex Deucherdeab48f2012-10-22 12:32:54 -04004143 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4144 if (flags & RADEON_VM_PAGE_SYSTEM) {
4145 value = radeon_vm_map_gart(rdev, addr);
4146 value &= 0xFFFFFFFFFFFFF000ULL;
4147 } else if (flags & RADEON_VM_PAGE_VALID) {
4148 value = addr;
4149 } else {
4150 value = 0;
4151 }
4152 addr += incr;
4153 value |= r600_flags;
Alex Deucher43f12142013-02-01 17:32:42 +01004154 ib->ptr[ib->length_dw++] = value;
4155 ib->ptr[ib->length_dw++] = upper_32_bits(value);
Alex Deucherdeab48f2012-10-22 12:32:54 -04004156 }
4157 }
4158 } else {
4159 /* DMA */
4160 if (flags & RADEON_VM_PAGE_SYSTEM) {
4161 while (count) {
4162 ndw = count * 2;
4163 if (ndw > 0xFFFFE)
4164 ndw = 0xFFFFE;
4165
4166 /* for non-physically contiguous pages (system) */
Alex Deucher43f12142013-02-01 17:32:42 +01004167 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4168 ib->ptr[ib->length_dw++] = pe;
4169 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
Alex Deucherdeab48f2012-10-22 12:32:54 -04004170 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4171 if (flags & RADEON_VM_PAGE_SYSTEM) {
4172 value = radeon_vm_map_gart(rdev, addr);
4173 value &= 0xFFFFFFFFFFFFF000ULL;
4174 } else if (flags & RADEON_VM_PAGE_VALID) {
4175 value = addr;
4176 } else {
4177 value = 0;
4178 }
4179 addr += incr;
4180 value |= r600_flags;
Alex Deucher43f12142013-02-01 17:32:42 +01004181 ib->ptr[ib->length_dw++] = value;
4182 ib->ptr[ib->length_dw++] = upper_32_bits(value);
Alex Deucherdeab48f2012-10-22 12:32:54 -04004183 }
4184 }
4185 } else {
4186 while (count) {
4187 ndw = count * 2;
4188 if (ndw > 0xFFFFE)
4189 ndw = 0xFFFFE;
4190
4191 if (flags & RADEON_VM_PAGE_VALID)
4192 value = addr;
4193 else
4194 value = 0;
4195 /* for physically contiguous pages (vram) */
Alex Deucher43f12142013-02-01 17:32:42 +01004196 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4197 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4198 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4199 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4200 ib->ptr[ib->length_dw++] = 0;
4201 ib->ptr[ib->length_dw++] = value; /* value */
4202 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4203 ib->ptr[ib->length_dw++] = incr; /* increment size */
4204 ib->ptr[ib->length_dw++] = 0;
Alex Deucherdeab48f2012-10-22 12:32:54 -04004205 pe += ndw * 4;
4206 addr += (ndw / 2) * incr;
4207 count -= ndw / 2;
4208 }
Christian Königd7025d82012-10-22 17:42:37 +02004209 }
Alex Deucher43f12142013-02-01 17:32:42 +01004210 while (ib->length_dw & 0x7)
4211 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
Alex Deucher82ffd922012-10-02 14:47:46 -04004212 }
Alex Deucherd2800ee2012-03-20 17:18:13 -04004213}
4214
Alex Deucher498522b2012-10-02 14:43:38 -04004215void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004216{
Alex Deucher498522b2012-10-02 14:43:38 -04004217 struct radeon_ring *ring = &rdev->ring[ridx];
Alex Deucherd2800ee2012-03-20 17:18:13 -04004218
Christian Königee60e292012-08-09 16:21:08 +02004219 if (vm == NULL)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004220 return;
4221
Alex Deucher76c44f22012-10-02 14:39:18 -04004222 /* write new base address */
4223 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4224 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4225 WRITE_DATA_DST_SEL(0)));
4226
Christian Königee60e292012-08-09 16:21:08 +02004227 if (vm->id < 8) {
Alex Deucher76c44f22012-10-02 14:39:18 -04004228 radeon_ring_write(ring,
4229 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
Christian Königee60e292012-08-09 16:21:08 +02004230 } else {
Alex Deucher76c44f22012-10-02 14:39:18 -04004231 radeon_ring_write(ring,
4232 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
Christian Königee60e292012-08-09 16:21:08 +02004233 }
Alex Deucher76c44f22012-10-02 14:39:18 -04004234 radeon_ring_write(ring, 0);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02004235 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
Christian Königee60e292012-08-09 16:21:08 +02004236
Alex Deucherd2800ee2012-03-20 17:18:13 -04004237 /* flush hdp cache */
Alex Deucher76c44f22012-10-02 14:39:18 -04004238 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4239 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4240 WRITE_DATA_DST_SEL(0)));
4241 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4242 radeon_ring_write(ring, 0);
Christian Königee60e292012-08-09 16:21:08 +02004243 radeon_ring_write(ring, 0x1);
4244
Alex Deucherd2800ee2012-03-20 17:18:13 -04004245 /* bits 0-15 are the VM contexts0-15 */
Alex Deucher76c44f22012-10-02 14:39:18 -04004246 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4247 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4248 WRITE_DATA_DST_SEL(0)));
4249 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4250 radeon_ring_write(ring, 0);
Alex Deucher498522b2012-10-02 14:43:38 -04004251 radeon_ring_write(ring, 1 << vm->id);
Christian König58f8cf52012-10-22 17:42:35 +02004252
4253 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4254 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4255 radeon_ring_write(ring, 0x0);
Alex Deucherd2800ee2012-03-20 17:18:13 -04004256}
4257
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004258void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4259{
4260 struct radeon_ring *ring = &rdev->ring[ridx];
4261
4262 if (vm == NULL)
4263 return;
4264
4265 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4266 if (vm->id < 8) {
4267 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4268 } else {
4269 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4270 }
4271 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4272
4273 /* flush hdp cache */
4274 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4275 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4276 radeon_ring_write(ring, 1);
4277
4278 /* bits 0-7 are the VM contexts0-7 */
4279 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4280 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4281 radeon_ring_write(ring, 1 << vm->id);
4282}
4283
Alex Deucher347e7592012-03-20 17:18:21 -04004284/*
4285 * RLC
4286 */
Alex Deucherc420c742012-03-20 17:18:39 -04004287void si_rlc_fini(struct radeon_device *rdev)
Alex Deucher347e7592012-03-20 17:18:21 -04004288{
4289 int r;
4290
4291 /* save restore block */
4292 if (rdev->rlc.save_restore_obj) {
4293 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4294 if (unlikely(r != 0))
4295 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4296 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4297 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4298
4299 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4300 rdev->rlc.save_restore_obj = NULL;
4301 }
4302
4303 /* clear state block */
4304 if (rdev->rlc.clear_state_obj) {
4305 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4306 if (unlikely(r != 0))
4307 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4308 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4309 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4310
4311 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4312 rdev->rlc.clear_state_obj = NULL;
4313 }
4314}
4315
Alex Deucherc420c742012-03-20 17:18:39 -04004316int si_rlc_init(struct radeon_device *rdev)
Alex Deucher347e7592012-03-20 17:18:21 -04004317{
4318 int r;
4319
4320 /* save restore block */
4321 if (rdev->rlc.save_restore_obj == NULL) {
4322 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
Alex Deucher40f5cf92012-05-10 18:33:13 -04004323 RADEON_GEM_DOMAIN_VRAM, NULL,
4324 &rdev->rlc.save_restore_obj);
Alex Deucher347e7592012-03-20 17:18:21 -04004325 if (r) {
4326 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4327 return r;
4328 }
4329 }
4330
4331 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4332 if (unlikely(r != 0)) {
4333 si_rlc_fini(rdev);
4334 return r;
4335 }
4336 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4337 &rdev->rlc.save_restore_gpu_addr);
Alex Deucher5273db72012-04-13 10:26:36 -04004338 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
Alex Deucher347e7592012-03-20 17:18:21 -04004339 if (r) {
Alex Deucher347e7592012-03-20 17:18:21 -04004340 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4341 si_rlc_fini(rdev);
4342 return r;
4343 }
4344
4345 /* clear state block */
4346 if (rdev->rlc.clear_state_obj == NULL) {
4347 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
Alex Deucher40f5cf92012-05-10 18:33:13 -04004348 RADEON_GEM_DOMAIN_VRAM, NULL,
4349 &rdev->rlc.clear_state_obj);
Alex Deucher347e7592012-03-20 17:18:21 -04004350 if (r) {
4351 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4352 si_rlc_fini(rdev);
4353 return r;
4354 }
4355 }
4356 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4357 if (unlikely(r != 0)) {
4358 si_rlc_fini(rdev);
4359 return r;
4360 }
4361 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4362 &rdev->rlc.clear_state_gpu_addr);
Alex Deucher5273db72012-04-13 10:26:36 -04004363 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
Alex Deucher347e7592012-03-20 17:18:21 -04004364 if (r) {
Alex Deucher347e7592012-03-20 17:18:21 -04004365 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4366 si_rlc_fini(rdev);
4367 return r;
4368 }
4369
4370 return 0;
4371}
4372
4373static void si_rlc_stop(struct radeon_device *rdev)
4374{
4375 WREG32(RLC_CNTL, 0);
4376}
4377
4378static void si_rlc_start(struct radeon_device *rdev)
4379{
4380 WREG32(RLC_CNTL, RLC_ENABLE);
4381}
4382
4383static int si_rlc_resume(struct radeon_device *rdev)
4384{
4385 u32 i;
4386 const __be32 *fw_data;
4387
4388 if (!rdev->rlc_fw)
4389 return -EINVAL;
4390
4391 si_rlc_stop(rdev);
4392
4393 WREG32(RLC_RL_BASE, 0);
4394 WREG32(RLC_RL_SIZE, 0);
4395 WREG32(RLC_LB_CNTL, 0);
4396 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4397 WREG32(RLC_LB_CNTR_INIT, 0);
4398
4399 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4400 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4401
4402 WREG32(RLC_MC_CNTL, 0);
4403 WREG32(RLC_UCODE_CNTL, 0);
4404
4405 fw_data = (const __be32 *)rdev->rlc_fw->data;
4406 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4407 WREG32(RLC_UCODE_ADDR, i);
4408 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4409 }
4410 WREG32(RLC_UCODE_ADDR, 0);
4411
4412 si_rlc_start(rdev);
4413
4414 return 0;
4415}
4416
Alex Deucher25a857f2012-03-20 17:18:22 -04004417static void si_enable_interrupts(struct radeon_device *rdev)
4418{
4419 u32 ih_cntl = RREG32(IH_CNTL);
4420 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4421
4422 ih_cntl |= ENABLE_INTR;
4423 ih_rb_cntl |= IH_RB_ENABLE;
4424 WREG32(IH_CNTL, ih_cntl);
4425 WREG32(IH_RB_CNTL, ih_rb_cntl);
4426 rdev->ih.enabled = true;
4427}
4428
4429static void si_disable_interrupts(struct radeon_device *rdev)
4430{
4431 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4432 u32 ih_cntl = RREG32(IH_CNTL);
4433
4434 ih_rb_cntl &= ~IH_RB_ENABLE;
4435 ih_cntl &= ~ENABLE_INTR;
4436 WREG32(IH_RB_CNTL, ih_rb_cntl);
4437 WREG32(IH_CNTL, ih_cntl);
4438 /* set rptr, wptr to 0 */
4439 WREG32(IH_RB_RPTR, 0);
4440 WREG32(IH_RB_WPTR, 0);
4441 rdev->ih.enabled = false;
Alex Deucher25a857f2012-03-20 17:18:22 -04004442 rdev->ih.rptr = 0;
4443}
4444
4445static void si_disable_interrupt_state(struct radeon_device *rdev)
4446{
4447 u32 tmp;
4448
4449 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4450 WREG32(CP_INT_CNTL_RING1, 0);
4451 WREG32(CP_INT_CNTL_RING2, 0);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004452 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4453 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4454 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4455 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
Alex Deucher25a857f2012-03-20 17:18:22 -04004456 WREG32(GRBM_INT_CNTL, 0);
Alex Deucher51535502012-08-30 14:34:30 -04004457 if (rdev->num_crtc >= 2) {
4458 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4459 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4460 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004461 if (rdev->num_crtc >= 4) {
4462 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4463 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4464 }
4465 if (rdev->num_crtc >= 6) {
4466 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4467 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4468 }
4469
Alex Deucher51535502012-08-30 14:34:30 -04004470 if (rdev->num_crtc >= 2) {
4471 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4472 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4473 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004474 if (rdev->num_crtc >= 4) {
4475 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4476 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4477 }
4478 if (rdev->num_crtc >= 6) {
4479 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4480 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4481 }
4482
Alex Deucher51535502012-08-30 14:34:30 -04004483 if (!ASIC_IS_NODCE(rdev)) {
4484 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
Alex Deucher25a857f2012-03-20 17:18:22 -04004485
Alex Deucher51535502012-08-30 14:34:30 -04004486 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4487 WREG32(DC_HPD1_INT_CONTROL, tmp);
4488 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4489 WREG32(DC_HPD2_INT_CONTROL, tmp);
4490 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4491 WREG32(DC_HPD3_INT_CONTROL, tmp);
4492 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4493 WREG32(DC_HPD4_INT_CONTROL, tmp);
4494 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4495 WREG32(DC_HPD5_INT_CONTROL, tmp);
4496 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4497 WREG32(DC_HPD6_INT_CONTROL, tmp);
4498 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004499}
4500
4501static int si_irq_init(struct radeon_device *rdev)
4502{
4503 int ret = 0;
4504 int rb_bufsz;
4505 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4506
4507 /* allocate ring */
4508 ret = r600_ih_ring_alloc(rdev);
4509 if (ret)
4510 return ret;
4511
4512 /* disable irqs */
4513 si_disable_interrupts(rdev);
4514
4515 /* init rlc */
4516 ret = si_rlc_resume(rdev);
4517 if (ret) {
4518 r600_ih_ring_fini(rdev);
4519 return ret;
4520 }
4521
4522 /* setup interrupt control */
4523 /* set dummy read address to ring address */
4524 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4525 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4526 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4527 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4528 */
4529 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4530 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4531 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4532 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4533
4534 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4535 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4536
4537 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4538 IH_WPTR_OVERFLOW_CLEAR |
4539 (rb_bufsz << 1));
4540
4541 if (rdev->wb.enabled)
4542 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4543
4544 /* set the writeback address whether it's enabled or not */
4545 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4546 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4547
4548 WREG32(IH_RB_CNTL, ih_rb_cntl);
4549
4550 /* set rptr, wptr to 0 */
4551 WREG32(IH_RB_RPTR, 0);
4552 WREG32(IH_RB_WPTR, 0);
4553
4554 /* Default settings for IH_CNTL (disabled at first) */
4555 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4556 /* RPTR_REARM only works if msi's are enabled */
4557 if (rdev->msi_enabled)
4558 ih_cntl |= RPTR_REARM;
4559 WREG32(IH_CNTL, ih_cntl);
4560
4561 /* force the active interrupt state to all disabled */
4562 si_disable_interrupt_state(rdev);
4563
Dave Airlie20998102012-04-03 11:53:05 +01004564 pci_set_master(rdev->pdev);
4565
Alex Deucher25a857f2012-03-20 17:18:22 -04004566 /* enable irqs */
4567 si_enable_interrupts(rdev);
4568
4569 return ret;
4570}
4571
4572int si_irq_set(struct radeon_device *rdev)
4573{
4574 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4575 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4576 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
Alex Deucher51535502012-08-30 14:34:30 -04004577 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
Alex Deucher25a857f2012-03-20 17:18:22 -04004578 u32 grbm_int_cntl = 0;
4579 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004580 u32 dma_cntl, dma_cntl1;
Alex Deucher25a857f2012-03-20 17:18:22 -04004581
4582 if (!rdev->irq.installed) {
4583 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4584 return -EINVAL;
4585 }
4586 /* don't enable anything if the ih is disabled */
4587 if (!rdev->ih.enabled) {
4588 si_disable_interrupts(rdev);
4589 /* force the active interrupt state to all disabled */
4590 si_disable_interrupt_state(rdev);
4591 return 0;
4592 }
4593
Alex Deucher51535502012-08-30 14:34:30 -04004594 if (!ASIC_IS_NODCE(rdev)) {
4595 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4596 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4597 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4598 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4599 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4600 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4601 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004602
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004603 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4604 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4605
Alex Deucher25a857f2012-03-20 17:18:22 -04004606 /* enable CP interrupts on all rings */
Christian Koenig736fc372012-05-17 19:52:00 +02004607 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004608 DRM_DEBUG("si_irq_set: sw int gfx\n");
4609 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4610 }
Christian Koenig736fc372012-05-17 19:52:00 +02004611 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004612 DRM_DEBUG("si_irq_set: sw int cp1\n");
4613 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4614 }
Christian Koenig736fc372012-05-17 19:52:00 +02004615 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004616 DRM_DEBUG("si_irq_set: sw int cp2\n");
4617 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4618 }
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004619 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4620 DRM_DEBUG("si_irq_set: sw int dma\n");
4621 dma_cntl |= TRAP_ENABLE;
4622 }
4623
4624 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4625 DRM_DEBUG("si_irq_set: sw int dma1\n");
4626 dma_cntl1 |= TRAP_ENABLE;
4627 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004628 if (rdev->irq.crtc_vblank_int[0] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004629 atomic_read(&rdev->irq.pflip[0])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004630 DRM_DEBUG("si_irq_set: vblank 0\n");
4631 crtc1 |= VBLANK_INT_MASK;
4632 }
4633 if (rdev->irq.crtc_vblank_int[1] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004634 atomic_read(&rdev->irq.pflip[1])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004635 DRM_DEBUG("si_irq_set: vblank 1\n");
4636 crtc2 |= VBLANK_INT_MASK;
4637 }
4638 if (rdev->irq.crtc_vblank_int[2] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004639 atomic_read(&rdev->irq.pflip[2])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004640 DRM_DEBUG("si_irq_set: vblank 2\n");
4641 crtc3 |= VBLANK_INT_MASK;
4642 }
4643 if (rdev->irq.crtc_vblank_int[3] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004644 atomic_read(&rdev->irq.pflip[3])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004645 DRM_DEBUG("si_irq_set: vblank 3\n");
4646 crtc4 |= VBLANK_INT_MASK;
4647 }
4648 if (rdev->irq.crtc_vblank_int[4] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004649 atomic_read(&rdev->irq.pflip[4])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004650 DRM_DEBUG("si_irq_set: vblank 4\n");
4651 crtc5 |= VBLANK_INT_MASK;
4652 }
4653 if (rdev->irq.crtc_vblank_int[5] ||
Christian Koenig736fc372012-05-17 19:52:00 +02004654 atomic_read(&rdev->irq.pflip[5])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04004655 DRM_DEBUG("si_irq_set: vblank 5\n");
4656 crtc6 |= VBLANK_INT_MASK;
4657 }
4658 if (rdev->irq.hpd[0]) {
4659 DRM_DEBUG("si_irq_set: hpd 1\n");
4660 hpd1 |= DC_HPDx_INT_EN;
4661 }
4662 if (rdev->irq.hpd[1]) {
4663 DRM_DEBUG("si_irq_set: hpd 2\n");
4664 hpd2 |= DC_HPDx_INT_EN;
4665 }
4666 if (rdev->irq.hpd[2]) {
4667 DRM_DEBUG("si_irq_set: hpd 3\n");
4668 hpd3 |= DC_HPDx_INT_EN;
4669 }
4670 if (rdev->irq.hpd[3]) {
4671 DRM_DEBUG("si_irq_set: hpd 4\n");
4672 hpd4 |= DC_HPDx_INT_EN;
4673 }
4674 if (rdev->irq.hpd[4]) {
4675 DRM_DEBUG("si_irq_set: hpd 5\n");
4676 hpd5 |= DC_HPDx_INT_EN;
4677 }
4678 if (rdev->irq.hpd[5]) {
4679 DRM_DEBUG("si_irq_set: hpd 6\n");
4680 hpd6 |= DC_HPDx_INT_EN;
4681 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004682
4683 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4684 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4685 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4686
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05004687 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4688 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4689
Alex Deucher25a857f2012-03-20 17:18:22 -04004690 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4691
Alex Deucher51535502012-08-30 14:34:30 -04004692 if (rdev->num_crtc >= 2) {
4693 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4694 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4695 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004696 if (rdev->num_crtc >= 4) {
4697 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4698 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4699 }
4700 if (rdev->num_crtc >= 6) {
4701 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4702 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4703 }
4704
Alex Deucher51535502012-08-30 14:34:30 -04004705 if (rdev->num_crtc >= 2) {
4706 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4707 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4708 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004709 if (rdev->num_crtc >= 4) {
4710 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4711 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4712 }
4713 if (rdev->num_crtc >= 6) {
4714 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4715 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4716 }
4717
Alex Deucher51535502012-08-30 14:34:30 -04004718 if (!ASIC_IS_NODCE(rdev)) {
4719 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4720 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4721 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4722 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4723 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4724 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4725 }
Alex Deucher25a857f2012-03-20 17:18:22 -04004726
4727 return 0;
4728}
4729
4730static inline void si_irq_ack(struct radeon_device *rdev)
4731{
4732 u32 tmp;
4733
Alex Deucher51535502012-08-30 14:34:30 -04004734 if (ASIC_IS_NODCE(rdev))
4735 return;
4736
Alex Deucher25a857f2012-03-20 17:18:22 -04004737 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4738 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4739 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4740 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4741 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4742 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4743 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4744 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4745 if (rdev->num_crtc >= 4) {
4746 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4747 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4748 }
4749 if (rdev->num_crtc >= 6) {
4750 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4751 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4752 }
4753
4754 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4755 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4756 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4757 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4758 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4759 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4760 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4761 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4762 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4763 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4764 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4765 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4766
4767 if (rdev->num_crtc >= 4) {
4768 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4769 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4770 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4771 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4772 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4773 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4774 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4775 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4776 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4777 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4778 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4779 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4780 }
4781
4782 if (rdev->num_crtc >= 6) {
4783 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4784 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4785 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4786 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4787 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4788 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4789 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4790 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4791 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4792 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4793 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4794 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4795 }
4796
4797 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4798 tmp = RREG32(DC_HPD1_INT_CONTROL);
4799 tmp |= DC_HPDx_INT_ACK;
4800 WREG32(DC_HPD1_INT_CONTROL, tmp);
4801 }
4802 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4803 tmp = RREG32(DC_HPD2_INT_CONTROL);
4804 tmp |= DC_HPDx_INT_ACK;
4805 WREG32(DC_HPD2_INT_CONTROL, tmp);
4806 }
4807 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4808 tmp = RREG32(DC_HPD3_INT_CONTROL);
4809 tmp |= DC_HPDx_INT_ACK;
4810 WREG32(DC_HPD3_INT_CONTROL, tmp);
4811 }
4812 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4813 tmp = RREG32(DC_HPD4_INT_CONTROL);
4814 tmp |= DC_HPDx_INT_ACK;
4815 WREG32(DC_HPD4_INT_CONTROL, tmp);
4816 }
4817 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4818 tmp = RREG32(DC_HPD5_INT_CONTROL);
4819 tmp |= DC_HPDx_INT_ACK;
4820 WREG32(DC_HPD5_INT_CONTROL, tmp);
4821 }
4822 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4823 tmp = RREG32(DC_HPD5_INT_CONTROL);
4824 tmp |= DC_HPDx_INT_ACK;
4825 WREG32(DC_HPD6_INT_CONTROL, tmp);
4826 }
4827}
4828
4829static void si_irq_disable(struct radeon_device *rdev)
4830{
4831 si_disable_interrupts(rdev);
4832 /* Wait and acknowledge irq */
4833 mdelay(1);
4834 si_irq_ack(rdev);
4835 si_disable_interrupt_state(rdev);
4836}
4837
4838static void si_irq_suspend(struct radeon_device *rdev)
4839{
4840 si_irq_disable(rdev);
4841 si_rlc_stop(rdev);
4842}
4843
Alex Deucher9b136d52012-03-20 17:18:23 -04004844static void si_irq_fini(struct radeon_device *rdev)
4845{
4846 si_irq_suspend(rdev);
4847 r600_ih_ring_fini(rdev);
4848}
4849
Alex Deucher25a857f2012-03-20 17:18:22 -04004850static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4851{
4852 u32 wptr, tmp;
4853
4854 if (rdev->wb.enabled)
4855 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4856 else
4857 wptr = RREG32(IH_RB_WPTR);
4858
4859 if (wptr & RB_OVERFLOW) {
4860 /* When a ring buffer overflow happen start parsing interrupt
4861 * from the last not overwritten vector (wptr + 16). Hopefully
4862 * this should allow us to catchup.
4863 */
4864 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4865 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4866 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4867 tmp = RREG32(IH_RB_CNTL);
4868 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4869 WREG32(IH_RB_CNTL, tmp);
4870 }
4871 return (wptr & rdev->ih.ptr_mask);
4872}
4873
4874/* SI IV Ring
4875 * Each IV ring entry is 128 bits:
4876 * [7:0] - interrupt source id
4877 * [31:8] - reserved
4878 * [59:32] - interrupt source data
4879 * [63:60] - reserved
4880 * [71:64] - RINGID
4881 * [79:72] - VMID
4882 * [127:80] - reserved
4883 */
4884int si_irq_process(struct radeon_device *rdev)
4885{
4886 u32 wptr;
4887 u32 rptr;
4888 u32 src_id, src_data, ring_id;
4889 u32 ring_index;
Alex Deucher25a857f2012-03-20 17:18:22 -04004890 bool queue_hotplug = false;
4891
4892 if (!rdev->ih.enabled || rdev->shutdown)
4893 return IRQ_NONE;
4894
4895 wptr = si_get_ih_wptr(rdev);
Christian Koenigc20dc362012-05-16 21:45:24 +02004896
4897restart_ih:
4898 /* is somebody else already processing irqs? */
4899 if (atomic_xchg(&rdev->ih.lock, 1))
4900 return IRQ_NONE;
4901
Alex Deucher25a857f2012-03-20 17:18:22 -04004902 rptr = rdev->ih.rptr;
4903 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4904
Alex Deucher25a857f2012-03-20 17:18:22 -04004905 /* Order reading of wptr vs. reading of IH ring data */
4906 rmb();
4907
4908 /* display interrupts */
4909 si_irq_ack(rdev);
4910
Alex Deucher25a857f2012-03-20 17:18:22 -04004911 while (rptr != wptr) {
4912 /* wptr/rptr are in bytes! */
4913 ring_index = rptr / 4;
4914 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4915 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4916 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4917
4918 switch (src_id) {
4919 case 1: /* D1 vblank/vline */
4920 switch (src_data) {
4921 case 0: /* D1 vblank */
4922 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4923 if (rdev->irq.crtc_vblank_int[0]) {
4924 drm_handle_vblank(rdev->ddev, 0);
4925 rdev->pm.vblank_sync = true;
4926 wake_up(&rdev->irq.vblank_queue);
4927 }
Christian Koenig736fc372012-05-17 19:52:00 +02004928 if (atomic_read(&rdev->irq.pflip[0]))
Alex Deucher25a857f2012-03-20 17:18:22 -04004929 radeon_crtc_handle_flip(rdev, 0);
4930 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4931 DRM_DEBUG("IH: D1 vblank\n");
4932 }
4933 break;
4934 case 1: /* D1 vline */
4935 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4936 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4937 DRM_DEBUG("IH: D1 vline\n");
4938 }
4939 break;
4940 default:
4941 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4942 break;
4943 }
4944 break;
4945 case 2: /* D2 vblank/vline */
4946 switch (src_data) {
4947 case 0: /* D2 vblank */
4948 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4949 if (rdev->irq.crtc_vblank_int[1]) {
4950 drm_handle_vblank(rdev->ddev, 1);
4951 rdev->pm.vblank_sync = true;
4952 wake_up(&rdev->irq.vblank_queue);
4953 }
Christian Koenig736fc372012-05-17 19:52:00 +02004954 if (atomic_read(&rdev->irq.pflip[1]))
Alex Deucher25a857f2012-03-20 17:18:22 -04004955 radeon_crtc_handle_flip(rdev, 1);
4956 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4957 DRM_DEBUG("IH: D2 vblank\n");
4958 }
4959 break;
4960 case 1: /* D2 vline */
4961 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4962 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4963 DRM_DEBUG("IH: D2 vline\n");
4964 }
4965 break;
4966 default:
4967 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4968 break;
4969 }
4970 break;
4971 case 3: /* D3 vblank/vline */
4972 switch (src_data) {
4973 case 0: /* D3 vblank */
4974 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4975 if (rdev->irq.crtc_vblank_int[2]) {
4976 drm_handle_vblank(rdev->ddev, 2);
4977 rdev->pm.vblank_sync = true;
4978 wake_up(&rdev->irq.vblank_queue);
4979 }
Christian Koenig736fc372012-05-17 19:52:00 +02004980 if (atomic_read(&rdev->irq.pflip[2]))
Alex Deucher25a857f2012-03-20 17:18:22 -04004981 radeon_crtc_handle_flip(rdev, 2);
4982 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4983 DRM_DEBUG("IH: D3 vblank\n");
4984 }
4985 break;
4986 case 1: /* D3 vline */
4987 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4988 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4989 DRM_DEBUG("IH: D3 vline\n");
4990 }
4991 break;
4992 default:
4993 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4994 break;
4995 }
4996 break;
4997 case 4: /* D4 vblank/vline */
4998 switch (src_data) {
4999 case 0: /* D4 vblank */
5000 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5001 if (rdev->irq.crtc_vblank_int[3]) {
5002 drm_handle_vblank(rdev->ddev, 3);
5003 rdev->pm.vblank_sync = true;
5004 wake_up(&rdev->irq.vblank_queue);
5005 }
Christian Koenig736fc372012-05-17 19:52:00 +02005006 if (atomic_read(&rdev->irq.pflip[3]))
Alex Deucher25a857f2012-03-20 17:18:22 -04005007 radeon_crtc_handle_flip(rdev, 3);
5008 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5009 DRM_DEBUG("IH: D4 vblank\n");
5010 }
5011 break;
5012 case 1: /* D4 vline */
5013 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5014 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5015 DRM_DEBUG("IH: D4 vline\n");
5016 }
5017 break;
5018 default:
5019 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5020 break;
5021 }
5022 break;
5023 case 5: /* D5 vblank/vline */
5024 switch (src_data) {
5025 case 0: /* D5 vblank */
5026 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5027 if (rdev->irq.crtc_vblank_int[4]) {
5028 drm_handle_vblank(rdev->ddev, 4);
5029 rdev->pm.vblank_sync = true;
5030 wake_up(&rdev->irq.vblank_queue);
5031 }
Christian Koenig736fc372012-05-17 19:52:00 +02005032 if (atomic_read(&rdev->irq.pflip[4]))
Alex Deucher25a857f2012-03-20 17:18:22 -04005033 radeon_crtc_handle_flip(rdev, 4);
5034 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5035 DRM_DEBUG("IH: D5 vblank\n");
5036 }
5037 break;
5038 case 1: /* D5 vline */
5039 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5040 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5041 DRM_DEBUG("IH: D5 vline\n");
5042 }
5043 break;
5044 default:
5045 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5046 break;
5047 }
5048 break;
5049 case 6: /* D6 vblank/vline */
5050 switch (src_data) {
5051 case 0: /* D6 vblank */
5052 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5053 if (rdev->irq.crtc_vblank_int[5]) {
5054 drm_handle_vblank(rdev->ddev, 5);
5055 rdev->pm.vblank_sync = true;
5056 wake_up(&rdev->irq.vblank_queue);
5057 }
Christian Koenig736fc372012-05-17 19:52:00 +02005058 if (atomic_read(&rdev->irq.pflip[5]))
Alex Deucher25a857f2012-03-20 17:18:22 -04005059 radeon_crtc_handle_flip(rdev, 5);
5060 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5061 DRM_DEBUG("IH: D6 vblank\n");
5062 }
5063 break;
5064 case 1: /* D6 vline */
5065 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5066 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5067 DRM_DEBUG("IH: D6 vline\n");
5068 }
5069 break;
5070 default:
5071 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5072 break;
5073 }
5074 break;
5075 case 42: /* HPD hotplug */
5076 switch (src_data) {
5077 case 0:
5078 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5079 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5080 queue_hotplug = true;
5081 DRM_DEBUG("IH: HPD1\n");
5082 }
5083 break;
5084 case 1:
5085 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5086 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5087 queue_hotplug = true;
5088 DRM_DEBUG("IH: HPD2\n");
5089 }
5090 break;
5091 case 2:
5092 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5093 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5094 queue_hotplug = true;
5095 DRM_DEBUG("IH: HPD3\n");
5096 }
5097 break;
5098 case 3:
5099 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5100 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5101 queue_hotplug = true;
5102 DRM_DEBUG("IH: HPD4\n");
5103 }
5104 break;
5105 case 4:
5106 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5107 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5108 queue_hotplug = true;
5109 DRM_DEBUG("IH: HPD5\n");
5110 }
5111 break;
5112 case 5:
5113 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5114 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5115 queue_hotplug = true;
5116 DRM_DEBUG("IH: HPD6\n");
5117 }
5118 break;
5119 default:
5120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5121 break;
5122 }
5123 break;
Christian Königae133a12012-09-18 15:30:44 -04005124 case 146:
5125 case 147:
5126 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5127 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5128 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5129 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5130 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5131 /* reset addr and status */
5132 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5133 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04005134 case 176: /* RINGID0 CP_INT */
5135 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5136 break;
5137 case 177: /* RINGID1 CP_INT */
5138 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5139 break;
5140 case 178: /* RINGID2 CP_INT */
5141 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5142 break;
5143 case 181: /* CP EOP event */
5144 DRM_DEBUG("IH: CP EOP\n");
5145 switch (ring_id) {
5146 case 0:
5147 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5148 break;
5149 case 1:
5150 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5151 break;
5152 case 2:
5153 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5154 break;
5155 }
5156 break;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005157 case 224: /* DMA trap event */
5158 DRM_DEBUG("IH: DMA trap\n");
5159 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5160 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04005161 case 233: /* GUI IDLE */
5162 DRM_DEBUG("IH: GUI idle\n");
Alex Deucher25a857f2012-03-20 17:18:22 -04005163 break;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005164 case 244: /* DMA trap event */
5165 DRM_DEBUG("IH: DMA1 trap\n");
5166 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5167 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04005168 default:
5169 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5170 break;
5171 }
5172
5173 /* wptr/rptr are in bytes! */
5174 rptr += 16;
5175 rptr &= rdev->ih.ptr_mask;
5176 }
Alex Deucher25a857f2012-03-20 17:18:22 -04005177 if (queue_hotplug)
5178 schedule_work(&rdev->hotplug_work);
5179 rdev->ih.rptr = rptr;
5180 WREG32(IH_RB_RPTR, rdev->ih.rptr);
Christian Koenigc20dc362012-05-16 21:45:24 +02005181 atomic_set(&rdev->ih.lock, 0);
5182
5183 /* make sure wptr hasn't changed while processing */
5184 wptr = si_get_ih_wptr(rdev);
5185 if (wptr != rptr)
5186 goto restart_ih;
5187
Alex Deucher25a857f2012-03-20 17:18:22 -04005188 return IRQ_HANDLED;
5189}
5190
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005191/**
5192 * si_copy_dma - copy pages using the DMA engine
5193 *
5194 * @rdev: radeon_device pointer
5195 * @src_offset: src GPU address
5196 * @dst_offset: dst GPU address
5197 * @num_gpu_pages: number of GPU pages to xfer
5198 * @fence: radeon fence object
5199 *
5200 * Copy GPU paging using the DMA engine (SI).
5201 * Used by the radeon ttm implementation to move pages if
5202 * registered as the asic copy callback.
5203 */
5204int si_copy_dma(struct radeon_device *rdev,
5205 uint64_t src_offset, uint64_t dst_offset,
5206 unsigned num_gpu_pages,
5207 struct radeon_fence **fence)
5208{
5209 struct radeon_semaphore *sem = NULL;
5210 int ring_index = rdev->asic->copy.dma_ring_index;
5211 struct radeon_ring *ring = &rdev->ring[ring_index];
5212 u32 size_in_bytes, cur_size_in_bytes;
5213 int i, num_loops;
5214 int r = 0;
5215
5216 r = radeon_semaphore_create(rdev, &sem);
5217 if (r) {
5218 DRM_ERROR("radeon: moving bo (%d).\n", r);
5219 return r;
5220 }
5221
5222 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5223 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5224 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5225 if (r) {
5226 DRM_ERROR("radeon: moving bo (%d).\n", r);
5227 radeon_semaphore_free(rdev, &sem, NULL);
5228 return r;
5229 }
5230
5231 if (radeon_fence_need_sync(*fence, ring->idx)) {
5232 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5233 ring->idx);
5234 radeon_fence_note_sync(*fence, ring->idx);
5235 } else {
5236 radeon_semaphore_free(rdev, &sem, NULL);
5237 }
5238
5239 for (i = 0; i < num_loops; i++) {
5240 cur_size_in_bytes = size_in_bytes;
5241 if (cur_size_in_bytes > 0xFFFFF)
5242 cur_size_in_bytes = 0xFFFFF;
5243 size_in_bytes -= cur_size_in_bytes;
5244 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5245 radeon_ring_write(ring, dst_offset & 0xffffffff);
5246 radeon_ring_write(ring, src_offset & 0xffffffff);
5247 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5248 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5249 src_offset += cur_size_in_bytes;
5250 dst_offset += cur_size_in_bytes;
5251 }
5252
5253 r = radeon_fence_emit(rdev, fence, ring->idx);
5254 if (r) {
5255 radeon_ring_unlock_undo(rdev, ring);
5256 return r;
5257 }
5258
5259 radeon_ring_unlock_commit(rdev, ring);
5260 radeon_semaphore_free(rdev, &sem, *fence);
5261
5262 return r;
5263}
5264
Alex Deucher9b136d52012-03-20 17:18:23 -04005265/*
5266 * startup/shutdown callbacks
5267 */
5268static int si_startup(struct radeon_device *rdev)
5269{
5270 struct radeon_ring *ring;
5271 int r;
5272
5273 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5274 !rdev->rlc_fw || !rdev->mc_fw) {
5275 r = si_init_microcode(rdev);
5276 if (r) {
5277 DRM_ERROR("Failed to load firmware!\n");
5278 return r;
5279 }
5280 }
5281
5282 r = si_mc_load_microcode(rdev);
5283 if (r) {
5284 DRM_ERROR("Failed to load MC firmware!\n");
5285 return r;
5286 }
5287
5288 r = r600_vram_scratch_init(rdev);
5289 if (r)
5290 return r;
5291
5292 si_mc_program(rdev);
5293 r = si_pcie_gart_enable(rdev);
5294 if (r)
5295 return r;
5296 si_gpu_init(rdev);
5297
Alex Deucher9b136d52012-03-20 17:18:23 -04005298 /* allocate rlc buffers */
5299 r = si_rlc_init(rdev);
5300 if (r) {
5301 DRM_ERROR("Failed to init rlc BOs!\n");
5302 return r;
5303 }
5304
5305 /* allocate wb buffer */
5306 r = radeon_wb_init(rdev);
5307 if (r)
5308 return r;
5309
5310 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5311 if (r) {
5312 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5313 return r;
5314 }
5315
5316 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5317 if (r) {
5318 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5319 return r;
5320 }
5321
5322 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5323 if (r) {
5324 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5325 return r;
5326 }
5327
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005328 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5329 if (r) {
5330 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5331 return r;
5332 }
5333
5334 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5335 if (r) {
5336 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5337 return r;
5338 }
5339
Alex Deucher1df0d522013-04-26 18:03:44 -04005340 if (rdev->has_uvd) {
5341 r = rv770_uvd_resume(rdev);
5342 if (!r) {
5343 r = radeon_fence_driver_start_ring(rdev,
5344 R600_RING_TYPE_UVD_INDEX);
5345 if (r)
5346 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5347 }
Christian Königf2ba57b2013-04-08 12:41:29 +02005348 if (r)
Alex Deucher1df0d522013-04-26 18:03:44 -04005349 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
Christian Königf2ba57b2013-04-08 12:41:29 +02005350 }
Christian Königf2ba57b2013-04-08 12:41:29 +02005351
Alex Deucher9b136d52012-03-20 17:18:23 -04005352 /* Enable IRQ */
Adis Hamziće49f3952013-06-02 16:47:54 +02005353 if (!rdev->irq.installed) {
5354 r = radeon_irq_kms_init(rdev);
5355 if (r)
5356 return r;
5357 }
5358
Alex Deucher9b136d52012-03-20 17:18:23 -04005359 r = si_irq_init(rdev);
5360 if (r) {
5361 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5362 radeon_irq_kms_fini(rdev);
5363 return r;
5364 }
5365 si_irq_set(rdev);
5366
5367 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5368 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5369 CP_RB0_RPTR, CP_RB0_WPTR,
5370 0, 0xfffff, RADEON_CP_PACKET2);
5371 if (r)
5372 return r;
5373
5374 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5375 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5376 CP_RB1_RPTR, CP_RB1_WPTR,
5377 0, 0xfffff, RADEON_CP_PACKET2);
5378 if (r)
5379 return r;
5380
5381 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5382 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5383 CP_RB2_RPTR, CP_RB2_WPTR,
5384 0, 0xfffff, RADEON_CP_PACKET2);
5385 if (r)
5386 return r;
5387
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005388 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5389 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5390 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5391 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5392 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5393 if (r)
5394 return r;
5395
5396 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5397 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5398 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5399 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5400 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5401 if (r)
5402 return r;
5403
Alex Deucher9b136d52012-03-20 17:18:23 -04005404 r = si_cp_load_microcode(rdev);
5405 if (r)
5406 return r;
5407 r = si_cp_resume(rdev);
5408 if (r)
5409 return r;
5410
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005411 r = cayman_dma_resume(rdev);
5412 if (r)
5413 return r;
5414
Alex Deucher1df0d522013-04-26 18:03:44 -04005415 if (rdev->has_uvd) {
5416 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5417 if (ring->ring_size) {
5418 r = radeon_ring_init(rdev, ring, ring->ring_size,
5419 R600_WB_UVD_RPTR_OFFSET,
5420 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5421 0, 0xfffff, RADEON_CP_PACKET2);
5422 if (!r)
5423 r = r600_uvd_init(rdev);
5424 if (r)
5425 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5426 }
Christian Königf2ba57b2013-04-08 12:41:29 +02005427 }
5428
Christian König2898c342012-07-05 11:55:34 +02005429 r = radeon_ib_pool_init(rdev);
5430 if (r) {
5431 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
Alex Deucher9b136d52012-03-20 17:18:23 -04005432 return r;
Christian König2898c342012-07-05 11:55:34 +02005433 }
Alex Deucher9b136d52012-03-20 17:18:23 -04005434
Christian Königc6105f22012-07-05 14:32:00 +02005435 r = radeon_vm_manager_init(rdev);
5436 if (r) {
5437 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
Alex Deucher9b136d52012-03-20 17:18:23 -04005438 return r;
Christian Königc6105f22012-07-05 14:32:00 +02005439 }
Alex Deucher9b136d52012-03-20 17:18:23 -04005440
5441 return 0;
5442}
5443
5444int si_resume(struct radeon_device *rdev)
5445{
5446 int r;
5447
5448 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5449 * posting will perform necessary task to bring back GPU into good
5450 * shape.
5451 */
5452 /* post card */
5453 atom_asic_init(rdev->mode_info.atom_context);
5454
Alex Deucher205996c2013-03-01 17:08:42 -05005455 /* init golden registers */
5456 si_init_golden_registers(rdev);
5457
Alex Deucher9b136d52012-03-20 17:18:23 -04005458 rdev->accel_working = true;
5459 r = si_startup(rdev);
5460 if (r) {
5461 DRM_ERROR("si startup failed on resume\n");
5462 rdev->accel_working = false;
5463 return r;
5464 }
5465
5466 return r;
5467
5468}
5469
5470int si_suspend(struct radeon_device *rdev)
5471{
Alex Deucherfa3daf92013-03-11 15:32:26 -04005472 radeon_vm_manager_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005473 si_cp_enable(rdev, false);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005474 cayman_dma_stop(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04005475 if (rdev->has_uvd) {
5476 r600_uvd_rbc_stop(rdev);
5477 radeon_uvd_suspend(rdev);
5478 }
Alex Deucher9b136d52012-03-20 17:18:23 -04005479 si_irq_suspend(rdev);
5480 radeon_wb_disable(rdev);
5481 si_pcie_gart_disable(rdev);
5482 return 0;
5483}
5484
5485/* Plan is to move initialization in that function and use
5486 * helper function so that radeon_device_init pretty much
5487 * do nothing more than calling asic specific function. This
5488 * should also allow to remove a bunch of callback function
5489 * like vram_info.
5490 */
5491int si_init(struct radeon_device *rdev)
5492{
5493 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5494 int r;
5495
Alex Deucher9b136d52012-03-20 17:18:23 -04005496 /* Read BIOS */
5497 if (!radeon_get_bios(rdev)) {
5498 if (ASIC_IS_AVIVO(rdev))
5499 return -EINVAL;
5500 }
5501 /* Must be an ATOMBIOS */
5502 if (!rdev->is_atom_bios) {
5503 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5504 return -EINVAL;
5505 }
5506 r = radeon_atombios_init(rdev);
5507 if (r)
5508 return r;
5509
5510 /* Post card if necessary */
5511 if (!radeon_card_posted(rdev)) {
5512 if (!rdev->bios) {
5513 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5514 return -EINVAL;
5515 }
5516 DRM_INFO("GPU not posted. posting now...\n");
5517 atom_asic_init(rdev->mode_info.atom_context);
5518 }
Alex Deucher205996c2013-03-01 17:08:42 -05005519 /* init golden registers */
5520 si_init_golden_registers(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005521 /* Initialize scratch registers */
5522 si_scratch_init(rdev);
5523 /* Initialize surface registers */
5524 radeon_surface_init(rdev);
5525 /* Initialize clocks */
5526 radeon_get_clock_info(rdev->ddev);
5527
5528 /* Fence driver */
5529 r = radeon_fence_driver_init(rdev);
5530 if (r)
5531 return r;
5532
5533 /* initialize memory controller */
5534 r = si_mc_init(rdev);
5535 if (r)
5536 return r;
5537 /* Memory manager */
5538 r = radeon_bo_init(rdev);
5539 if (r)
5540 return r;
5541
Alex Deucher9b136d52012-03-20 17:18:23 -04005542 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5543 ring->ring_obj = NULL;
5544 r600_ring_init(rdev, ring, 1024 * 1024);
5545
5546 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5547 ring->ring_obj = NULL;
5548 r600_ring_init(rdev, ring, 1024 * 1024);
5549
5550 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5551 ring->ring_obj = NULL;
5552 r600_ring_init(rdev, ring, 1024 * 1024);
5553
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005554 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5555 ring->ring_obj = NULL;
5556 r600_ring_init(rdev, ring, 64 * 1024);
5557
5558 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5559 ring->ring_obj = NULL;
5560 r600_ring_init(rdev, ring, 64 * 1024);
5561
Alex Deucher1df0d522013-04-26 18:03:44 -04005562 if (rdev->has_uvd) {
5563 r = radeon_uvd_init(rdev);
5564 if (!r) {
5565 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5566 ring->ring_obj = NULL;
5567 r600_ring_init(rdev, ring, 4096);
5568 }
Christian Königf2ba57b2013-04-08 12:41:29 +02005569 }
5570
Alex Deucher9b136d52012-03-20 17:18:23 -04005571 rdev->ih.ring_obj = NULL;
5572 r600_ih_ring_init(rdev, 64 * 1024);
5573
5574 r = r600_pcie_gart_init(rdev);
5575 if (r)
5576 return r;
5577
Alex Deucher9b136d52012-03-20 17:18:23 -04005578 rdev->accel_working = true;
Alex Deucher9b136d52012-03-20 17:18:23 -04005579 r = si_startup(rdev);
5580 if (r) {
5581 dev_err(rdev->dev, "disabling GPU acceleration\n");
5582 si_cp_fini(rdev);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005583 cayman_dma_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005584 si_irq_fini(rdev);
5585 si_rlc_fini(rdev);
5586 radeon_wb_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02005587 radeon_ib_pool_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005588 radeon_vm_manager_fini(rdev);
5589 radeon_irq_kms_fini(rdev);
5590 si_pcie_gart_fini(rdev);
5591 rdev->accel_working = false;
5592 }
5593
5594 /* Don't start up if the MC ucode is missing.
5595 * The default clocks and voltages before the MC ucode
5596 * is loaded are not suffient for advanced operations.
5597 */
5598 if (!rdev->mc_fw) {
5599 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5600 return -EINVAL;
5601 }
5602
5603 return 0;
5604}
5605
5606void si_fini(struct radeon_device *rdev)
5607{
Alex Deucher9b136d52012-03-20 17:18:23 -04005608 si_cp_fini(rdev);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005609 cayman_dma_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005610 si_irq_fini(rdev);
5611 si_rlc_fini(rdev);
5612 radeon_wb_fini(rdev);
5613 radeon_vm_manager_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02005614 radeon_ib_pool_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005615 radeon_irq_kms_fini(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04005616 if (rdev->has_uvd)
5617 radeon_uvd_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005618 si_pcie_gart_fini(rdev);
5619 r600_vram_scratch_fini(rdev);
5620 radeon_gem_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04005621 radeon_fence_driver_fini(rdev);
5622 radeon_bo_fini(rdev);
5623 radeon_atombios_fini(rdev);
5624 kfree(rdev->bios);
5625 rdev->bios = NULL;
5626}
5627
Marek Olšák6759a0a2012-08-09 16:34:17 +02005628/**
Alex Deucherd0418892013-01-24 10:35:23 -05005629 * si_get_gpu_clock_counter - return GPU clock counter snapshot
Marek Olšák6759a0a2012-08-09 16:34:17 +02005630 *
5631 * @rdev: radeon_device pointer
5632 *
5633 * Fetches a GPU clock counter snapshot (SI).
5634 * Returns the 64 bit clock counter snapshot.
5635 */
Alex Deucherd0418892013-01-24 10:35:23 -05005636uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
Marek Olšák6759a0a2012-08-09 16:34:17 +02005637{
5638 uint64_t clock;
5639
5640 mutex_lock(&rdev->gpu_clock_mutex);
5641 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5642 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5643 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5644 mutex_unlock(&rdev->gpu_clock_mutex);
5645 return clock;
5646}
Christian König2539eb02013-04-08 12:41:34 +02005647
Christian König2539eb02013-04-08 12:41:34 +02005648int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5649{
Christian Königfacd1122013-04-29 11:55:02 +02005650 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
Christian König2539eb02013-04-08 12:41:34 +02005651 int r;
5652
Christian König4ed10832013-04-18 15:25:58 +02005653 /* bypass vclk and dclk with bclk */
5654 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5655 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5656 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5657
5658 /* put PLL in bypass mode */
5659 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5660
5661 if (!vclk || !dclk) {
5662 /* keep the Bypass mode, put PLL to sleep */
5663 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5664 return 0;
5665 }
5666
Christian Königfacd1122013-04-29 11:55:02 +02005667 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5668 16384, 0x03FFFFFF, 0, 128, 5,
5669 &fb_div, &vclk_div, &dclk_div);
5670 if (r)
5671 return r;
Christian König2539eb02013-04-08 12:41:34 +02005672
5673 /* set RESET_ANTI_MUX to 0 */
5674 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5675
5676 /* set VCO_MODE to 1 */
5677 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5678
5679 /* toggle UPLL_SLEEP to 1 then back to 0 */
5680 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5681 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5682
5683 /* deassert UPLL_RESET */
5684 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5685
5686 mdelay(1);
5687
Christian Königfacd1122013-04-29 11:55:02 +02005688 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Christian König2539eb02013-04-08 12:41:34 +02005689 if (r)
5690 return r;
5691
5692 /* assert UPLL_RESET again */
5693 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5694
5695 /* disable spread spectrum. */
5696 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5697
5698 /* set feedback divider */
Christian Königfacd1122013-04-29 11:55:02 +02005699 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
Christian König2539eb02013-04-08 12:41:34 +02005700
5701 /* set ref divider to 0 */
5702 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5703
Christian Königfacd1122013-04-29 11:55:02 +02005704 if (fb_div < 307200)
Christian König2539eb02013-04-08 12:41:34 +02005705 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5706 else
5707 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5708
5709 /* set PDIV_A and PDIV_B */
5710 WREG32_P(CG_UPLL_FUNC_CNTL_2,
Christian Königfacd1122013-04-29 11:55:02 +02005711 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
Christian König2539eb02013-04-08 12:41:34 +02005712 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5713
5714 /* give the PLL some time to settle */
5715 mdelay(15);
5716
5717 /* deassert PLL_RESET */
5718 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5719
5720 mdelay(15);
5721
5722 /* switch from bypass mode to normal mode */
5723 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5724
Christian Königfacd1122013-04-29 11:55:02 +02005725 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Christian König2539eb02013-04-08 12:41:34 +02005726 if (r)
5727 return r;
5728
5729 /* switch VCLK and DCLK selection */
5730 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5731 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5732 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5733
5734 mdelay(100);
5735
5736 return 0;
5737}