blob: dfb5a1db87d4a8651fd3a9bf7494d14287af7803 [file] [log] [blame]
Jerome Glisse771fe6b2009-06-05 14:42:42 +02001/*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <glisse@freedesktop.org>
26 */
David Howells760285e2012-10-02 18:01:07 +010027#include <drm/drmP.h>
28#include <drm/radeon_drm.h>
Jerome Glisse771fe6b2009-06-05 14:42:42 +020029#include "radeon_reg.h"
30#include "radeon.h"
Christian König860024e2013-09-07 18:29:01 +020031#include "radeon_trace.h"
Jerome Glisse771fe6b2009-06-05 14:42:42 +020032
Lauri Kasanen1109ca02012-08-31 13:43:50 -040033static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
Jerome Glisse771fe6b2009-06-05 14:42:42 +020034{
35 struct drm_device *ddev = p->rdev->ddev;
36 struct radeon_cs_chunk *chunk;
37 unsigned i, j;
38 bool duplicate;
39
40 if (p->chunk_relocs_idx == -1) {
41 return 0;
42 }
43 chunk = &p->chunks[p->chunk_relocs_idx];
Alex Deuchercf4ccd02011-11-18 10:19:47 -050044 p->dma_reloc_idx = 0;
Jerome Glisse771fe6b2009-06-05 14:42:42 +020045 /* FIXME: we assume that each relocs use 4 dwords */
46 p->nrelocs = chunk->length_dw / 4;
47 p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
48 if (p->relocs_ptr == NULL) {
49 return -ENOMEM;
50 }
51 p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
52 if (p->relocs == NULL) {
53 return -ENOMEM;
54 }
55 for (i = 0; i < p->nrelocs; i++) {
56 struct drm_radeon_cs_reloc *r;
57
58 duplicate = false;
59 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
Christian König16557f12011-10-24 14:59:17 +020060 for (j = 0; j < i; j++) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +020061 if (r->handle == p->relocs[j].handle) {
62 p->relocs_ptr[i] = &p->relocs[j];
63 duplicate = true;
64 break;
65 }
66 }
Christian König4474f3a2013-04-08 12:41:28 +020067 if (duplicate) {
Christian König16557f12011-10-24 14:59:17 +020068 p->relocs[i].handle = 0;
Christian König4474f3a2013-04-08 12:41:28 +020069 continue;
70 }
71
72 p->relocs[i].gobj = drm_gem_object_lookup(ddev, p->filp,
73 r->handle);
74 if (p->relocs[i].gobj == NULL) {
75 DRM_ERROR("gem object lookup failed 0x%x\n",
76 r->handle);
77 return -ENOENT;
78 }
79 p->relocs_ptr[i] = &p->relocs[i];
80 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
81 p->relocs[i].lobj.bo = p->relocs[i].robj;
82 p->relocs[i].lobj.written = !!r->write_domain;
83
Christian König4f66c592013-09-15 13:31:28 +020084 /* the first reloc of an UVD job is the msg and that must be in
85 VRAM, also but everything into VRAM on AGP cards to avoid
86 image corruptions */
87 if (p->ring == R600_RING_TYPE_UVD_INDEX &&
Alex Deucher4ca5a6c2013-09-15 23:23:07 -040088 (i == 0 || drm_pci_device_is_agp(p->rdev->ddev))) {
Christian Königbcf6f1e2013-10-15 20:12:03 +020089 /* TODO: is this still needed for NI+ ? */
Christian Königf2ba57b2013-04-08 12:41:29 +020090 p->relocs[i].lobj.domain =
91 RADEON_GEM_DOMAIN_VRAM;
92
93 p->relocs[i].lobj.alt_domain =
94 RADEON_GEM_DOMAIN_VRAM;
95
96 } else {
97 uint32_t domain = r->write_domain ?
98 r->write_domain : r->read_domains;
99
100 p->relocs[i].lobj.domain = domain;
101 if (domain == RADEON_GEM_DOMAIN_VRAM)
102 domain |= RADEON_GEM_DOMAIN_GTT;
103 p->relocs[i].lobj.alt_domain = domain;
104 }
Christian König4474f3a2013-04-08 12:41:28 +0200105
106 p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
107 p->relocs[i].handle = r->handle;
108
109 radeon_bo_list_add_object(&p->relocs[i].lobj,
110 &p->validated);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200111 }
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200112 return radeon_bo_list_validate(&p->ticket, &p->validated, p->ring);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200113}
114
Jerome Glisse721604a2012-01-05 22:11:05 -0500115static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
116{
117 p->priority = priority;
118
119 switch (ring) {
120 default:
121 DRM_ERROR("unknown ring id: %d\n", ring);
122 return -EINVAL;
123 case RADEON_CS_RING_GFX:
124 p->ring = RADEON_RING_TYPE_GFX_INDEX;
125 break;
126 case RADEON_CS_RING_COMPUTE:
Alex Deucher963e81f2013-06-26 17:37:11 -0400127 if (p->rdev->family >= CHIP_TAHITI) {
Alex Deucher8d5ef7b2012-03-20 17:18:24 -0400128 if (p->priority > 0)
129 p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
130 else
131 p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
132 } else
133 p->ring = RADEON_RING_TYPE_GFX_INDEX;
Jerome Glisse721604a2012-01-05 22:11:05 -0500134 break;
Alex Deucher278a3342012-12-13 12:27:28 -0500135 case RADEON_CS_RING_DMA:
136 if (p->rdev->family >= CHIP_CAYMAN) {
137 if (p->priority > 0)
138 p->ring = R600_RING_TYPE_DMA_INDEX;
139 else
140 p->ring = CAYMAN_RING_TYPE_DMA1_INDEX;
Alex Deucherb9ace362014-01-27 10:59:51 -0500141 } else if (p->rdev->family >= CHIP_RV770) {
Alex Deucher278a3342012-12-13 12:27:28 -0500142 p->ring = R600_RING_TYPE_DMA_INDEX;
143 } else {
144 return -EINVAL;
145 }
146 break;
Christian Königf2ba57b2013-04-08 12:41:29 +0200147 case RADEON_CS_RING_UVD:
148 p->ring = R600_RING_TYPE_UVD_INDEX;
149 break;
Jerome Glisse721604a2012-01-05 22:11:05 -0500150 }
151 return 0;
152}
153
Christian König220907d2012-05-10 16:46:43 +0200154static void radeon_cs_sync_rings(struct radeon_cs_parser *p)
Christian König93504fc2012-01-05 22:11:06 -0500155{
Christian König220907d2012-05-10 16:46:43 +0200156 int i;
Christian König93504fc2012-01-05 22:11:06 -0500157
Christian Königcdac5502012-02-23 15:18:42 +0100158 for (i = 0; i < p->nrelocs; i++) {
Christian Königf82cbdd2012-08-09 16:35:36 +0200159 if (!p->relocs[i].robj)
Christian Königcdac5502012-02-23 15:18:42 +0100160 continue;
161
Christian König1654b812013-11-12 12:58:05 +0100162 radeon_semaphore_sync_to(p->ib.semaphore,
163 p->relocs[i].robj->tbo.sync_obj);
Christian Königcdac5502012-02-23 15:18:42 +0100164 }
Christian König93504fc2012-01-05 22:11:06 -0500165}
166
Alex Deucher9b001472012-05-30 10:09:30 -0400167/* XXX: note that this is called from the legacy UMS CS ioctl as well */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200168int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
169{
170 struct drm_radeon_cs *cs = data;
171 uint64_t *chunk_array_ptr;
Jerome Glisse721604a2012-01-05 22:11:05 -0500172 unsigned size, i;
173 u32 ring = RADEON_CS_RING_GFX;
174 s32 priority = 0;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200175
176 if (!cs->num_chunks) {
177 return 0;
178 }
179 /* get chunks */
180 INIT_LIST_HEAD(&p->validated);
181 p->idx = 0;
Jerome Glissef2e39222012-05-09 15:35:02 +0200182 p->ib.sa_bo = NULL;
183 p->ib.semaphore = NULL;
184 p->const_ib.sa_bo = NULL;
185 p->const_ib.semaphore = NULL;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200186 p->chunk_ib_idx = -1;
187 p->chunk_relocs_idx = -1;
Jerome Glisse721604a2012-01-05 22:11:05 -0500188 p->chunk_flags_idx = -1;
Alex Deucherdfcf5f32012-03-20 17:18:14 -0400189 p->chunk_const_ib_idx = -1;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200190 p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
191 if (p->chunks_array == NULL) {
192 return -ENOMEM;
193 }
194 chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100195 if (copy_from_user(p->chunks_array, chunk_array_ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200196 sizeof(uint64_t)*cs->num_chunks)) {
197 return -EFAULT;
198 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500199 p->cs_flags = 0;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200200 p->nchunks = cs->num_chunks;
201 p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
202 if (p->chunks == NULL) {
203 return -ENOMEM;
204 }
205 for (i = 0; i < p->nchunks; i++) {
206 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
207 struct drm_radeon_cs_chunk user_chunk;
208 uint32_t __user *cdata;
209
210 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100211 if (copy_from_user(&user_chunk, chunk_ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200212 sizeof(struct drm_radeon_cs_chunk))) {
213 return -EFAULT;
214 }
Dave Airlie5176fdc2009-06-30 11:47:14 +1000215 p->chunks[i].length_dw = user_chunk.length_dw;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200216 p->chunks[i].chunk_id = user_chunk.chunk_id;
217 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
218 p->chunk_relocs_idx = i;
219 }
220 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
221 p->chunk_ib_idx = i;
Dave Airlie5176fdc2009-06-30 11:47:14 +1000222 /* zero length IB isn't useful */
223 if (p->chunks[i].length_dw == 0)
224 return -EINVAL;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200225 }
Alex Deucherdfcf5f32012-03-20 17:18:14 -0400226 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) {
227 p->chunk_const_ib_idx = i;
228 /* zero length CONST IB isn't useful */
229 if (p->chunks[i].length_dw == 0)
230 return -EINVAL;
231 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500232 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
233 p->chunk_flags_idx = i;
234 /* zero length flags aren't useful */
235 if (p->chunks[i].length_dw == 0)
236 return -EINVAL;
Marek Olšáke70f2242011-10-25 01:38:45 +0200237 }
Dave Airlie5176fdc2009-06-30 11:47:14 +1000238
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200239 size = p->chunks[i].length_dw;
240 cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
241 p->chunks[i].user_ptr = cdata;
242 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB)
243 continue;
244
245 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
246 if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP))
247 continue;
248 }
249
250 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
251 size *= sizeof(uint32_t);
252 if (p->chunks[i].kdata == NULL) {
253 return -ENOMEM;
254 }
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100255 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200256 return -EFAULT;
257 }
258 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
259 p->cs_flags = p->chunks[i].kdata[0];
260 if (p->chunks[i].length_dw > 1)
261 ring = p->chunks[i].kdata[1];
262 if (p->chunks[i].length_dw > 2)
263 priority = (s32)p->chunks[i].kdata[2];
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200264 }
265 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500266
Alex Deucher9b001472012-05-30 10:09:30 -0400267 /* these are KMS only */
268 if (p->rdev) {
269 if ((p->cs_flags & RADEON_CS_USE_VM) &&
270 !p->rdev->vm_manager.enabled) {
271 DRM_ERROR("VM not active on asic!\n");
272 return -EINVAL;
273 }
274
Alex Deucher9b001472012-05-30 10:09:30 -0400275 if (radeon_cs_get_ring(p, ring, priority))
276 return -EINVAL;
Christian König57449042013-04-08 12:41:27 +0200277
278 /* we only support VM on some SI+ rings */
Christian König76a0df82013-08-13 11:56:50 +0200279 if ((p->rdev->asic->ring[p->ring]->cs_parse == NULL) &&
Christian König57449042013-04-08 12:41:27 +0200280 ((p->cs_flags & RADEON_CS_USE_VM) == 0)) {
281 DRM_ERROR("Ring %d requires VM!\n", p->ring);
282 return -EINVAL;
283 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200284 }
Marek Olšáke70f2242011-10-25 01:38:45 +0200285
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200286 return 0;
287}
288
289/**
290 * cs_parser_fini() - clean parser states
291 * @parser: parser structure holding parsing context.
292 * @error: error number
293 *
294 * If error is set than unvalidate buffer, otherwise just free memory
295 * used by parsing context.
296 **/
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200297static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200298{
299 unsigned i;
300
Jerome Glissee43b5ec2012-08-06 12:32:21 -0400301 if (!error) {
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200302 ttm_eu_fence_buffer_objects(&parser->ticket,
303 &parser->validated,
Jerome Glissef2e39222012-05-09 15:35:02 +0200304 parser->ib.fence);
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200305 } else if (backoff) {
306 ttm_eu_backoff_reservation(&parser->ticket,
307 &parser->validated);
Jerome Glissee43b5ec2012-08-06 12:32:21 -0400308 }
Thomas Hellstrom147666f2010-11-17 12:38:32 +0000309
Pauli Nieminenfcbc4512010-03-19 07:44:33 +0000310 if (parser->relocs != NULL) {
311 for (i = 0; i < parser->nrelocs; i++) {
312 if (parser->relocs[i].gobj)
313 drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
314 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200315 }
Michel Dänzer48e113e2009-09-15 17:09:32 +0200316 kfree(parser->track);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200317 kfree(parser->relocs);
318 kfree(parser->relocs_ptr);
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200319 for (i = 0; i < parser->nchunks; i++)
320 drm_free_large(parser->chunks[i].kdata);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200321 kfree(parser->chunks);
322 kfree(parser->chunks_array);
323 radeon_ib_free(parser->rdev, &parser->ib);
Jerome Glissef2e39222012-05-09 15:35:02 +0200324 radeon_ib_free(parser->rdev, &parser->const_ib);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200325}
326
Jerome Glisse721604a2012-01-05 22:11:05 -0500327static int radeon_cs_ib_chunk(struct radeon_device *rdev,
328 struct radeon_cs_parser *parser)
329{
Jerome Glisse721604a2012-01-05 22:11:05 -0500330 int r;
331
332 if (parser->chunk_ib_idx == -1)
333 return 0;
334
335 if (parser->cs_flags & RADEON_CS_USE_VM)
336 return 0;
337
Christian Königeb0c19c2012-02-23 15:18:44 +0100338 r = radeon_cs_parse(rdev, parser->ring, parser);
Jerome Glisse721604a2012-01-05 22:11:05 -0500339 if (r || parser->parser_error) {
340 DRM_ERROR("Invalid command stream !\n");
341 return r;
342 }
Alex Deucherce3537d2013-07-24 12:12:49 -0400343
344 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
345 radeon_uvd_note_usage(rdev);
346
Christian König220907d2012-05-10 16:46:43 +0200347 radeon_cs_sync_rings(parser);
Christian König4ef72562012-07-13 13:06:00 +0200348 r = radeon_ib_schedule(rdev, &parser->ib, NULL);
Jerome Glisse721604a2012-01-05 22:11:05 -0500349 if (r) {
350 DRM_ERROR("Failed to schedule IB !\n");
351 }
Christian König93bf8882012-07-03 14:05:41 +0200352 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -0500353}
354
355static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
356 struct radeon_vm *vm)
357{
Jerome Glisse3e8970f2012-08-13 12:07:33 -0400358 struct radeon_device *rdev = parser->rdev;
Jerome Glisse721604a2012-01-05 22:11:05 -0500359 struct radeon_bo_list *lobj;
360 struct radeon_bo *bo;
361 int r;
362
Christian König9c57a6b2013-11-25 15:42:11 +0100363 r = radeon_vm_bo_update(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem);
Jerome Glisse3e8970f2012-08-13 12:07:33 -0400364 if (r) {
365 return r;
366 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500367 list_for_each_entry(lobj, &parser->validated, tv.head) {
368 bo = lobj->bo;
Christian König9c57a6b2013-11-25 15:42:11 +0100369 r = radeon_vm_bo_update(parser->rdev, vm, bo, &bo->tbo.mem);
Jerome Glisse721604a2012-01-05 22:11:05 -0500370 if (r) {
371 return r;
372 }
373 }
374 return 0;
375}
376
377static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
378 struct radeon_cs_parser *parser)
379{
Jerome Glisse721604a2012-01-05 22:11:05 -0500380 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
381 struct radeon_vm *vm = &fpriv->vm;
382 int r;
383
384 if (parser->chunk_ib_idx == -1)
385 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500386 if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
387 return 0;
388
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200389 if (parser->const_ib.length_dw) {
Jerome Glissef2e39222012-05-09 15:35:02 +0200390 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
Alex Deucherdfcf5f32012-03-20 17:18:14 -0400391 if (r) {
392 return r;
393 }
394 }
395
Jerome Glissef2e39222012-05-09 15:35:02 +0200396 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
Jerome Glisse721604a2012-01-05 22:11:05 -0500397 if (r) {
398 return r;
399 }
400
Alex Deucherce3537d2013-07-24 12:12:49 -0400401 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
402 radeon_uvd_note_usage(rdev);
403
Christian König36ff39c2012-05-09 10:07:08 +0200404 mutex_lock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -0500405 mutex_lock(&vm->mutex);
Christian Königddf03f52012-08-09 20:02:28 +0200406 r = radeon_vm_alloc_pt(rdev, vm);
Jerome Glisse721604a2012-01-05 22:11:05 -0500407 if (r) {
408 goto out;
409 }
410 r = radeon_bo_vm_update_pte(parser, vm);
411 if (r) {
412 goto out;
413 }
Christian König220907d2012-05-10 16:46:43 +0200414 radeon_cs_sync_rings(parser);
Christian König1654b812013-11-12 12:58:05 +0100415 radeon_semaphore_sync_to(parser->ib.semaphore, vm->fence);
416 radeon_semaphore_sync_to(parser->ib.semaphore,
417 radeon_vm_grab_id(rdev, vm, parser->ring));
Christian König4ef72562012-07-13 13:06:00 +0200418
Alex Deucherdfcf5f32012-03-20 17:18:14 -0400419 if ((rdev->family >= CHIP_TAHITI) &&
420 (parser->chunk_const_ib_idx != -1)) {
Christian König4ef72562012-07-13 13:06:00 +0200421 r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib);
422 } else {
423 r = radeon_ib_schedule(rdev, &parser->ib, NULL);
Alex Deucherdfcf5f32012-03-20 17:18:14 -0400424 }
425
Jerome Glisse721604a2012-01-05 22:11:05 -0500426 if (!r) {
Christian Königee60e292012-08-09 16:21:08 +0200427 radeon_vm_fence(rdev, vm, parser->ib.fence);
Jerome Glisse721604a2012-01-05 22:11:05 -0500428 }
Christian Königee60e292012-08-09 16:21:08 +0200429
430out:
Christian König13e55c32012-10-09 13:31:19 +0200431 radeon_vm_add_to_lru(rdev, vm);
Christian König36ff39c2012-05-09 10:07:08 +0200432 mutex_unlock(&vm->mutex);
433 mutex_unlock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -0500434 return r;
435}
436
Christian König6c6f4782012-05-02 15:11:19 +0200437static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
438{
439 if (r == -EDEADLK) {
440 r = radeon_gpu_reset(rdev);
441 if (!r)
442 r = -EAGAIN;
443 }
444 return r;
445}
446
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200447static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser)
448{
449 struct radeon_cs_chunk *ib_chunk;
450 struct radeon_vm *vm = NULL;
451 int r;
452
453 if (parser->chunk_ib_idx == -1)
454 return 0;
455
456 if (parser->cs_flags & RADEON_CS_USE_VM) {
457 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
458 vm = &fpriv->vm;
459
460 if ((rdev->family >= CHIP_TAHITI) &&
461 (parser->chunk_const_ib_idx != -1)) {
462 ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
463 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
464 DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
465 return -EINVAL;
466 }
467 r = radeon_ib_get(rdev, parser->ring, &parser->const_ib,
468 vm, ib_chunk->length_dw * 4);
469 if (r) {
470 DRM_ERROR("Failed to get const ib !\n");
471 return r;
472 }
473 parser->const_ib.is_const_ib = true;
474 parser->const_ib.length_dw = ib_chunk->length_dw;
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100475 if (copy_from_user(parser->const_ib.ptr,
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200476 ib_chunk->user_ptr,
477 ib_chunk->length_dw * 4))
478 return -EFAULT;
479 }
480
481 ib_chunk = &parser->chunks[parser->chunk_ib_idx];
482 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
483 DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
484 return -EINVAL;
485 }
486 }
487 ib_chunk = &parser->chunks[parser->chunk_ib_idx];
488
489 r = radeon_ib_get(rdev, parser->ring, &parser->ib,
490 vm, ib_chunk->length_dw * 4);
491 if (r) {
492 DRM_ERROR("Failed to get ib !\n");
493 return r;
494 }
495 parser->ib.length_dw = ib_chunk->length_dw;
496 if (ib_chunk->kdata)
497 memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4);
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100498 else if (copy_from_user(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4))
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200499 return -EFAULT;
500 return 0;
501}
502
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200503int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
504{
505 struct radeon_device *rdev = dev->dev_private;
506 struct radeon_cs_parser parser;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200507 int r;
508
Jerome Glissedee53e72012-07-02 12:45:19 -0400509 down_read(&rdev->exclusive_lock);
Jerome Glisse6b7746e2012-02-20 17:57:20 -0500510 if (!rdev->accel_working) {
Jerome Glissedee53e72012-07-02 12:45:19 -0400511 up_read(&rdev->exclusive_lock);
Jerome Glisse6b7746e2012-02-20 17:57:20 -0500512 return -EBUSY;
513 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200514 /* initialize parser */
515 memset(&parser, 0, sizeof(struct radeon_cs_parser));
516 parser.filp = filp;
517 parser.rdev = rdev;
Jerome Glissec8c15ff2010-01-18 13:01:36 +0100518 parser.dev = rdev->dev;
Dave Airlie428c6e32011-06-08 19:58:29 +1000519 parser.family = rdev->family;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200520 r = radeon_cs_parser_init(&parser, data);
521 if (r) {
522 DRM_ERROR("Failed to initialize parser !\n");
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200523 radeon_cs_parser_fini(&parser, r, false);
Jerome Glissedee53e72012-07-02 12:45:19 -0400524 up_read(&rdev->exclusive_lock);
Christian König6c6f4782012-05-02 15:11:19 +0200525 r = radeon_cs_handle_lockup(rdev, r);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200526 return r;
527 }
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200528
529 r = radeon_cs_ib_fill(rdev, &parser);
530 if (!r) {
531 r = radeon_cs_parser_relocs(&parser);
532 if (r && r != -ERESTARTSYS)
Dave Airlie97f23b32010-03-19 10:33:44 +1000533 DRM_ERROR("Failed to parse relocation %d!\n", r);
Maarten Lankhorst28a326c2013-10-09 14:36:57 +0200534 }
535
536 if (r) {
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200537 radeon_cs_parser_fini(&parser, r, false);
Jerome Glissedee53e72012-07-02 12:45:19 -0400538 up_read(&rdev->exclusive_lock);
Christian König6c6f4782012-05-02 15:11:19 +0200539 r = radeon_cs_handle_lockup(rdev, r);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200540 return r;
541 }
Christian König55b51c82013-04-18 15:25:59 +0200542
Christian König860024e2013-09-07 18:29:01 +0200543 trace_radeon_cs(&parser);
544
Jerome Glisse721604a2012-01-05 22:11:05 -0500545 r = radeon_cs_ib_chunk(rdev, &parser);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200546 if (r) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500547 goto out;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200548 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500549 r = radeon_cs_ib_vm_chunk(rdev, &parser);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200550 if (r) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500551 goto out;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200552 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500553out:
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200554 radeon_cs_parser_fini(&parser, r, true);
Jerome Glissedee53e72012-07-02 12:45:19 -0400555 up_read(&rdev->exclusive_lock);
Christian König6c6f4782012-05-02 15:11:19 +0200556 r = radeon_cs_handle_lockup(rdev, r);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200557 return r;
558}
Dave Airlie513bcb42009-09-23 16:56:27 +1000559
Ilija Hadzic4db01312013-01-02 18:27:40 -0500560/**
561 * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet
562 * @parser: parser structure holding parsing context.
563 * @pkt: where to store packet information
564 *
565 * Assume that chunk_ib_index is properly set. Will return -EINVAL
566 * if packet is bigger than remaining ib size. or if packets is unknown.
567 **/
568int radeon_cs_packet_parse(struct radeon_cs_parser *p,
569 struct radeon_cs_packet *pkt,
570 unsigned idx)
571{
572 struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx];
573 struct radeon_device *rdev = p->rdev;
574 uint32_t header;
575
576 if (idx >= ib_chunk->length_dw) {
577 DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
578 idx, ib_chunk->length_dw);
579 return -EINVAL;
580 }
581 header = radeon_get_ib_value(p, idx);
582 pkt->idx = idx;
583 pkt->type = RADEON_CP_PACKET_GET_TYPE(header);
584 pkt->count = RADEON_CP_PACKET_GET_COUNT(header);
585 pkt->one_reg_wr = 0;
586 switch (pkt->type) {
587 case RADEON_PACKET_TYPE0:
588 if (rdev->family < CHIP_R600) {
589 pkt->reg = R100_CP_PACKET0_GET_REG(header);
590 pkt->one_reg_wr =
591 RADEON_CP_PACKET0_GET_ONE_REG_WR(header);
592 } else
593 pkt->reg = R600_CP_PACKET0_GET_REG(header);
594 break;
595 case RADEON_PACKET_TYPE3:
596 pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header);
597 break;
598 case RADEON_PACKET_TYPE2:
599 pkt->count = -1;
600 break;
601 default:
602 DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx);
603 return -EINVAL;
604 }
605 if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) {
606 DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n",
607 pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw);
608 return -EINVAL;
609 }
610 return 0;
611}
Ilija Hadzic9ffb7a62013-01-02 18:27:42 -0500612
613/**
614 * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP
615 * @p: structure holding the parser context.
616 *
617 * Check if the next packet is NOP relocation packet3.
618 **/
619bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p)
620{
621 struct radeon_cs_packet p3reloc;
622 int r;
623
624 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
625 if (r)
626 return false;
627 if (p3reloc.type != RADEON_PACKET_TYPE3)
628 return false;
629 if (p3reloc.opcode != RADEON_PACKET3_NOP)
630 return false;
631 return true;
632}
Ilija Hadzicc3ad63a2013-01-02 18:27:45 -0500633
634/**
635 * radeon_cs_dump_packet() - dump raw packet context
636 * @p: structure holding the parser context.
637 * @pkt: structure holding the packet.
638 *
639 * Used mostly for debugging and error reporting.
640 **/
641void radeon_cs_dump_packet(struct radeon_cs_parser *p,
642 struct radeon_cs_packet *pkt)
643{
644 volatile uint32_t *ib;
645 unsigned i;
646 unsigned idx;
647
648 ib = p->ib.ptr;
649 idx = pkt->idx;
650 for (i = 0; i <= (pkt->count + 1); i++, idx++)
651 DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]);
652}
653
Ilija Hadzice9716992013-01-02 18:27:46 -0500654/**
655 * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet
656 * @parser: parser structure holding parsing context.
657 * @data: pointer to relocation data
658 * @offset_start: starting offset
659 * @offset_mask: offset mask (to align start offset on)
660 * @reloc: reloc informations
661 *
662 * Check if next packet is relocation packet3, do bo validation and compute
663 * GPU offset using the provided start.
664 **/
665int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
666 struct radeon_cs_reloc **cs_reloc,
667 int nomm)
668{
669 struct radeon_cs_chunk *relocs_chunk;
670 struct radeon_cs_packet p3reloc;
671 unsigned idx;
672 int r;
673
674 if (p->chunk_relocs_idx == -1) {
675 DRM_ERROR("No relocation chunk !\n");
676 return -EINVAL;
677 }
678 *cs_reloc = NULL;
679 relocs_chunk = &p->chunks[p->chunk_relocs_idx];
680 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
681 if (r)
682 return r;
683 p->idx += p3reloc.count + 2;
684 if (p3reloc.type != RADEON_PACKET_TYPE3 ||
685 p3reloc.opcode != RADEON_PACKET3_NOP) {
686 DRM_ERROR("No packet3 for relocation for packet at %d.\n",
687 p3reloc.idx);
688 radeon_cs_dump_packet(p, &p3reloc);
689 return -EINVAL;
690 }
691 idx = radeon_get_ib_value(p, p3reloc.idx + 1);
692 if (idx >= relocs_chunk->length_dw) {
693 DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
694 idx, relocs_chunk->length_dw);
695 radeon_cs_dump_packet(p, &p3reloc);
696 return -EINVAL;
697 }
698 /* FIXME: we assume reloc size is 4 dwords */
699 if (nomm) {
700 *cs_reloc = p->relocs;
701 (*cs_reloc)->lobj.gpu_offset =
702 (u64)relocs_chunk->kdata[idx + 3] << 32;
703 (*cs_reloc)->lobj.gpu_offset |= relocs_chunk->kdata[idx + 0];
704 } else
705 *cs_reloc = p->relocs_ptr[(idx / 4)];
706 return 0;
707}