blob: a28b5102c4a5b1ed8b0b0b662ec22ae0b17f3853 [file] [log] [blame]
Ben Skeggs6ee73862009-12-11 19:24:15 +10001/*
2 * Copyright (C) 2008 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
Ben Skeggs6ee73862009-12-11 19:24:15 +100026
Ben Skeggsebb945a2012-07-20 08:17:34 +100027#include "nouveau_drm.h"
Ben Skeggs6ee73862009-12-11 19:24:15 +100028#include "nouveau_dma.h"
Ben Skeggsd375e7d52012-04-30 13:30:00 +100029#include "nouveau_fence.h"
Ben Skeggsebb945a2012-07-20 08:17:34 +100030#include "nouveau_abi16.h"
Ben Skeggs6ee73862009-12-11 19:24:15 +100031
Ben Skeggsebb945a2012-07-20 08:17:34 +100032#include "nouveau_ttm.h"
33#include "nouveau_gem.h"
Ben Skeggs6ee73862009-12-11 19:24:15 +100034
Ben Skeggs6ee73862009-12-11 19:24:15 +100035void
36nouveau_gem_object_del(struct drm_gem_object *gem)
37{
David Herrmann55fb74a2013-10-02 10:15:17 +020038 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
Ben Skeggs6ee73862009-12-11 19:24:15 +100039 struct ttm_buffer_object *bo = &nvbo->bo;
40
Dave Airlie22b33e82012-04-02 11:53:06 +010041 if (gem->import_attach)
42 drm_prime_gem_destroy(gem, nvbo->bo.sg);
43
Daniel Vetterfd632aa2010-04-09 19:05:05 +000044 drm_gem_object_release(gem);
David Herrmann55fb74a2013-10-02 10:15:17 +020045
46 /* reset filp so nouveau_bo_del_ttm() can test for it */
47 gem->filp = NULL;
48 ttm_bo_unref(&bo);
Ben Skeggs6ee73862009-12-11 19:24:15 +100049}
50
51int
Ben Skeggs639212d2011-06-03 16:18:26 +100052nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
53{
Ben Skeggsebb945a2012-07-20 08:17:34 +100054 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggs2fd3db62011-06-07 15:25:12 +100055 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
56 struct nouveau_vma *vma;
57 int ret;
Ben Skeggs639212d2011-06-03 16:18:26 +100058
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +100059 if (!cli->vm)
Ben Skeggs639212d2011-06-03 16:18:26 +100060 return 0;
61
Thierry Redingee3939e2014-07-21 13:15:51 +020062 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
Ben Skeggs2fd3db62011-06-07 15:25:12 +100063 if (ret)
64 return ret;
65
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +100066 vma = nouveau_bo_vma_find(nvbo, cli->vm);
Ben Skeggs2fd3db62011-06-07 15:25:12 +100067 if (!vma) {
68 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
69 if (!vma) {
70 ret = -ENOMEM;
71 goto out;
72 }
73
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +100074 ret = nouveau_bo_vma_add(nvbo, cli->vm, vma);
Ben Skeggs2fd3db62011-06-07 15:25:12 +100075 if (ret) {
76 kfree(vma);
77 goto out;
78 }
79 } else {
80 vma->refcount++;
81 }
82
83out:
84 ttm_bo_unreserve(&nvbo->bo);
85 return ret;
Ben Skeggs639212d2011-06-03 16:18:26 +100086}
87
Ben Skeggsc4c70442013-05-07 09:48:30 +100088static void
89nouveau_gem_object_delete(void *data)
90{
91 struct nouveau_vma *vma = data;
92 nouveau_vm_unmap(vma);
93 nouveau_vm_put(vma);
94 kfree(vma);
95}
96
97static void
98nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
99{
100 const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM;
Maarten Lankhorstf2c24b82014-04-02 17:14:48 +0200101 struct fence *fence = NULL;
Ben Skeggsc4c70442013-05-07 09:48:30 +1000102
103 list_del(&vma->head);
104
Maarten Lankhorstf2c24b82014-04-02 17:14:48 +0200105 if (mapped)
106 fence = reservation_object_get_excl(nvbo->bo.resv);
Ben Skeggsc4c70442013-05-07 09:48:30 +1000107
108 if (fence) {
109 nouveau_fence_work(fence, nouveau_gem_object_delete, vma);
110 } else {
111 if (mapped)
112 nouveau_vm_unmap(vma);
113 nouveau_vm_put(vma);
114 kfree(vma);
115 }
Ben Skeggsc4c70442013-05-07 09:48:30 +1000116}
117
Ben Skeggs639212d2011-06-03 16:18:26 +1000118void
119nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
120{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000121 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggs2fd3db62011-06-07 15:25:12 +1000122 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
123 struct nouveau_vma *vma;
124 int ret;
Ben Skeggs639212d2011-06-03 16:18:26 +1000125
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +1000126 if (!cli->vm)
Ben Skeggs639212d2011-06-03 16:18:26 +1000127 return;
Ben Skeggs2fd3db62011-06-07 15:25:12 +1000128
Thierry Redingee3939e2014-07-21 13:15:51 +0200129 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
Ben Skeggs2fd3db62011-06-07 15:25:12 +1000130 if (ret)
131 return;
132
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +1000133 vma = nouveau_bo_vma_find(nvbo, cli->vm);
Ben Skeggs2fd3db62011-06-07 15:25:12 +1000134 if (vma) {
Ben Skeggsc4c70442013-05-07 09:48:30 +1000135 if (--vma->refcount == 0)
136 nouveau_gem_object_unmap(nvbo, vma);
Ben Skeggs2fd3db62011-06-07 15:25:12 +1000137 }
138 ttm_bo_unreserve(&nvbo->bo);
Ben Skeggs639212d2011-06-03 16:18:26 +1000139}
140
141int
Ben Skeggsf6d4e622011-06-07 12:25:36 +1000142nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
143 uint32_t tile_mode, uint32_t tile_flags,
144 struct nouveau_bo **pnvbo)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000145{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000146 struct nouveau_drm *drm = nouveau_drm(dev);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000147 struct nouveau_bo *nvbo;
Ben Skeggs6ba9a682011-02-10 14:42:08 +1000148 u32 flags = 0;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000149 int ret;
150
Ben Skeggs6ba9a682011-02-10 14:42:08 +1000151 if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
152 flags |= TTM_PL_FLAG_VRAM;
153 if (domain & NOUVEAU_GEM_DOMAIN_GART)
154 flags |= TTM_PL_FLAG_TT;
155 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
156 flags |= TTM_PL_FLAG_SYSTEM;
157
Ben Skeggs7375c952011-06-07 14:21:29 +1000158 ret = nouveau_bo_new(dev, size, align, flags, tile_mode,
Dave Airlie22b33e82012-04-02 11:53:06 +0100159 tile_flags, NULL, pnvbo);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000160 if (ret)
161 return ret;
162 nvbo = *pnvbo;
163
Ben Skeggsdb5c8e22011-02-10 13:41:01 +1000164 /* we restrict allowed domains on nv50+ to only the types
165 * that were requested at creation time. not possibly on
166 * earlier chips without busting the ABI.
167 */
168 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
169 NOUVEAU_GEM_DOMAIN_GART;
Ben Skeggs967e7bd2014-08-10 04:10:22 +1000170 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
Ben Skeggsdb5c8e22011-02-10 13:41:01 +1000171 nvbo->valid_domains &= domain;
172
David Herrmann55fb74a2013-10-02 10:15:17 +0200173 /* Initialize the embedded gem-object. We return a single gem-reference
174 * to the caller, instead of a normal nouveau_bo ttm reference. */
175 ret = drm_gem_object_init(dev, &nvbo->gem, nvbo->bo.mem.size);
176 if (ret) {
Ben Skeggs6ee73862009-12-11 19:24:15 +1000177 nouveau_bo_ref(NULL, pnvbo);
178 return -ENOMEM;
179 }
180
David Herrmann55fb74a2013-10-02 10:15:17 +0200181 nvbo->bo.persistent_swap_storage = nvbo->gem.filp;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000182 return 0;
183}
184
185static int
Ben Skeggse758a312011-06-07 13:28:11 +1000186nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
187 struct drm_nouveau_gem_info *rep)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000188{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000189 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000190 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
Ben Skeggse758a312011-06-07 13:28:11 +1000191 struct nouveau_vma *vma;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000192
193 if (nvbo->bo.mem.mem_type == TTM_PL_TT)
194 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
195 else
196 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
197
Ben Skeggse758a312011-06-07 13:28:11 +1000198 rep->offset = nvbo->bo.offset;
Ben Skeggs3ee6f5b2014-08-10 04:10:20 +1000199 if (cli->vm) {
200 vma = nouveau_bo_vma_find(nvbo, cli->vm);
Ben Skeggse758a312011-06-07 13:28:11 +1000201 if (!vma)
202 return -EINVAL;
203
204 rep->offset = vma->offset;
205 }
206
Ben Skeggs6ee73862009-12-11 19:24:15 +1000207 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
David Herrmann72525b32013-07-24 21:08:53 +0200208 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.vma_node);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000209 rep->tile_mode = nvbo->tile_mode;
210 rep->tile_flags = nvbo->tile_flags;
211 return 0;
212}
213
Ben Skeggs6ee73862009-12-11 19:24:15 +1000214int
215nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
216 struct drm_file *file_priv)
217{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000218 struct nouveau_drm *drm = nouveau_drm(dev);
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100219 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggs967e7bd2014-08-10 04:10:22 +1000220 struct nouveau_fb *pfb = nvkm_fb(&drm->device);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000221 struct drm_nouveau_gem_new *req = data;
222 struct nouveau_bo *nvbo = NULL;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000223 int ret = 0;
224
Ben Skeggsebb945a2012-07-20 08:17:34 +1000225 if (!pfb->memtype_valid(pfb, req->info.tile_flags)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000226 NV_PRINTK(error, cli, "bad page flags: 0x%08x\n", req->info.tile_flags);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000227 return -EINVAL;
Ben Skeggs60d2a882010-12-06 15:28:54 +1000228 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000229
Ben Skeggsf6d4e622011-06-07 12:25:36 +1000230 ret = nouveau_gem_new(dev, req->info.size, req->align,
Ben Skeggs6ba9a682011-02-10 14:42:08 +1000231 req->info.domain, req->info.tile_mode,
232 req->info.tile_flags, &nvbo);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000233 if (ret)
234 return ret;
235
David Herrmann55fb74a2013-10-02 10:15:17 +0200236 ret = drm_gem_handle_create(file_priv, &nvbo->gem, &req->info.handle);
Ben Skeggse758a312011-06-07 13:28:11 +1000237 if (ret == 0) {
David Herrmann55fb74a2013-10-02 10:15:17 +0200238 ret = nouveau_gem_info(file_priv, &nvbo->gem, &req->info);
Ben Skeggse758a312011-06-07 13:28:11 +1000239 if (ret)
240 drm_gem_handle_delete(file_priv, req->info.handle);
241 }
242
Dave Airlie29d08b32010-09-27 16:17:17 +1000243 /* drop reference from allocate - handle holds it now */
David Herrmann55fb74a2013-10-02 10:15:17 +0200244 drm_gem_object_unreference_unlocked(&nvbo->gem);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000245 return ret;
246}
247
248static int
249nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
250 uint32_t write_domains, uint32_t valid_domains)
251{
David Herrmann55fb74a2013-10-02 10:15:17 +0200252 struct nouveau_bo *nvbo = nouveau_gem_object(gem);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000253 struct ttm_buffer_object *bo = &nvbo->bo;
Ben Skeggsdb5c8e22011-02-10 13:41:01 +1000254 uint32_t domains = valid_domains & nvbo->valid_domains &
Francisco Jerez78ad0f72010-03-18 13:07:47 +0100255 (write_domains ? write_domains : read_domains);
256 uint32_t pref_flags = 0, valid_flags = 0;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000257
Francisco Jerez78ad0f72010-03-18 13:07:47 +0100258 if (!domains)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000259 return -EINVAL;
260
Francisco Jerez78ad0f72010-03-18 13:07:47 +0100261 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
262 valid_flags |= TTM_PL_FLAG_VRAM;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000263
Francisco Jerez78ad0f72010-03-18 13:07:47 +0100264 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
265 valid_flags |= TTM_PL_FLAG_TT;
266
267 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
268 bo->mem.mem_type == TTM_PL_VRAM)
269 pref_flags |= TTM_PL_FLAG_VRAM;
270
271 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
272 bo->mem.mem_type == TTM_PL_TT)
273 pref_flags |= TTM_PL_FLAG_TT;
274
275 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
276 pref_flags |= TTM_PL_FLAG_VRAM;
277
278 else
279 pref_flags |= TTM_PL_FLAG_TT;
280
281 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
282
Ben Skeggs6ee73862009-12-11 19:24:15 +1000283 return 0;
284}
285
286struct validate_op {
Ben Skeggs6ee73862009-12-11 19:24:15 +1000287 struct list_head vram_list;
288 struct list_head gart_list;
289 struct list_head both_list;
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200290 struct ww_acquire_ctx ticket;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000291};
292
293static void
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200294validate_fini_list(struct list_head *list, struct nouveau_fence *fence,
295 struct ww_acquire_ctx *ticket)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000296{
297 struct list_head *entry, *tmp;
298 struct nouveau_bo *nvbo;
299
300 list_for_each_safe(entry, tmp, list) {
301 nvbo = list_entry(entry, struct nouveau_bo, entry);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000302
Ben Skeggs9360bd12013-11-13 15:18:32 +1000303 if (likely(fence))
304 nouveau_bo_fence(nvbo, fence);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000305
Ben Skeggsa1606a92010-02-12 10:27:35 +1000306 if (unlikely(nvbo->validate_mapped)) {
307 ttm_bo_kunmap(&nvbo->kmap);
308 nvbo->validate_mapped = false;
309 }
310
Ben Skeggs6ee73862009-12-11 19:24:15 +1000311 list_del(&nvbo->entry);
312 nvbo->reserved_by = NULL;
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200313 ttm_bo_unreserve_ticket(&nvbo->bo, ticket);
David Herrmann55fb74a2013-10-02 10:15:17 +0200314 drm_gem_object_unreference_unlocked(&nvbo->gem);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000315 }
316}
317
318static void
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200319validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000320{
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200321 validate_fini_list(&op->vram_list, fence, &op->ticket);
322 validate_fini_list(&op->gart_list, fence, &op->ticket);
323 validate_fini_list(&op->both_list, fence, &op->ticket);
324}
325
326static void
327validate_fini(struct validate_op *op, struct nouveau_fence *fence)
328{
329 validate_fini_no_ticket(op, fence);
330 ww_acquire_fini(&op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000331}
332
333static int
334validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
335 struct drm_nouveau_gem_pushbuf_bo *pbbo,
336 int nr_buffers, struct validate_op *op)
337{
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100338 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000339 struct drm_device *dev = chan->drm->dev;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000340 int trycnt = 0;
341 int ret, i;
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100342 struct nouveau_bo *res_bo = NULL;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000343
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200344 ww_acquire_init(&op->ticket, &reservation_ww_class);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000345retry:
346 if (++trycnt > 100000) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000347 NV_PRINTK(error, cli, "%s failed and gave up.\n", __func__);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000348 return -EINVAL;
349 }
350
351 for (i = 0; i < nr_buffers; i++) {
352 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
353 struct drm_gem_object *gem;
354 struct nouveau_bo *nvbo;
355
356 gem = drm_gem_object_lookup(dev, file_priv, b->handle);
357 if (!gem) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000358 NV_PRINTK(error, cli, "Unknown handle 0x%08x\n", b->handle);
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200359 ww_acquire_done(&op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000360 validate_fini(op, NULL);
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100361 return -ENOENT;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000362 }
David Herrmann55fb74a2013-10-02 10:15:17 +0200363 nvbo = nouveau_gem_object(gem);
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100364 if (nvbo == res_bo) {
365 res_bo = NULL;
366 drm_gem_object_unreference_unlocked(gem);
367 continue;
368 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000369
370 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000371 NV_PRINTK(error, cli, "multiple instances of buffer %d on "
Ben Skeggs6ee73862009-12-11 19:24:15 +1000372 "validation list\n", b->handle);
Ben Skeggs5086f692012-06-25 18:04:27 +1000373 drm_gem_object_unreference_unlocked(gem);
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200374 ww_acquire_done(&op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000375 validate_fini(op, NULL);
376 return -EINVAL;
377 }
378
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200379 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, &op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000380 if (ret) {
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200381 validate_fini_no_ticket(op, NULL);
Maarten Lankhorst5e338402013-06-27 13:48:19 +0200382 if (unlikely(ret == -EDEADLK)) {
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100383 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200384 &op->ticket);
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100385 if (!ret)
386 res_bo = nvbo;
387 }
Ben Skeggs938c40e2010-10-12 09:54:54 +1000388 if (unlikely(ret)) {
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200389 ww_acquire_done(&op->ticket);
390 ww_acquire_fini(&op->ticket);
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100391 drm_gem_object_unreference_unlocked(gem);
Ben Skeggs938c40e2010-10-12 09:54:54 +1000392 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000393 NV_PRINTK(error, cli, "fail reserve\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000394 return ret;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000395 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000396 }
397
Ben Skeggsa1606a92010-02-12 10:27:35 +1000398 b->user_priv = (uint64_t)(unsigned long)nvbo;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000399 nvbo->reserved_by = file_priv;
400 nvbo->pbbo_index = i;
401 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
402 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
403 list_add_tail(&nvbo->entry, &op->both_list);
404 else
405 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
406 list_add_tail(&nvbo->entry, &op->vram_list);
407 else
408 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
409 list_add_tail(&nvbo->entry, &op->gart_list);
410 else {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000411 NV_PRINTK(error, cli, "invalid valid domains: 0x%08x\n",
Ben Skeggs6ee73862009-12-11 19:24:15 +1000412 b->valid_domains);
Ben Skeggs02088432010-01-21 15:03:23 +1000413 list_add_tail(&nvbo->entry, &op->both_list);
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200414 ww_acquire_done(&op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000415 validate_fini(op, NULL);
416 return -EINVAL;
417 }
Maarten Lankhorstc354c892013-01-15 14:57:20 +0100418 if (nvbo == res_bo)
419 goto retry;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000420 }
421
Maarten Lankhorstecff6652013-06-27 13:48:17 +0200422 ww_acquire_done(&op->ticket);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000423 return 0;
424}
425
426static int
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100427validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
428 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo,
429 uint64_t user_pbbo_ptr)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000430{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000431 struct nouveau_drm *drm = chan->drm;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000432 struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
433 (void __force __user *)(uintptr_t)user_pbbo_ptr;
434 struct nouveau_bo *nvbo;
435 int ret, relocs = 0;
436
437 list_for_each_entry(nvbo, list, entry) {
438 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
Ben Skeggs6ee73862009-12-11 19:24:15 +1000439
David Herrmann55fb74a2013-10-02 10:15:17 +0200440 ret = nouveau_gem_set_domain(&nvbo->gem, b->read_domains,
Ben Skeggs6ee73862009-12-11 19:24:15 +1000441 b->write_domains,
442 b->valid_domains);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000443 if (unlikely(ret)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000444 NV_PRINTK(error, cli, "fail set_domain\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000445 return ret;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000446 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000447
Maarten Lankhorst97a875c2012-11-28 11:25:44 +0000448 ret = nouveau_bo_validate(nvbo, true, false);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000449 if (unlikely(ret)) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000450 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000451 NV_PRINTK(error, cli, "fail ttm_validate\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000452 return ret;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000453 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000454
Maarten Lankhorst29ba89b2014-01-09 11:03:11 +0100455 ret = nouveau_fence_sync(nvbo, chan);
Ben Skeggs415e6182010-07-23 09:06:52 +1000456 if (unlikely(ret)) {
Maarten Lankhorst29ba89b2014-01-09 11:03:11 +0100457 if (ret != -ERESTARTSYS)
458 NV_PRINTK(error, cli, "fail post-validate sync\n");
Ben Skeggs415e6182010-07-23 09:06:52 +1000459 return ret;
460 }
461
Ben Skeggs967e7bd2014-08-10 04:10:22 +1000462 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
Ben Skeggsa3fcd0a2011-06-07 11:12:39 +1000463 if (nvbo->bo.offset == b->presumed.offset &&
464 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
465 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
466 (nvbo->bo.mem.mem_type == TTM_PL_TT &&
467 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
468 continue;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000469
Ben Skeggsa3fcd0a2011-06-07 11:12:39 +1000470 if (nvbo->bo.mem.mem_type == TTM_PL_TT)
471 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
472 else
473 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
474 b->presumed.offset = nvbo->bo.offset;
475 b->presumed.valid = 0;
476 relocs++;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000477
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100478 if (copy_to_user(&upbbo[nvbo->pbbo_index].presumed,
Ben Skeggsa3fcd0a2011-06-07 11:12:39 +1000479 &b->presumed, sizeof(b->presumed)))
480 return -EFAULT;
481 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000482 }
483
484 return relocs;
485}
486
487static int
488nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
489 struct drm_file *file_priv,
490 struct drm_nouveau_gem_pushbuf_bo *pbbo,
491 uint64_t user_buffers, int nr_buffers,
492 struct validate_op *op, int *apply_relocs)
493{
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100494 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000495 int ret, relocs = 0;
496
497 INIT_LIST_HEAD(&op->vram_list);
498 INIT_LIST_HEAD(&op->gart_list);
499 INIT_LIST_HEAD(&op->both_list);
500
Ben Skeggs6ee73862009-12-11 19:24:15 +1000501 if (nr_buffers == 0)
502 return 0;
503
504 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000505 if (unlikely(ret)) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000506 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000507 NV_PRINTK(error, cli, "validate_init\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000508 return ret;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000509 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000510
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100511 ret = validate_list(chan, cli, &op->vram_list, pbbo, user_buffers);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000512 if (unlikely(ret < 0)) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000513 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000514 NV_PRINTK(error, cli, "validate vram_list\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000515 validate_fini(op, NULL);
516 return ret;
517 }
518 relocs += ret;
519
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100520 ret = validate_list(chan, cli, &op->gart_list, pbbo, user_buffers);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000521 if (unlikely(ret < 0)) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000522 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000523 NV_PRINTK(error, cli, "validate gart_list\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000524 validate_fini(op, NULL);
525 return ret;
526 }
527 relocs += ret;
528
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100529 ret = validate_list(chan, cli, &op->both_list, pbbo, user_buffers);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000530 if (unlikely(ret < 0)) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000531 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000532 NV_PRINTK(error, cli, "validate both_list\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000533 validate_fini(op, NULL);
534 return ret;
535 }
536 relocs += ret;
537
538 *apply_relocs = relocs;
539 return 0;
540}
541
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200542static inline void
543u_free(void *addr)
544{
545 if (!is_vmalloc_addr(addr))
546 kfree(addr);
547 else
548 vfree(addr);
549}
550
Ben Skeggs6ee73862009-12-11 19:24:15 +1000551static inline void *
552u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
553{
554 void *mem;
555 void __user *userptr = (void __force __user *)(uintptr_t)user;
556
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200557 size *= nmemb;
558
559 mem = kmalloc(size, GFP_KERNEL | __GFP_NOWARN);
560 if (!mem)
561 mem = vmalloc(size);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000562 if (!mem)
563 return ERR_PTR(-ENOMEM);
564
Daniel Vetter1d6ac182013-12-11 11:34:44 +0100565 if (copy_from_user(mem, userptr, size)) {
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200566 u_free(mem);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000567 return ERR_PTR(-EFAULT);
568 }
569
570 return mem;
571}
572
573static int
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100574nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
Ben Skeggsa1606a92010-02-12 10:27:35 +1000575 struct drm_nouveau_gem_pushbuf *req,
576 struct drm_nouveau_gem_pushbuf_bo *bo)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000577{
578 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
Luca Barbieri12f735b2010-01-10 20:10:53 +0100579 int ret = 0;
580 unsigned i;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000581
Ben Skeggsa1606a92010-02-12 10:27:35 +1000582 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000583 if (IS_ERR(reloc))
584 return PTR_ERR(reloc);
585
Ben Skeggsa1606a92010-02-12 10:27:35 +1000586 for (i = 0; i < req->nr_relocs; i++) {
Ben Skeggs6ee73862009-12-11 19:24:15 +1000587 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
588 struct drm_nouveau_gem_pushbuf_bo *b;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000589 struct nouveau_bo *nvbo;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000590 uint32_t data;
591
Ben Skeggsa1606a92010-02-12 10:27:35 +1000592 if (unlikely(r->bo_index > req->nr_buffers)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000593 NV_PRINTK(error, cli, "reloc bo index invalid\n");
Ben Skeggs6ee73862009-12-11 19:24:15 +1000594 ret = -EINVAL;
595 break;
596 }
597
598 b = &bo[r->bo_index];
Ben Skeggsa1606a92010-02-12 10:27:35 +1000599 if (b->presumed.valid)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000600 continue;
601
Ben Skeggsa1606a92010-02-12 10:27:35 +1000602 if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000603 NV_PRINTK(error, cli, "reloc container bo index invalid\n");
Ben Skeggsa1606a92010-02-12 10:27:35 +1000604 ret = -EINVAL;
605 break;
606 }
607 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
608
609 if (unlikely(r->reloc_bo_offset + 4 >
610 nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000611 NV_PRINTK(error, cli, "reloc outside of bo\n");
Ben Skeggsa1606a92010-02-12 10:27:35 +1000612 ret = -EINVAL;
613 break;
614 }
615
616 if (!nvbo->kmap.virtual) {
617 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
618 &nvbo->kmap);
619 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000620 NV_PRINTK(error, cli, "failed kmap for reloc\n");
Ben Skeggsa1606a92010-02-12 10:27:35 +1000621 break;
622 }
623 nvbo->validate_mapped = true;
624 }
625
Ben Skeggs6ee73862009-12-11 19:24:15 +1000626 if (r->flags & NOUVEAU_GEM_RELOC_LOW)
Ben Skeggsa1606a92010-02-12 10:27:35 +1000627 data = b->presumed.offset + r->data;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000628 else
629 if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
Ben Skeggsa1606a92010-02-12 10:27:35 +1000630 data = (b->presumed.offset + r->data) >> 32;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000631 else
632 data = r->data;
633
634 if (r->flags & NOUVEAU_GEM_RELOC_OR) {
Ben Skeggsa1606a92010-02-12 10:27:35 +1000635 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
Ben Skeggs6ee73862009-12-11 19:24:15 +1000636 data |= r->tor;
637 else
638 data |= r->vor;
639 }
640
Ben Skeggsa1606a92010-02-12 10:27:35 +1000641 ret = ttm_bo_wait(&nvbo->bo, false, false, false);
642 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000643 NV_PRINTK(error, cli, "reloc wait_idle failed: %d\n", ret);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000644 break;
645 }
Ben Skeggsa1606a92010-02-12 10:27:35 +1000646
647 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000648 }
649
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200650 u_free(reloc);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000651 return ret;
652}
653
654int
655nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
656 struct drm_file *file_priv)
657{
Ben Skeggsebb945a2012-07-20 08:17:34 +1000658 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100659 struct nouveau_cli *cli = nouveau_cli(file_priv);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000660 struct nouveau_abi16_chan *temp;
661 struct nouveau_drm *drm = nouveau_drm(dev);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000662 struct drm_nouveau_gem_pushbuf *req = data;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000663 struct drm_nouveau_gem_pushbuf_push *push;
664 struct drm_nouveau_gem_pushbuf_bo *bo;
Ben Skeggsebb945a2012-07-20 08:17:34 +1000665 struct nouveau_channel *chan = NULL;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000666 struct validate_op op;
Francisco Jerez6e86e042010-07-03 18:36:39 +0200667 struct nouveau_fence *fence = NULL;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000668 int i, j, ret = 0, do_reloc = 0;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000669
Ben Skeggsebb945a2012-07-20 08:17:34 +1000670 if (unlikely(!abi16))
671 return -ENOMEM;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000672
Ben Skeggsebb945a2012-07-20 08:17:34 +1000673 list_for_each_entry(temp, &abi16->channels, head) {
Ben Skeggs0ad72862014-08-10 04:10:22 +1000674 if (temp->chan->object->handle == (NVDRM_CHAN | req->channel)) {
Ben Skeggsebb945a2012-07-20 08:17:34 +1000675 chan = temp->chan;
676 break;
677 }
678 }
679
680 if (!chan)
681 return nouveau_abi16_put(abi16, -ENOENT);
682
683 req->vram_available = drm->gem.vram_available;
684 req->gart_available = drm->gem.gart_available;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000685 if (unlikely(req->nr_push == 0))
686 goto out_next;
687
688 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000689 NV_PRINTK(error, cli, "pushbuf push count exceeds limit: %d max %d\n",
Ben Skeggsa1606a92010-02-12 10:27:35 +1000690 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000691 return nouveau_abi16_put(abi16, -EINVAL);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000692 }
693
Ben Skeggsa1606a92010-02-12 10:27:35 +1000694 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000695 NV_PRINTK(error, cli, "pushbuf bo count exceeds limit: %d max %d\n",
Ben Skeggsa1606a92010-02-12 10:27:35 +1000696 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000697 return nouveau_abi16_put(abi16, -EINVAL);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000698 }
699
700 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000701 NV_PRINTK(error, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
Ben Skeggsa1606a92010-02-12 10:27:35 +1000702 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000703 return nouveau_abi16_put(abi16, -EINVAL);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000704 }
705
706 push = u_memcpya(req->push, req->nr_push, sizeof(*push));
Ben Skeggsebb945a2012-07-20 08:17:34 +1000707 if (IS_ERR(push))
708 return nouveau_abi16_put(abi16, PTR_ERR(push));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000709
710 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
711 if (IS_ERR(bo)) {
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200712 u_free(push);
Ben Skeggsebb945a2012-07-20 08:17:34 +1000713 return nouveau_abi16_put(abi16, PTR_ERR(bo));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000714 }
715
Ben Skeggsaccf9492012-03-16 12:40:17 +1000716 /* Ensure all push buffers are on validate list */
Ben Skeggs415e6182010-07-23 09:06:52 +1000717 for (i = 0; i < req->nr_push; i++) {
718 if (push[i].bo_index >= req->nr_buffers) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000719 NV_PRINTK(error, cli, "push %d buffer not in list\n", i);
Ben Skeggs415e6182010-07-23 09:06:52 +1000720 ret = -EINVAL;
Marcin Slusarz7fa0cba22011-03-07 12:31:35 +0100721 goto out_prevalid;
Ben Skeggs415e6182010-07-23 09:06:52 +1000722 }
Ben Skeggs415e6182010-07-23 09:06:52 +1000723 }
724
Ben Skeggs6ee73862009-12-11 19:24:15 +1000725 /* Validate buffer list */
726 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
727 req->nr_buffers, &op, &do_reloc);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000728 if (ret) {
Ben Skeggs938c40e2010-10-12 09:54:54 +1000729 if (ret != -ERESTARTSYS)
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000730 NV_PRINTK(error, cli, "validate: %d\n", ret);
Marcin Slusarz7fa0cba22011-03-07 12:31:35 +0100731 goto out_prevalid;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000732 }
733
Ben Skeggs6ee73862009-12-11 19:24:15 +1000734 /* Apply any relocations that are required */
735 if (do_reloc) {
Marcin Slusarza84fa1a2013-02-03 22:02:47 +0100736 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, bo);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000737 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000738 NV_PRINTK(error, cli, "reloc apply: %d\n", ret);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000739 goto out;
740 }
741 }
742
Ben Skeggs9a391ad2010-02-11 16:37:26 +1000743 if (chan->dma.ib_max) {
Ben Skeggs5e120f62012-04-30 13:55:29 +1000744 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
Ben Skeggs9a391ad2010-02-11 16:37:26 +1000745 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000746 NV_PRINTK(error, cli, "nv50cal_space: %d\n", ret);
Ben Skeggs9a391ad2010-02-11 16:37:26 +1000747 goto out;
748 }
749
Ben Skeggsa1606a92010-02-12 10:27:35 +1000750 for (i = 0; i < req->nr_push; i++) {
751 struct nouveau_bo *nvbo = (void *)(unsigned long)
752 bo[push[i].bo_index].user_priv;
753
754 nv50_dma_push(chan, nvbo, push[i].offset,
755 push[i].length);
756 }
Ben Skeggs9a391ad2010-02-11 16:37:26 +1000757 } else
Ben Skeggs967e7bd2014-08-10 04:10:22 +1000758 if (drm->device.info.chipset >= 0x25) {
Ben Skeggsa1606a92010-02-12 10:27:35 +1000759 ret = RING_SPACE(chan, req->nr_push * 2);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000760 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000761 NV_PRINTK(error, cli, "cal_space: %d\n", ret);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000762 goto out;
763 }
Ben Skeggsa1606a92010-02-12 10:27:35 +1000764
765 for (i = 0; i < req->nr_push; i++) {
766 struct nouveau_bo *nvbo = (void *)(unsigned long)
767 bo[push[i].bo_index].user_priv;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000768
Ben Skeggs3a92d372012-08-04 05:46:01 +1000769 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000770 OUT_RING(chan, 0);
771 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000772 } else {
Ben Skeggsa1606a92010-02-12 10:27:35 +1000773 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000774 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000775 NV_PRINTK(error, cli, "jmp_space: %d\n", ret);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000776 goto out;
777 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000778
Ben Skeggsa1606a92010-02-12 10:27:35 +1000779 for (i = 0; i < req->nr_push; i++) {
780 struct nouveau_bo *nvbo = (void *)(unsigned long)
781 bo[push[i].bo_index].user_priv;
Ben Skeggsa1606a92010-02-12 10:27:35 +1000782 uint32_t cmd;
783
Ben Skeggsebb945a2012-07-20 08:17:34 +1000784 cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000785 cmd |= 0x20000000;
786 if (unlikely(cmd != req->suffix0)) {
787 if (!nvbo->kmap.virtual) {
788 ret = ttm_bo_kmap(&nvbo->bo, 0,
789 nvbo->bo.mem.
790 num_pages,
791 &nvbo->kmap);
792 if (ret) {
793 WIND_RING(chan);
794 goto out;
795 }
796 nvbo->validate_mapped = true;
797 }
798
799 nouveau_bo_wr32(nvbo, (push[i].offset +
800 push[i].length - 8) / 4, cmd);
801 }
802
Ben Skeggs3a92d372012-08-04 05:46:01 +1000803 OUT_RING(chan, 0x20000000 |
804 (nvbo->bo.offset + push[i].offset));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000805 OUT_RING(chan, 0);
Ben Skeggsa1606a92010-02-12 10:27:35 +1000806 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
807 OUT_RING(chan, 0);
808 }
Ben Skeggs6ee73862009-12-11 19:24:15 +1000809 }
810
Ben Skeggs264ce192013-02-14 13:43:21 +1000811 ret = nouveau_fence_new(chan, false, &fence);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000812 if (ret) {
Ben Skeggsfa2bade2014-08-10 04:10:22 +1000813 NV_PRINTK(error, cli, "error fencing pushbuf: %d\n", ret);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000814 WIND_RING(chan);
815 goto out;
816 }
817
818out:
Luca Barbieri234896a2010-01-06 04:02:45 +0100819 validate_fini(&op, fence);
Marcin Slusarz382d62e2010-10-20 21:50:24 +0200820 nouveau_fence_unref(&fence);
Marcin Slusarz7fa0cba22011-03-07 12:31:35 +0100821
822out_prevalid:
Maarten Lankhorstc8590742013-09-02 16:31:31 +0200823 u_free(bo);
824 u_free(push);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000825
826out_next:
Ben Skeggs9a391ad2010-02-11 16:37:26 +1000827 if (chan->dma.ib_max) {
828 req->suffix0 = 0x00000000;
829 req->suffix1 = 0x00000000;
830 } else
Ben Skeggs967e7bd2014-08-10 04:10:22 +1000831 if (drm->device.info.chipset >= 0x25) {
Ben Skeggs6ee73862009-12-11 19:24:15 +1000832 req->suffix0 = 0x00020000;
833 req->suffix1 = 0x00000000;
834 } else {
835 req->suffix0 = 0x20000000 |
Ben Skeggsebb945a2012-07-20 08:17:34 +1000836 (chan->push.vma.offset + ((chan->dma.cur + 2) << 2));
Ben Skeggs6ee73862009-12-11 19:24:15 +1000837 req->suffix1 = 0x00000000;
838 }
839
Ben Skeggsebb945a2012-07-20 08:17:34 +1000840 return nouveau_abi16_put(abi16, ret);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000841}
842
Ben Skeggs6ee73862009-12-11 19:24:15 +1000843static inline uint32_t
844domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain)
845{
846 uint32_t flags = 0;
847
848 if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
849 flags |= TTM_PL_FLAG_VRAM;
850 if (domain & NOUVEAU_GEM_DOMAIN_GART)
851 flags |= TTM_PL_FLAG_TT;
852
853 return flags;
854}
855
856int
Ben Skeggs6ee73862009-12-11 19:24:15 +1000857nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
858 struct drm_file *file_priv)
859{
860 struct drm_nouveau_gem_cpu_prep *req = data;
861 struct drm_gem_object *gem;
862 struct nouveau_bo *nvbo;
863 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
Maarten Lankhorstd0b3c3b2014-01-21 13:00:24 +0100864 int ret;
865 struct nouveau_fence *fence = NULL;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000866
Ben Skeggs6ee73862009-12-11 19:24:15 +1000867 gem = drm_gem_object_lookup(dev, file_priv, req->handle);
868 if (!gem)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100869 return -ENOENT;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000870 nvbo = nouveau_gem_object(gem);
871
Maarten Lankhorstd0b3c3b2014-01-21 13:00:24 +0100872 ret = ttm_bo_reserve(&nvbo->bo, true, false, false, NULL);
873 if (!ret) {
Maarten Lankhorstd0b3c3b2014-01-21 13:00:24 +0100874 ret = ttm_bo_wait(&nvbo->bo, true, true, true);
Maarten Lankhorstf2c24b82014-04-02 17:14:48 +0200875 if (!no_wait && ret) {
876 struct fence *excl;
877
878 excl = reservation_object_get_excl(nvbo->bo.resv);
879 fence = nouveau_fence_ref((struct nouveau_fence *)excl);
880 }
Maarten Lankhorstd0b3c3b2014-01-21 13:00:24 +0100881
882 ttm_bo_unreserve(&nvbo->bo);
883 }
Luca Barbieribc9025b2010-02-09 05:49:12 +0000884 drm_gem_object_unreference_unlocked(gem);
Maarten Lankhorstd0b3c3b2014-01-21 13:00:24 +0100885
886 if (fence) {
887 ret = nouveau_fence_wait(fence, true, no_wait);
888 nouveau_fence_unref(&fence);
889 }
890
Ben Skeggs6ee73862009-12-11 19:24:15 +1000891 return ret;
892}
893
894int
895nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
896 struct drm_file *file_priv)
897{
Ben Skeggs21e86c12010-10-11 11:48:45 +1000898 return 0;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000899}
900
901int
902nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
903 struct drm_file *file_priv)
904{
905 struct drm_nouveau_gem_info *req = data;
906 struct drm_gem_object *gem;
907 int ret;
908
Ben Skeggs6ee73862009-12-11 19:24:15 +1000909 gem = drm_gem_object_lookup(dev, file_priv, req->handle);
910 if (!gem)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100911 return -ENOENT;
Ben Skeggs6ee73862009-12-11 19:24:15 +1000912
Ben Skeggse758a312011-06-07 13:28:11 +1000913 ret = nouveau_gem_info(file_priv, gem, req);
Luca Barbieribc9025b2010-02-09 05:49:12 +0000914 drm_gem_object_unreference_unlocked(gem);
Ben Skeggs6ee73862009-12-11 19:24:15 +1000915 return ret;
916}
917